airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[int] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomPaginationStrategy(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomPaginationStrategy"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomPaginationStrategy"], 186 title="Class Name", 187 ) 188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 189 190 191class CustomRecordExtractor(BaseModel): 192 class Config: 193 extra = Extra.allow 194 195 type: Literal["CustomRecordExtractor"] 196 class_name: str = Field( 197 ..., 198 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 199 examples=["source_railz.components.MyCustomRecordExtractor"], 200 title="Class Name", 201 ) 202 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 203 204 205class CustomRecordFilter(BaseModel): 206 class Config: 207 extra = Extra.allow 208 209 type: Literal["CustomRecordFilter"] 210 class_name: str = Field( 211 ..., 212 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 213 examples=["source_railz.components.MyCustomCustomRecordFilter"], 214 title="Class Name", 215 ) 216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 217 218 219class CustomRequester(BaseModel): 220 class Config: 221 extra = Extra.allow 222 223 type: Literal["CustomRequester"] 224 class_name: str = Field( 225 ..., 226 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 227 examples=["source_railz.components.MyCustomRecordExtractor"], 228 title="Class Name", 229 ) 230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 231 232 233class CustomRetriever(BaseModel): 234 class Config: 235 extra = Extra.allow 236 237 type: Literal["CustomRetriever"] 238 class_name: str = Field( 239 ..., 240 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 241 examples=["source_railz.components.MyCustomRetriever"], 242 title="Class Name", 243 ) 244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 245 246 247class CustomPartitionRouter(BaseModel): 248 class Config: 249 extra = Extra.allow 250 251 type: Literal["CustomPartitionRouter"] 252 class_name: str = Field( 253 ..., 254 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 255 examples=["source_railz.components.MyCustomPartitionRouter"], 256 title="Class Name", 257 ) 258 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 259 260 261class CustomSchemaLoader(BaseModel): 262 class Config: 263 extra = Extra.allow 264 265 type: Literal["CustomSchemaLoader"] 266 class_name: str = Field( 267 ..., 268 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 269 examples=["source_railz.components.MyCustomSchemaLoader"], 270 title="Class Name", 271 ) 272 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 273 274 275class CustomSchemaNormalization(BaseModel): 276 class Config: 277 extra = Extra.allow 278 279 type: Literal["CustomSchemaNormalization"] 280 class_name: str = Field( 281 ..., 282 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 283 examples=[ 284 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 285 ], 286 title="Class Name", 287 ) 288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 289 290 291class CustomStateMigration(BaseModel): 292 class Config: 293 extra = Extra.allow 294 295 type: Literal["CustomStateMigration"] 296 class_name: str = Field( 297 ..., 298 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 299 examples=["source_railz.components.MyCustomStateMigration"], 300 title="Class Name", 301 ) 302 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 303 304 305class CustomTransformation(BaseModel): 306 class Config: 307 extra = Extra.allow 308 309 type: Literal["CustomTransformation"] 310 class_name: str = Field( 311 ..., 312 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 313 examples=["source_railz.components.MyCustomTransformation"], 314 title="Class Name", 315 ) 316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 317 318 319class LegacyToPerPartitionStateMigration(BaseModel): 320 class Config: 321 extra = Extra.allow 322 323 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 324 325 326class Clamping(BaseModel): 327 target: str = Field( 328 ..., 329 description="The period of time that datetime windows will be clamped by", 330 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 331 title="Target", 332 ) 333 target_details: Optional[Dict[str, Any]] = None 334 335 336class Algorithm(Enum): 337 HS256 = "HS256" 338 HS384 = "HS384" 339 HS512 = "HS512" 340 ES256 = "ES256" 341 ES256K = "ES256K" 342 ES384 = "ES384" 343 ES512 = "ES512" 344 RS256 = "RS256" 345 RS384 = "RS384" 346 RS512 = "RS512" 347 PS256 = "PS256" 348 PS384 = "PS384" 349 PS512 = "PS512" 350 EdDSA = "EdDSA" 351 352 353class JwtHeaders(BaseModel): 354 class Config: 355 extra = Extra.forbid 356 357 kid: Optional[str] = Field( 358 None, 359 description="Private key ID for user account.", 360 examples=["{{ config['kid'] }}"], 361 title="Key Identifier", 362 ) 363 typ: Optional[str] = Field( 364 "JWT", 365 description="The media type of the complete JWT.", 366 examples=["JWT"], 367 title="Type", 368 ) 369 cty: Optional[str] = Field( 370 None, 371 description="Content type of JWT header.", 372 examples=["JWT"], 373 title="Content Type", 374 ) 375 376 377class JwtPayload(BaseModel): 378 class Config: 379 extra = Extra.forbid 380 381 iss: Optional[str] = Field( 382 None, 383 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 384 examples=["{{ config['iss'] }}"], 385 title="Issuer", 386 ) 387 sub: Optional[str] = Field( 388 None, 389 description="The subject of the JWT. Commonly defined by the API.", 390 title="Subject", 391 ) 392 aud: Optional[str] = Field( 393 None, 394 description="The recipient that the JWT is intended for. Commonly defined by the API.", 395 examples=["appstoreconnect-v1"], 396 title="Audience", 397 ) 398 399 400class JwtAuthenticator(BaseModel): 401 type: Literal["JwtAuthenticator"] 402 secret_key: str = Field( 403 ..., 404 description="Secret used to sign the JSON web token.", 405 examples=["{{ config['secret_key'] }}"], 406 title="Secret Key", 407 ) 408 base64_encode_secret_key: Optional[bool] = Field( 409 False, 410 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 411 title="Base64-encode Secret Key", 412 ) 413 algorithm: Algorithm = Field( 414 ..., 415 description="Algorithm used to sign the JSON web token.", 416 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 417 title="Algorithm", 418 ) 419 token_duration: Optional[int] = Field( 420 1200, 421 description="The amount of time in seconds a JWT token can be valid after being issued.", 422 examples=[1200, 3600], 423 title="Token Duration", 424 ) 425 header_prefix: Optional[str] = Field( 426 None, 427 description="The prefix to be used within the Authentication header.", 428 examples=["Bearer", "Basic"], 429 title="Header Prefix", 430 ) 431 jwt_headers: Optional[JwtHeaders] = Field( 432 None, 433 description="JWT headers used when signing JSON web token.", 434 title="JWT Headers", 435 ) 436 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 437 None, 438 description="Additional headers to be included with the JWT headers object.", 439 title="Additional JWT Headers", 440 ) 441 jwt_payload: Optional[JwtPayload] = Field( 442 None, 443 description="JWT Payload used when signing JSON web token.", 444 title="JWT Payload", 445 ) 446 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 447 None, 448 description="Additional properties to be added to the JWT payload.", 449 title="Additional JWT Payload Properties", 450 ) 451 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 452 453 454class RefreshTokenUpdater(BaseModel): 455 refresh_token_name: Optional[str] = Field( 456 "refresh_token", 457 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 458 examples=["refresh_token"], 459 title="Refresh Token Property Name", 460 ) 461 access_token_config_path: Optional[List[str]] = Field( 462 ["credentials", "access_token"], 463 description="Config path to the access token. Make sure the field actually exists in the config.", 464 examples=[["credentials", "access_token"], ["access_token"]], 465 title="Config Path To Access Token", 466 ) 467 refresh_token_config_path: Optional[List[str]] = Field( 468 ["credentials", "refresh_token"], 469 description="Config path to the access token. Make sure the field actually exists in the config.", 470 examples=[["credentials", "refresh_token"], ["refresh_token"]], 471 title="Config Path To Refresh Token", 472 ) 473 token_expiry_date_config_path: Optional[List[str]] = Field( 474 ["credentials", "token_expiry_date"], 475 description="Config path to the expiry date. Make sure actually exists in the config.", 476 examples=[["credentials", "token_expiry_date"]], 477 title="Config Path To Expiry Date", 478 ) 479 refresh_token_error_status_codes: Optional[List[int]] = Field( 480 [], 481 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 482 examples=[[400, 500]], 483 title="Refresh Token Error Status Codes", 484 ) 485 refresh_token_error_key: Optional[str] = Field( 486 "", 487 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 488 examples=["error"], 489 title="Refresh Token Error Key", 490 ) 491 refresh_token_error_values: Optional[List[str]] = Field( 492 [], 493 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 494 examples=[["invalid_grant", "invalid_permissions"]], 495 title="Refresh Token Error Values", 496 ) 497 498 499class OAuthAuthenticator(BaseModel): 500 type: Literal["OAuthAuthenticator"] 501 client_id_name: Optional[str] = Field( 502 "client_id", 503 description="The name of the property to use to refresh the `access_token`.", 504 examples=["custom_app_id"], 505 title="Client ID Property Name", 506 ) 507 client_id: Optional[str] = Field( 508 None, 509 description="The OAuth client ID. Fill it in the user inputs.", 510 examples=[ 511 "{{ config['client_id'] }}", 512 "{{ config['credentials']['client_id }}", 513 ], 514 title="Client ID", 515 ) 516 client_secret_name: Optional[str] = Field( 517 "client_secret", 518 description="The name of the property to use to refresh the `access_token`.", 519 examples=["custom_app_secret"], 520 title="Client Secret Property Name", 521 ) 522 client_secret: Optional[str] = Field( 523 None, 524 description="The OAuth client secret. Fill it in the user inputs.", 525 examples=[ 526 "{{ config['client_secret'] }}", 527 "{{ config['credentials']['client_secret }}", 528 ], 529 title="Client Secret", 530 ) 531 refresh_token_name: Optional[str] = Field( 532 "refresh_token", 533 description="The name of the property to use to refresh the `access_token`.", 534 examples=["custom_app_refresh_value"], 535 title="Refresh Token Property Name", 536 ) 537 refresh_token: Optional[str] = Field( 538 None, 539 description="Credential artifact used to get a new access token.", 540 examples=[ 541 "{{ config['refresh_token'] }}", 542 "{{ config['credentials]['refresh_token'] }}", 543 ], 544 title="Refresh Token", 545 ) 546 token_refresh_endpoint: Optional[str] = Field( 547 None, 548 description="The full URL to call to obtain a new access token.", 549 examples=["https://connect.squareup.com/oauth2/token"], 550 title="Token Refresh Endpoint", 551 ) 552 access_token_name: Optional[str] = Field( 553 "access_token", 554 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 555 examples=["access_token"], 556 title="Access Token Property Name", 557 ) 558 access_token_value: Optional[str] = Field( 559 None, 560 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 561 examples=["secret_access_token_value"], 562 title="Access Token Value", 563 ) 564 expires_in_name: Optional[str] = Field( 565 "expires_in", 566 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 567 examples=["expires_in"], 568 title="Token Expiry Property Name", 569 ) 570 grant_type_name: Optional[str] = Field( 571 "grant_type", 572 description="The name of the property to use to refresh the `access_token`.", 573 examples=["custom_grant_type"], 574 title="Grant Type Property Name", 575 ) 576 grant_type: Optional[str] = Field( 577 "refresh_token", 578 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 579 examples=["refresh_token", "client_credentials"], 580 title="Grant Type", 581 ) 582 refresh_request_body: Optional[Dict[str, Any]] = Field( 583 None, 584 description="Body of the request sent to get a new access token.", 585 examples=[ 586 { 587 "applicationId": "{{ config['application_id'] }}", 588 "applicationSecret": "{{ config['application_secret'] }}", 589 "token": "{{ config['token'] }}", 590 } 591 ], 592 title="Refresh Request Body", 593 ) 594 refresh_request_headers: Optional[Dict[str, Any]] = Field( 595 None, 596 description="Headers of the request sent to get a new access token.", 597 examples=[ 598 { 599 "Authorization": "<AUTH_TOKEN>", 600 "Content-Type": "application/x-www-form-urlencoded", 601 } 602 ], 603 title="Refresh Request Headers", 604 ) 605 scopes: Optional[List[str]] = Field( 606 None, 607 description="List of scopes that should be granted to the access token.", 608 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 609 title="Scopes", 610 ) 611 token_expiry_date: Optional[str] = Field( 612 None, 613 description="The access token expiry date.", 614 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 615 title="Token Expiry Date", 616 ) 617 token_expiry_date_format: Optional[str] = Field( 618 None, 619 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 620 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 621 title="Token Expiry Date Format", 622 ) 623 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 624 None, 625 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 626 title="Refresh Token Updater", 627 ) 628 profile_assertion: Optional[JwtAuthenticator] = Field( 629 None, 630 description="The authenticator being used to authenticate the client authenticator.", 631 title="Profile Assertion", 632 ) 633 use_profile_assertion: Optional[bool] = Field( 634 False, 635 description="Enable using profile assertion as a flow for OAuth authorization.", 636 title="Use Profile Assertion", 637 ) 638 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 639 640 641class Rate(BaseModel): 642 class Config: 643 extra = Extra.allow 644 645 limit: Union[int, str] = Field( 646 ..., 647 description="The maximum number of calls allowed within the interval.", 648 title="Limit", 649 ) 650 interval: str = Field( 651 ..., 652 description="The time interval for the rate limit.", 653 examples=["PT1H", "P1D"], 654 title="Interval", 655 ) 656 657 658class HttpRequestRegexMatcher(BaseModel): 659 class Config: 660 extra = Extra.allow 661 662 method: Optional[str] = Field( 663 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 664 ) 665 url_base: Optional[str] = Field( 666 None, 667 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 668 title="URL Base", 669 ) 670 url_path_pattern: Optional[str] = Field( 671 None, 672 description="A regular expression pattern to match the URL path.", 673 title="URL Path Pattern", 674 ) 675 params: Optional[Dict[str, Any]] = Field( 676 None, description="The query parameters to match.", title="Parameters" 677 ) 678 headers: Optional[Dict[str, Any]] = Field( 679 None, description="The headers to match.", title="Headers" 680 ) 681 682 683class DpathExtractor(BaseModel): 684 type: Literal["DpathExtractor"] 685 field_path: List[str] = Field( 686 ..., 687 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 688 examples=[ 689 ["data"], 690 ["data", "records"], 691 ["data", "{{ parameters.name }}"], 692 ["data", "*", "record"], 693 ], 694 title="Field Path", 695 ) 696 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 697 698 699class ResponseToFileExtractor(BaseModel): 700 type: Literal["ResponseToFileExtractor"] 701 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 702 703 704class ExponentialBackoffStrategy(BaseModel): 705 type: Literal["ExponentialBackoffStrategy"] 706 factor: Optional[Union[float, str]] = Field( 707 5, 708 description="Multiplicative constant applied on each retry.", 709 examples=[5, 5.5, "10"], 710 title="Factor", 711 ) 712 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 713 714 715class GroupByKeyMergeStrategy(BaseModel): 716 type: Literal["GroupByKeyMergeStrategy"] 717 key: Union[str, List[str]] = Field( 718 ..., 719 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 720 examples=["id", ["parent_id", "end_date"]], 721 title="Key", 722 ) 723 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 724 725 726class SessionTokenRequestBearerAuthenticator(BaseModel): 727 type: Literal["Bearer"] 728 729 730class HttpMethod(Enum): 731 GET = "GET" 732 POST = "POST" 733 734 735class Action(Enum): 736 SUCCESS = "SUCCESS" 737 FAIL = "FAIL" 738 RETRY = "RETRY" 739 IGNORE = "IGNORE" 740 RATE_LIMITED = "RATE_LIMITED" 741 742 743class FailureType(Enum): 744 system_error = "system_error" 745 config_error = "config_error" 746 transient_error = "transient_error" 747 748 749class HttpResponseFilter(BaseModel): 750 type: Literal["HttpResponseFilter"] 751 action: Optional[Action] = Field( 752 None, 753 description="Action to execute if a response matches the filter.", 754 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 755 title="Action", 756 ) 757 failure_type: Optional[FailureType] = Field( 758 None, 759 description="Failure type of traced exception if a response matches the filter.", 760 examples=["system_error", "config_error", "transient_error"], 761 title="Failure Type", 762 ) 763 error_message: Optional[str] = Field( 764 None, 765 description="Error Message to display if the response matches the filter.", 766 title="Error Message", 767 ) 768 error_message_contains: Optional[str] = Field( 769 None, 770 description="Match the response if its error message contains the substring.", 771 example=["This API operation is not enabled for this site"], 772 title="Error Message Substring", 773 ) 774 http_codes: Optional[List[int]] = Field( 775 None, 776 description="Match the response if its HTTP code is included in this list.", 777 examples=[[420, 429], [500]], 778 title="HTTP Codes", 779 unique_items=True, 780 ) 781 predicate: Optional[str] = Field( 782 None, 783 description="Match the response if the predicate evaluates to true.", 784 examples=[ 785 "{{ 'Too much requests' in response }}", 786 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 787 ], 788 title="Predicate", 789 ) 790 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 791 792 793class ComplexFieldType(BaseModel): 794 field_type: str 795 items: Optional[Union[str, ComplexFieldType]] = None 796 797 798class TypesMap(BaseModel): 799 target_type: Union[str, List[str], ComplexFieldType] 800 current_type: Union[str, List[str]] 801 condition: Optional[str] = None 802 803 804class SchemaTypeIdentifier(BaseModel): 805 type: Optional[Literal["SchemaTypeIdentifier"]] = None 806 schema_pointer: Optional[List[str]] = Field( 807 [], 808 description="List of nested fields defining the schema field path to extract. Defaults to [].", 809 title="Schema Path", 810 ) 811 key_pointer: List[str] = Field( 812 ..., 813 description="List of potentially nested fields describing the full path of the field key to extract.", 814 title="Key Path", 815 ) 816 type_pointer: Optional[List[str]] = Field( 817 None, 818 description="List of potentially nested fields describing the full path of the field type to extract.", 819 title="Type Path", 820 ) 821 types_mapping: Optional[List[TypesMap]] = None 822 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 823 824 825class InlineSchemaLoader(BaseModel): 826 type: Literal["InlineSchemaLoader"] 827 schema_: Optional[Dict[str, Any]] = Field( 828 None, 829 alias="schema", 830 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 831 title="Schema", 832 ) 833 834 835class JsonFileSchemaLoader(BaseModel): 836 type: Literal["JsonFileSchemaLoader"] 837 file_path: Optional[str] = Field( 838 None, 839 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 840 example=["./schemas/users.json"], 841 title="File Path", 842 ) 843 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 844 845 846class JsonDecoder(BaseModel): 847 type: Literal["JsonDecoder"] 848 849 850class JsonlDecoder(BaseModel): 851 type: Literal["JsonlDecoder"] 852 853 854class KeysToLower(BaseModel): 855 type: Literal["KeysToLower"] 856 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 857 858 859class KeysToSnakeCase(BaseModel): 860 type: Literal["KeysToSnakeCase"] 861 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 862 863 864class FlattenFields(BaseModel): 865 type: Literal["FlattenFields"] 866 flatten_lists: Optional[bool] = Field( 867 True, 868 description="Whether to flatten lists or leave it as is. Default is True.", 869 title="Flatten Lists", 870 ) 871 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 872 873 874class KeyTransformation(BaseModel): 875 type: Literal["KeyTransformation"] 876 prefix: Optional[str] = Field( 877 None, 878 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 879 examples=["flattened_"], 880 title="Key Prefix", 881 ) 882 suffix: Optional[str] = Field( 883 None, 884 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 885 examples=["_flattened"], 886 title="Key Suffix", 887 ) 888 889 890class DpathFlattenFields(BaseModel): 891 type: Literal["DpathFlattenFields"] 892 field_path: List[str] = Field( 893 ..., 894 description="A path to field that needs to be flattened.", 895 examples=[["data"], ["data", "*", "field"]], 896 title="Field Path", 897 ) 898 delete_origin_value: Optional[bool] = Field( 899 None, 900 description="Whether to delete the origin value or keep it. Default is False.", 901 title="Delete Origin Value", 902 ) 903 replace_record: Optional[bool] = Field( 904 None, 905 description="Whether to replace the origin record or not. Default is False.", 906 title="Replace Origin Record", 907 ) 908 key_transformation: Optional[KeyTransformation] = Field( 909 None, 910 description="Transformation for object keys. If not provided, original key will be used.", 911 title="Key transformation", 912 ) 913 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 914 915 916class KeysReplace(BaseModel): 917 type: Literal["KeysReplace"] 918 old: str = Field( 919 ..., 920 description="Old value to replace.", 921 examples=[ 922 " ", 923 "{{ record.id }}", 924 "{{ config['id'] }}", 925 "{{ stream_slice['id'] }}", 926 ], 927 title="Old value", 928 ) 929 new: str = Field( 930 ..., 931 description="New value to set.", 932 examples=[ 933 "_", 934 "{{ record.id }}", 935 "{{ config['id'] }}", 936 "{{ stream_slice['id'] }}", 937 ], 938 title="New value", 939 ) 940 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 941 942 943class IterableDecoder(BaseModel): 944 type: Literal["IterableDecoder"] 945 946 947class XmlDecoder(BaseModel): 948 type: Literal["XmlDecoder"] 949 950 951class CustomDecoder(BaseModel): 952 class Config: 953 extra = Extra.allow 954 955 type: Literal["CustomDecoder"] 956 class_name: str = Field( 957 ..., 958 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 959 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 960 title="Class Name", 961 ) 962 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 963 964 965class MinMaxDatetime(BaseModel): 966 type: Literal["MinMaxDatetime"] 967 datetime: str = Field( 968 ..., 969 description="Datetime value.", 970 examples=[ 971 "2021-01-01", 972 "2021-01-01T00:00:00Z", 973 "{{ config['start_time'] }}", 974 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 975 ], 976 title="Datetime", 977 ) 978 datetime_format: Optional[str] = Field( 979 "", 980 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 981 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 982 title="Datetime Format", 983 ) 984 max_datetime: Optional[str] = Field( 985 None, 986 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 987 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 988 title="Max Datetime", 989 ) 990 min_datetime: Optional[str] = Field( 991 None, 992 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 993 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 994 title="Min Datetime", 995 ) 996 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 997 998 999class NoAuth(BaseModel): 1000 type: Literal["NoAuth"] 1001 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1002 1003 1004class NoPagination(BaseModel): 1005 type: Literal["NoPagination"] 1006 1007 1008class State(BaseModel): 1009 class Config: 1010 extra = Extra.allow 1011 1012 min: int 1013 max: int 1014 1015 1016class OauthConnectorInputSpecification(BaseModel): 1017 class Config: 1018 extra = Extra.allow 1019 1020 consent_url: str = Field( 1021 ..., 1022 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1023 examples=[ 1024 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1025 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1026 ], 1027 title="Consent URL", 1028 ) 1029 scope: Optional[str] = Field( 1030 None, 1031 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1032 examples=["user:read user:read_orders workspaces:read"], 1033 title="Scopes", 1034 ) 1035 access_token_url: str = Field( 1036 ..., 1037 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1038 examples=[ 1039 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1040 ], 1041 title="Access Token URL", 1042 ) 1043 access_token_headers: Optional[Dict[str, Any]] = Field( 1044 None, 1045 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1046 examples=[ 1047 { 1048 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1049 } 1050 ], 1051 title="Access Token Headers", 1052 ) 1053 access_token_params: Optional[Dict[str, Any]] = Field( 1054 None, 1055 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1056 examples=[ 1057 { 1058 "{{ auth_code_key }}": "{{ auth_code_value }}", 1059 "{{ client_id_key }}": "{{ client_id_value }}", 1060 "{{ client_secret_key }}": "{{ client_secret_value }}", 1061 } 1062 ], 1063 title="Access Token Query Params (Json Encoded)", 1064 ) 1065 extract_output: Optional[List[str]] = Field( 1066 None, 1067 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1068 examples=[["access_token", "refresh_token", "other_field"]], 1069 title="Extract Output", 1070 ) 1071 state: Optional[State] = Field( 1072 None, 1073 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1074 examples=[{"min": 7, "max": 128}], 1075 title="Configurable State Query Param", 1076 ) 1077 client_id_key: Optional[str] = Field( 1078 None, 1079 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1080 examples=["my_custom_client_id_key_name"], 1081 title="Client ID Key Override", 1082 ) 1083 client_secret_key: Optional[str] = Field( 1084 None, 1085 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1086 examples=["my_custom_client_secret_key_name"], 1087 title="Client Secret Key Override", 1088 ) 1089 scope_key: Optional[str] = Field( 1090 None, 1091 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1092 examples=["my_custom_scope_key_key_name"], 1093 title="Scopes Key Override", 1094 ) 1095 state_key: Optional[str] = Field( 1096 None, 1097 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1098 examples=["my_custom_state_key_key_name"], 1099 title="State Key Override", 1100 ) 1101 auth_code_key: Optional[str] = Field( 1102 None, 1103 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1104 examples=["my_custom_auth_code_key_name"], 1105 title="Auth Code Key Override", 1106 ) 1107 redirect_uri_key: Optional[str] = Field( 1108 None, 1109 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1110 examples=["my_custom_redirect_uri_key_name"], 1111 title="Redirect URI Key Override", 1112 ) 1113 1114 1115class OAuthConfigSpecification(BaseModel): 1116 class Config: 1117 extra = Extra.allow 1118 1119 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1120 None, 1121 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1122 examples=[ 1123 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1124 { 1125 "app_id": { 1126 "type": "string", 1127 "path_in_connector_config": ["info", "app_id"], 1128 } 1129 }, 1130 ], 1131 title="OAuth user input", 1132 ) 1133 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1134 None, 1135 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1136 title="DeclarativeOAuth Connector Specification", 1137 ) 1138 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1139 None, 1140 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1141 examples=[ 1142 { 1143 "refresh_token": { 1144 "type": "string,", 1145 "path_in_connector_config": ["credentials", "refresh_token"], 1146 } 1147 } 1148 ], 1149 title="OAuth output specification", 1150 ) 1151 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1152 None, 1153 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1154 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1155 title="OAuth input specification", 1156 ) 1157 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1158 None, 1159 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1160 examples=[ 1161 { 1162 "client_id": { 1163 "type": "string,", 1164 "path_in_connector_config": ["credentials", "client_id"], 1165 }, 1166 "client_secret": { 1167 "type": "string,", 1168 "path_in_connector_config": ["credentials", "client_secret"], 1169 }, 1170 } 1171 ], 1172 title="OAuth server output specification", 1173 ) 1174 1175 1176class OffsetIncrement(BaseModel): 1177 type: Literal["OffsetIncrement"] 1178 page_size: Optional[Union[int, str]] = Field( 1179 None, 1180 description="The number of records to include in each pages.", 1181 examples=[100, "{{ config['page_size'] }}"], 1182 title="Limit", 1183 ) 1184 inject_on_first_request: Optional[bool] = Field( 1185 False, 1186 description="Using the `offset` with value `0` during the first request", 1187 title="Inject Offset on First Request", 1188 ) 1189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1190 1191 1192class PageIncrement(BaseModel): 1193 type: Literal["PageIncrement"] 1194 page_size: Optional[Union[int, str]] = Field( 1195 None, 1196 description="The number of records to include in each pages.", 1197 examples=[100, "100", "{{ config['page_size'] }}"], 1198 title="Page Size", 1199 ) 1200 start_from_page: Optional[int] = Field( 1201 0, 1202 description="Index of the first page to request.", 1203 examples=[0, 1], 1204 title="Start From Page", 1205 ) 1206 inject_on_first_request: Optional[bool] = Field( 1207 False, 1208 description="Using the `page number` with value defined by `start_from_page` during the first request", 1209 title="Inject Page Number on First Request", 1210 ) 1211 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1212 1213 1214class PrimaryKey(BaseModel): 1215 __root__: Union[str, List[str], List[List[str]]] = Field( 1216 ..., 1217 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1218 examples=["id", ["code", "type"]], 1219 title="Primary Key", 1220 ) 1221 1222 1223class PropertyLimitType(Enum): 1224 characters = "characters" 1225 property_count = "property_count" 1226 1227 1228class PropertyChunking(BaseModel): 1229 type: Literal["PropertyChunking"] 1230 property_limit_type: PropertyLimitType = Field( 1231 ..., 1232 description="The type used to determine the maximum number of properties per chunk", 1233 title="Property Limit Type", 1234 ) 1235 property_limit: Optional[int] = Field( 1236 None, 1237 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1238 title="Property Limit", 1239 ) 1240 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1241 None, 1242 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1243 title="Record Merge Strategy", 1244 ) 1245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1246 1247 1248class RecordFilter(BaseModel): 1249 type: Literal["RecordFilter"] 1250 condition: Optional[str] = Field( 1251 "", 1252 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1253 examples=[ 1254 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1255 "{{ record.status in ['active', 'expired'] }}", 1256 ], 1257 title="Condition", 1258 ) 1259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1260 1261 1262class SchemaNormalization(Enum): 1263 Default = "Default" 1264 None_ = "None" 1265 1266 1267class RemoveFields(BaseModel): 1268 type: Literal["RemoveFields"] 1269 condition: Optional[str] = Field( 1270 "", 1271 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1272 examples=[ 1273 "{{ property|string == '' }}", 1274 "{{ property is integer }}", 1275 "{{ property|length > 5 }}", 1276 "{{ property == 'some_string_to_match' }}", 1277 ], 1278 ) 1279 field_pointers: List[List[str]] = Field( 1280 ..., 1281 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1282 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1283 title="Field Paths", 1284 ) 1285 1286 1287class RequestPath(BaseModel): 1288 type: Literal["RequestPath"] 1289 1290 1291class InjectInto(Enum): 1292 request_parameter = "request_parameter" 1293 header = "header" 1294 body_data = "body_data" 1295 body_json = "body_json" 1296 1297 1298class RequestOption(BaseModel): 1299 type: Literal["RequestOption"] 1300 inject_into: InjectInto = Field( 1301 ..., 1302 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1303 examples=["request_parameter", "header", "body_data", "body_json"], 1304 title="Inject Into", 1305 ) 1306 field_name: Optional[str] = Field( 1307 None, 1308 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1309 examples=["segment_id"], 1310 title="Field Name", 1311 ) 1312 field_path: Optional[List[str]] = Field( 1313 None, 1314 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1315 examples=[["data", "viewer", "id"]], 1316 title="Field Path", 1317 ) 1318 1319 1320class Schemas(BaseModel): 1321 pass 1322 1323 class Config: 1324 extra = Extra.allow 1325 1326 1327class LegacySessionTokenAuthenticator(BaseModel): 1328 type: Literal["LegacySessionTokenAuthenticator"] 1329 header: str = Field( 1330 ..., 1331 description="The name of the session token header that will be injected in the request", 1332 examples=["X-Session"], 1333 title="Session Request Header", 1334 ) 1335 login_url: str = Field( 1336 ..., 1337 description="Path of the login URL (do not include the base URL)", 1338 examples=["session"], 1339 title="Login Path", 1340 ) 1341 session_token: Optional[str] = Field( 1342 None, 1343 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1344 example=["{{ config['session_token'] }}"], 1345 title="Session Token", 1346 ) 1347 session_token_response_key: str = Field( 1348 ..., 1349 description="Name of the key of the session token to be extracted from the response", 1350 examples=["id"], 1351 title="Response Token Response Key", 1352 ) 1353 username: Optional[str] = Field( 1354 None, 1355 description="Username used to authenticate and obtain a session token", 1356 examples=[" {{ config['username'] }}"], 1357 title="Username", 1358 ) 1359 password: Optional[str] = Field( 1360 "", 1361 description="Password used to authenticate and obtain a session token", 1362 examples=["{{ config['password'] }}", ""], 1363 title="Password", 1364 ) 1365 validate_session_url: str = Field( 1366 ..., 1367 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1368 examples=["user/current"], 1369 title="Validate Session Path", 1370 ) 1371 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1372 1373 1374class CsvDecoder(BaseModel): 1375 type: Literal["CsvDecoder"] 1376 encoding: Optional[str] = "utf-8" 1377 delimiter: Optional[str] = "," 1378 set_values_to_none: Optional[List[str]] = None 1379 1380 1381class AsyncJobStatusMap(BaseModel): 1382 type: Optional[Literal["AsyncJobStatusMap"]] = None 1383 running: List[str] 1384 completed: List[str] 1385 failed: List[str] 1386 timeout: List[str] 1387 1388 1389class ValueType(Enum): 1390 string = "string" 1391 number = "number" 1392 integer = "integer" 1393 boolean = "boolean" 1394 1395 1396class WaitTimeFromHeader(BaseModel): 1397 type: Literal["WaitTimeFromHeader"] 1398 header: str = Field( 1399 ..., 1400 description="The name of the response header defining how long to wait before retrying.", 1401 examples=["Retry-After"], 1402 title="Response Header Name", 1403 ) 1404 regex: Optional[str] = Field( 1405 None, 1406 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1407 examples=["([-+]?\\d+)"], 1408 title="Extraction Regex", 1409 ) 1410 max_waiting_time_in_seconds: Optional[float] = Field( 1411 None, 1412 description="Given the value extracted from the header is greater than this value, stop the stream.", 1413 examples=[3600], 1414 title="Max Waiting Time in Seconds", 1415 ) 1416 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1417 1418 1419class WaitUntilTimeFromHeader(BaseModel): 1420 type: Literal["WaitUntilTimeFromHeader"] 1421 header: str = Field( 1422 ..., 1423 description="The name of the response header defining how long to wait before retrying.", 1424 examples=["wait_time"], 1425 title="Response Header", 1426 ) 1427 min_wait: Optional[Union[float, str]] = Field( 1428 None, 1429 description="Minimum time to wait before retrying.", 1430 examples=[10, "60"], 1431 title="Minimum Wait Time", 1432 ) 1433 regex: Optional[str] = Field( 1434 None, 1435 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1436 examples=["([-+]?\\d+)"], 1437 title="Extraction Regex", 1438 ) 1439 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1440 1441 1442class ComponentMappingDefinition(BaseModel): 1443 type: Literal["ComponentMappingDefinition"] 1444 field_path: List[str] = Field( 1445 ..., 1446 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1447 examples=[ 1448 ["name"], 1449 ["retriever", "requester", "url"], 1450 ["retriever", "requester", "{{ components_values.field }}"], 1451 ["*", "**", "name"], 1452 ], 1453 title="Field Path", 1454 ) 1455 value: str = Field( 1456 ..., 1457 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1458 examples=[ 1459 "{{ components_values['updates'] }}", 1460 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1461 "{{ config['segment_id'] }}", 1462 "{{ stream_slice['parent_id'] }}", 1463 "{{ stream_slice['extra_fields']['name'] }}", 1464 ], 1465 title="Value", 1466 ) 1467 value_type: Optional[ValueType] = Field( 1468 None, 1469 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1470 title="Value Type", 1471 ) 1472 create_or_update: Optional[bool] = Field( 1473 False, 1474 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1475 title="Create or Update", 1476 ) 1477 condition: Optional[str] = Field( 1478 None, 1479 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1480 examples=[ 1481 "{{ components_values.get('cursor_field', None) }}", 1482 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1483 ], 1484 title="Condition", 1485 ) 1486 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1487 1488 1489class StreamConfig(BaseModel): 1490 type: Literal["StreamConfig"] 1491 configs_pointer: List[str] = Field( 1492 ..., 1493 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1494 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1495 title="Configs Pointer", 1496 ) 1497 default_values: Optional[List[Dict[str, Any]]] = Field( 1498 None, 1499 description="A list of default values, each matching the structure expected from the parsed component value.", 1500 title="Default Values", 1501 ) 1502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1503 1504 1505class ConfigComponentsResolver(BaseModel): 1506 type: Literal["ConfigComponentsResolver"] 1507 stream_config: Union[List[StreamConfig], StreamConfig] 1508 components_mapping: List[ComponentMappingDefinition] 1509 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1510 1511 1512class StreamParametersDefinition(BaseModel): 1513 type: Literal["StreamParametersDefinition"] 1514 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1515 ..., 1516 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1517 examples=[ 1518 [ 1519 { 1520 "name": "test stream", 1521 "$parameters": {"entity": "test entity"}, 1522 "primary_key": "test key", 1523 } 1524 ] 1525 ], 1526 title="Stream Parameters", 1527 ) 1528 1529 1530class ParametrizedComponentsResolver(BaseModel): 1531 type: Literal["ParametrizedComponentsResolver"] 1532 stream_parameters: StreamParametersDefinition 1533 components_mapping: List[ComponentMappingDefinition] 1534 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1535 1536 1537class RequestBodyPlainText(BaseModel): 1538 type: Literal["RequestBodyPlainText"] 1539 value: str 1540 1541 1542class RequestBodyUrlEncodedForm(BaseModel): 1543 type: Literal["RequestBodyUrlEncodedForm"] 1544 value: Dict[str, str] 1545 1546 1547class RequestBodyJsonObject(BaseModel): 1548 type: Literal["RequestBodyJsonObject"] 1549 value: Dict[str, Any] 1550 1551 1552class RequestBodyGraphQlQuery(BaseModel): 1553 class Config: 1554 extra = Extra.allow 1555 1556 query: str = Field(..., description="The GraphQL query to be executed") 1557 1558 1559class ValidateAdheresToSchema(BaseModel): 1560 type: Literal["ValidateAdheresToSchema"] 1561 base_schema: Union[str, Dict[str, Any]] = Field( 1562 ..., 1563 description="The base JSON schema against which the user-provided schema will be validated.", 1564 examples=[ 1565 "{{ config['report_validation_schema'] }}", 1566 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1567 { 1568 "$schema": "http://json-schema.org/draft-07/schema#", 1569 "title": "Person", 1570 "type": "object", 1571 "properties": { 1572 "name": {"type": "string", "description": "The person's name"}, 1573 "age": { 1574 "type": "integer", 1575 "minimum": 0, 1576 "description": "The person's age", 1577 }, 1578 }, 1579 "required": ["name", "age"], 1580 }, 1581 ], 1582 title="Base JSON Schema", 1583 ) 1584 1585 1586class CustomValidationStrategy(BaseModel): 1587 class Config: 1588 extra = Extra.allow 1589 1590 type: Literal["CustomValidationStrategy"] 1591 class_name: str = Field( 1592 ..., 1593 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1594 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1595 title="Class Name", 1596 ) 1597 1598 1599class ConfigRemapField(BaseModel): 1600 type: Literal["ConfigRemapField"] 1601 map: Union[Dict[str, Any], str] = Field( 1602 ..., 1603 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1604 examples=[ 1605 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1606 "{{ config['status_mapping'] }}", 1607 ], 1608 title="Value Mapping", 1609 ) 1610 field_path: List[str] = Field( 1611 ..., 1612 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1613 examples=[ 1614 ["status"], 1615 ["data", "status"], 1616 ["data", "{{ config.name }}", "status"], 1617 ["data", "*", "status"], 1618 ], 1619 title="Field Path", 1620 ) 1621 1622 1623class ConfigRemoveFields(BaseModel): 1624 type: Literal["ConfigRemoveFields"] 1625 field_pointers: List[List[str]] = Field( 1626 ..., 1627 description="A list of field pointers to be removed from the config.", 1628 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1629 title="Field Pointers", 1630 ) 1631 condition: Optional[str] = Field( 1632 "", 1633 description="Fields will be removed if expression is evaluated to True.", 1634 examples=[ 1635 "{{ config['environemnt'] == 'sandbox' }}", 1636 "{{ property is integer }}", 1637 "{{ property|length > 5 }}", 1638 "{{ property == 'some_string_to_match' }}", 1639 ], 1640 ) 1641 1642 1643class CustomConfigTransformation(BaseModel): 1644 type: Literal["CustomConfigTransformation"] 1645 class_name: str = Field( 1646 ..., 1647 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1648 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1649 ) 1650 parameters: Optional[Dict[str, Any]] = Field( 1651 None, 1652 alias="$parameters", 1653 description="Additional parameters to be passed to the custom config transformation.", 1654 ) 1655 1656 1657class AddedFieldDefinition(BaseModel): 1658 type: Literal["AddedFieldDefinition"] 1659 path: List[str] = Field( 1660 ..., 1661 description="List of strings defining the path where to add the value on the record.", 1662 examples=[["segment_id"], ["metadata", "segment_id"]], 1663 title="Path", 1664 ) 1665 value: str = Field( 1666 ..., 1667 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1668 examples=[ 1669 "{{ record['updates'] }}", 1670 "{{ record['MetaData']['LastUpdatedTime'] }}", 1671 "{{ stream_partition['segment_id'] }}", 1672 ], 1673 title="Value", 1674 ) 1675 value_type: Optional[ValueType] = Field( 1676 None, 1677 description="Type of the value. If not specified, the type will be inferred from the value.", 1678 title="Value Type", 1679 ) 1680 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1681 1682 1683class AddFields(BaseModel): 1684 type: Literal["AddFields"] 1685 fields: List[AddedFieldDefinition] = Field( 1686 ..., 1687 description="List of transformations (path and corresponding value) that will be added to the record.", 1688 title="Fields", 1689 ) 1690 condition: Optional[str] = Field( 1691 "", 1692 description="Fields will be added if expression is evaluated to True.", 1693 examples=[ 1694 "{{ property|string == '' }}", 1695 "{{ property is integer }}", 1696 "{{ property|length > 5 }}", 1697 "{{ property == 'some_string_to_match' }}", 1698 ], 1699 ) 1700 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1701 1702 1703class ApiKeyAuthenticator(BaseModel): 1704 type: Literal["ApiKeyAuthenticator"] 1705 api_token: Optional[str] = Field( 1706 None, 1707 description="The API key to inject in the request. Fill it in the user inputs.", 1708 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1709 title="API Key", 1710 ) 1711 header: Optional[str] = Field( 1712 None, 1713 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1714 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1715 title="Header Name", 1716 ) 1717 inject_into: Optional[RequestOption] = Field( 1718 None, 1719 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1720 examples=[ 1721 {"inject_into": "header", "field_name": "Authorization"}, 1722 {"inject_into": "request_parameter", "field_name": "authKey"}, 1723 ], 1724 title="Inject API Key Into Outgoing HTTP Request", 1725 ) 1726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1727 1728 1729class AuthFlow(BaseModel): 1730 auth_flow_type: Optional[AuthFlowType] = Field( 1731 None, description="The type of auth to use", title="Auth flow type" 1732 ) 1733 predicate_key: Optional[List[str]] = Field( 1734 None, 1735 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1736 examples=[["credentials", "auth_type"]], 1737 title="Predicate key", 1738 ) 1739 predicate_value: Optional[str] = Field( 1740 None, 1741 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1742 examples=["Oauth"], 1743 title="Predicate value", 1744 ) 1745 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1746 1747 1748class CheckStream(BaseModel): 1749 type: Literal["CheckStream"] 1750 stream_names: Optional[List[str]] = Field( 1751 None, 1752 description="Names of the streams to try reading from when running a check operation.", 1753 examples=[["users"], ["users", "contacts"]], 1754 title="Stream Names", 1755 ) 1756 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1757 1758 1759class IncrementingCountCursor(BaseModel): 1760 type: Literal["IncrementingCountCursor"] 1761 cursor_field: str = Field( 1762 ..., 1763 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1764 examples=["created_at", "{{ config['record_cursor'] }}"], 1765 title="Cursor Field", 1766 ) 1767 start_value: Optional[Union[str, int]] = Field( 1768 None, 1769 description="The value that determines the earliest record that should be synced.", 1770 examples=[0, "{{ config['start_value'] }}"], 1771 title="Start Value", 1772 ) 1773 start_value_option: Optional[RequestOption] = Field( 1774 None, 1775 description="Optionally configures how the start value will be sent in requests to the source API.", 1776 title="Inject Start Value Into Outgoing HTTP Request", 1777 ) 1778 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1779 1780 1781class DatetimeBasedCursor(BaseModel): 1782 type: Literal["DatetimeBasedCursor"] 1783 clamping: Optional[Clamping] = Field( 1784 None, 1785 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1786 title="Date Range Clamping", 1787 ) 1788 cursor_field: str = Field( 1789 ..., 1790 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1791 examples=["created_at", "{{ config['record_cursor'] }}"], 1792 title="Cursor Field", 1793 ) 1794 cursor_datetime_formats: Optional[List[str]] = Field( 1795 None, 1796 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1797 examples=[ 1798 "%Y-%m-%d", 1799 "%Y-%m-%d %H:%M:%S", 1800 "%Y-%m-%dT%H:%M:%S", 1801 "%Y-%m-%dT%H:%M:%SZ", 1802 "%Y-%m-%dT%H:%M:%S%z", 1803 "%Y-%m-%dT%H:%M:%S.%fZ", 1804 "%Y-%m-%dT%H:%M:%S.%f%z", 1805 "%Y-%m-%d %H:%M:%S.%f+00:00", 1806 "%s", 1807 "%ms", 1808 ], 1809 title="Cursor Datetime Formats", 1810 ) 1811 start_datetime: Union[MinMaxDatetime, str] = Field( 1812 ..., 1813 description="The datetime that determines the earliest record that should be synced.", 1814 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1815 title="Start Datetime", 1816 ) 1817 start_time_option: Optional[RequestOption] = Field( 1818 None, 1819 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1820 title="Inject Start Time Into Outgoing HTTP Request", 1821 ) 1822 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1823 None, 1824 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1825 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1826 title="End Datetime", 1827 ) 1828 end_time_option: Optional[RequestOption] = Field( 1829 None, 1830 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1831 title="Inject End Time Into Outgoing HTTP Request", 1832 ) 1833 datetime_format: str = Field( 1834 ..., 1835 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1836 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1837 title="Outgoing Datetime Format", 1838 ) 1839 cursor_granularity: Optional[str] = Field( 1840 None, 1841 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1842 examples=["PT1S"], 1843 title="Cursor Granularity", 1844 ) 1845 is_data_feed: Optional[bool] = Field( 1846 None, 1847 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1848 title="Data Feed API", 1849 ) 1850 is_client_side_incremental: Optional[bool] = Field( 1851 None, 1852 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1853 title="Client-side Incremental Filtering", 1854 ) 1855 is_compare_strictly: Optional[bool] = Field( 1856 False, 1857 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1858 title="Strict Start-End Time Comparison", 1859 ) 1860 global_substream_cursor: Optional[bool] = Field( 1861 False, 1862 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1863 title="Global Substream Cursor", 1864 ) 1865 lookback_window: Optional[str] = Field( 1866 None, 1867 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1868 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1869 title="Lookback Window", 1870 ) 1871 partition_field_end: Optional[str] = Field( 1872 None, 1873 description="Name of the partition start time field.", 1874 examples=["ending_time"], 1875 title="Partition Field End", 1876 ) 1877 partition_field_start: Optional[str] = Field( 1878 None, 1879 description="Name of the partition end time field.", 1880 examples=["starting_time"], 1881 title="Partition Field Start", 1882 ) 1883 step: Optional[str] = Field( 1884 None, 1885 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1886 examples=["P1W", "{{ config['step_increment'] }}"], 1887 title="Step", 1888 ) 1889 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1890 1891 1892class FixedWindowCallRatePolicy(BaseModel): 1893 class Config: 1894 extra = Extra.allow 1895 1896 type: Literal["FixedWindowCallRatePolicy"] 1897 period: str = Field( 1898 ..., description="The time interval for the rate limit window.", title="Period" 1899 ) 1900 call_limit: int = Field( 1901 ..., 1902 description="The maximum number of calls allowed within the period.", 1903 title="Call Limit", 1904 ) 1905 matchers: List[HttpRequestRegexMatcher] = Field( 1906 ..., 1907 description="List of matchers that define which requests this policy applies to.", 1908 title="Matchers", 1909 ) 1910 1911 1912class MovingWindowCallRatePolicy(BaseModel): 1913 class Config: 1914 extra = Extra.allow 1915 1916 type: Literal["MovingWindowCallRatePolicy"] 1917 rates: List[Rate] = Field( 1918 ..., 1919 description="List of rates that define the call limits for different time intervals.", 1920 title="Rates", 1921 ) 1922 matchers: List[HttpRequestRegexMatcher] = Field( 1923 ..., 1924 description="List of matchers that define which requests this policy applies to.", 1925 title="Matchers", 1926 ) 1927 1928 1929class UnlimitedCallRatePolicy(BaseModel): 1930 class Config: 1931 extra = Extra.allow 1932 1933 type: Literal["UnlimitedCallRatePolicy"] 1934 matchers: List[HttpRequestRegexMatcher] = Field( 1935 ..., 1936 description="List of matchers that define which requests this policy applies to.", 1937 title="Matchers", 1938 ) 1939 1940 1941class DefaultErrorHandler(BaseModel): 1942 type: Literal["DefaultErrorHandler"] 1943 backoff_strategies: Optional[ 1944 List[ 1945 Union[ 1946 ConstantBackoffStrategy, 1947 ExponentialBackoffStrategy, 1948 WaitTimeFromHeader, 1949 WaitUntilTimeFromHeader, 1950 CustomBackoffStrategy, 1951 ] 1952 ] 1953 ] = Field( 1954 None, 1955 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1956 title="Backoff Strategies", 1957 ) 1958 max_retries: Optional[int] = Field( 1959 5, 1960 description="The maximum number of time to retry a retryable request before giving up and failing.", 1961 examples=[5, 0, 10], 1962 title="Max Retry Count", 1963 ) 1964 response_filters: Optional[List[HttpResponseFilter]] = Field( 1965 None, 1966 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1967 title="Response Filters", 1968 ) 1969 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1970 1971 1972class DefaultPaginator(BaseModel): 1973 type: Literal["DefaultPaginator"] 1974 pagination_strategy: Union[ 1975 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1976 ] = Field( 1977 ..., 1978 description="Strategy defining how records are paginated.", 1979 title="Pagination Strategy", 1980 ) 1981 page_size_option: Optional[RequestOption] = Field( 1982 None, title="Inject Page Size Into Outgoing HTTP Request" 1983 ) 1984 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1985 None, 1986 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 1987 title="Inject Page Token Into Outgoing HTTP Request", 1988 ) 1989 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1990 1991 1992class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1993 type: Literal["ApiKey"] 1994 inject_into: RequestOption = Field( 1995 ..., 1996 description="Configure how the API Key will be sent in requests to the source API.", 1997 examples=[ 1998 {"inject_into": "header", "field_name": "Authorization"}, 1999 {"inject_into": "request_parameter", "field_name": "authKey"}, 2000 ], 2001 title="Inject API Key Into Outgoing HTTP Request", 2002 ) 2003 2004 2005class ListPartitionRouter(BaseModel): 2006 type: Literal["ListPartitionRouter"] 2007 cursor_field: str = Field( 2008 ..., 2009 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2010 examples=["section", "{{ config['section_key'] }}"], 2011 title="Current Partition Value Identifier", 2012 ) 2013 values: Union[str, List[str]] = Field( 2014 ..., 2015 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2016 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2017 title="Partition Values", 2018 ) 2019 request_option: Optional[RequestOption] = Field( 2020 None, 2021 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2022 title="Inject Partition Value Into Outgoing HTTP Request", 2023 ) 2024 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2025 2026 2027class RecordSelector(BaseModel): 2028 type: Literal["RecordSelector"] 2029 extractor: Union[DpathExtractor, CustomRecordExtractor] 2030 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2031 None, 2032 description="Responsible for filtering records to be emitted by the Source.", 2033 title="Record Filter", 2034 ) 2035 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2036 None, 2037 description="Responsible for normalization according to the schema.", 2038 title="Schema Normalization", 2039 ) 2040 transform_before_filtering: Optional[bool] = Field( 2041 None, 2042 description="If true, transformation will be applied before record filtering.", 2043 title="Transform Before Filtering", 2044 ) 2045 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2046 2047 2048class GzipDecoder(BaseModel): 2049 type: Literal["GzipDecoder"] 2050 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2051 2052 2053class RequestBodyGraphQL(BaseModel): 2054 type: Literal["RequestBodyGraphQL"] 2055 value: RequestBodyGraphQlQuery 2056 2057 2058class DpathValidator(BaseModel): 2059 type: Literal["DpathValidator"] 2060 field_path: List[str] = Field( 2061 ..., 2062 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2063 examples=[ 2064 ["data"], 2065 ["data", "records"], 2066 ["data", "{{ parameters.name }}"], 2067 ["data", "*", "record"], 2068 ], 2069 title="Field Path", 2070 ) 2071 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2072 ..., 2073 description="The condition that the specified config value will be evaluated against", 2074 title="Validation Strategy", 2075 ) 2076 2077 2078class PredicateValidator(BaseModel): 2079 type: Literal["PredicateValidator"] 2080 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2081 ..., 2082 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2083 examples=[ 2084 "test-value", 2085 "{{ config['api_version'] }}", 2086 "{{ config['tenant_id'] }}", 2087 123, 2088 ], 2089 title="Value", 2090 ) 2091 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2092 ..., 2093 description="The validation strategy to apply to the value.", 2094 title="Validation Strategy", 2095 ) 2096 2097 2098class ConfigAddFields(BaseModel): 2099 type: Literal["ConfigAddFields"] 2100 fields: List[AddedFieldDefinition] = Field( 2101 ..., 2102 description="A list of transformations (path and corresponding value) that will be added to the config.", 2103 title="Fields", 2104 ) 2105 condition: Optional[str] = Field( 2106 "", 2107 description="Fields will be added if expression is evaluated to True.", 2108 examples=[ 2109 "{{ config['environemnt'] == 'sandbox' }}", 2110 "{{ property is integer }}", 2111 "{{ property|length > 5 }}", 2112 "{{ property == 'some_string_to_match' }}", 2113 ], 2114 ) 2115 2116 2117class CompositeErrorHandler(BaseModel): 2118 type: Literal["CompositeErrorHandler"] 2119 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2120 ..., 2121 description="List of error handlers to iterate on to determine how to handle a failed response.", 2122 title="Error Handlers", 2123 ) 2124 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2125 2126 2127class HTTPAPIBudget(BaseModel): 2128 class Config: 2129 extra = Extra.allow 2130 2131 type: Literal["HTTPAPIBudget"] 2132 policies: List[ 2133 Union[ 2134 FixedWindowCallRatePolicy, 2135 MovingWindowCallRatePolicy, 2136 UnlimitedCallRatePolicy, 2137 ] 2138 ] = Field( 2139 ..., 2140 description="List of call rate policies that define how many calls are allowed.", 2141 title="Policies", 2142 ) 2143 ratelimit_reset_header: Optional[str] = Field( 2144 "ratelimit-reset", 2145 description="The HTTP response header name that indicates when the rate limit resets.", 2146 title="Rate Limit Reset Header", 2147 ) 2148 ratelimit_remaining_header: Optional[str] = Field( 2149 "ratelimit-remaining", 2150 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2151 title="Rate Limit Remaining Header", 2152 ) 2153 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2154 [429], 2155 description="List of HTTP status codes that indicate a rate limit has been hit.", 2156 title="Status Codes for Rate Limit Hit", 2157 ) 2158 2159 2160class ZipfileDecoder(BaseModel): 2161 class Config: 2162 extra = Extra.allow 2163 2164 type: Literal["ZipfileDecoder"] 2165 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2166 ..., 2167 description="Parser to parse the decompressed data from the zipfile(s).", 2168 title="Parser", 2169 ) 2170 2171 2172class ConfigMigration(BaseModel): 2173 type: Literal["ConfigMigration"] 2174 description: Optional[str] = Field( 2175 None, description="The description/purpose of the config migration." 2176 ) 2177 transformations: List[ 2178 Union[ 2179 ConfigRemapField, 2180 ConfigAddFields, 2181 ConfigRemoveFields, 2182 CustomConfigTransformation, 2183 ] 2184 ] = Field( 2185 ..., 2186 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2187 title="Transformations", 2188 ) 2189 2190 2191class ConfigNormalizationRules(BaseModel): 2192 class Config: 2193 extra = Extra.forbid 2194 2195 type: Literal["ConfigNormalizationRules"] 2196 config_migrations: Optional[List[ConfigMigration]] = Field( 2197 [], 2198 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2199 title="Config Migrations", 2200 ) 2201 transformations: Optional[ 2202 List[ 2203 Union[ 2204 ConfigRemapField, 2205 ConfigAddFields, 2206 ConfigRemoveFields, 2207 CustomConfigTransformation, 2208 ] 2209 ] 2210 ] = Field( 2211 [], 2212 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2213 title="Transformations", 2214 ) 2215 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2216 [], 2217 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2218 title="Validations", 2219 ) 2220 2221 2222class Spec(BaseModel): 2223 type: Literal["Spec"] 2224 connection_specification: Dict[str, Any] = Field( 2225 ..., 2226 description="A connection specification describing how a the connector can be configured.", 2227 title="Connection Specification", 2228 ) 2229 documentation_url: Optional[str] = Field( 2230 None, 2231 description="URL of the connector's documentation page.", 2232 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2233 title="Documentation URL", 2234 ) 2235 advanced_auth: Optional[AuthFlow] = Field( 2236 None, 2237 description="Advanced specification for configuring the authentication flow.", 2238 title="Advanced Auth", 2239 ) 2240 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2241 None, title="Config Normalization Rules" 2242 ) 2243 2244 2245class DeclarativeSource1(BaseModel): 2246 class Config: 2247 extra = Extra.forbid 2248 2249 type: Literal["DeclarativeSource"] 2250 check: Union[CheckStream, CheckDynamicStream] 2251 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2252 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2253 version: str = Field( 2254 ..., 2255 description="The version of the Airbyte CDK used to build and test the source.", 2256 ) 2257 schemas: Optional[Schemas] = None 2258 definitions: Optional[Dict[str, Any]] = None 2259 spec: Optional[Spec] = None 2260 concurrency_level: Optional[ConcurrencyLevel] = None 2261 api_budget: Optional[HTTPAPIBudget] = None 2262 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2263 None, 2264 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2265 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2266 title="Maximum Concurrent Asynchronous Jobs", 2267 ) 2268 metadata: Optional[Dict[str, Any]] = Field( 2269 None, 2270 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2271 ) 2272 description: Optional[str] = Field( 2273 None, 2274 description="A description of the connector. It will be presented on the Source documentation page.", 2275 ) 2276 2277 2278class DeclarativeSource2(BaseModel): 2279 class Config: 2280 extra = Extra.forbid 2281 2282 type: Literal["DeclarativeSource"] 2283 check: Union[CheckStream, CheckDynamicStream] 2284 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2285 None 2286 ) 2287 dynamic_streams: List[DynamicDeclarativeStream] 2288 version: str = Field( 2289 ..., 2290 description="The version of the Airbyte CDK used to build and test the source.", 2291 ) 2292 schemas: Optional[Schemas] = None 2293 definitions: Optional[Dict[str, Any]] = None 2294 spec: Optional[Spec] = None 2295 concurrency_level: Optional[ConcurrencyLevel] = None 2296 api_budget: Optional[HTTPAPIBudget] = None 2297 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2298 None, 2299 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2300 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2301 title="Maximum Concurrent Asynchronous Jobs", 2302 ) 2303 metadata: Optional[Dict[str, Any]] = Field( 2304 None, 2305 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2306 ) 2307 description: Optional[str] = Field( 2308 None, 2309 description="A description of the connector. It will be presented on the Source documentation page.", 2310 ) 2311 2312 2313class DeclarativeSource(BaseModel): 2314 class Config: 2315 extra = Extra.forbid 2316 2317 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2318 ..., 2319 description="An API source that extracts data according to its declarative components.", 2320 title="DeclarativeSource", 2321 ) 2322 2323 2324class SelectiveAuthenticator(BaseModel): 2325 class Config: 2326 extra = Extra.allow 2327 2328 type: Literal["SelectiveAuthenticator"] 2329 authenticator_selection_path: List[str] = Field( 2330 ..., 2331 description="Path of the field in config with selected authenticator name", 2332 examples=[["auth"], ["auth", "type"]], 2333 title="Authenticator Selection Path", 2334 ) 2335 authenticators: Dict[ 2336 str, 2337 Union[ 2338 ApiKeyAuthenticator, 2339 BasicHttpAuthenticator, 2340 BearerAuthenticator, 2341 OAuthAuthenticator, 2342 JwtAuthenticator, 2343 SessionTokenAuthenticator, 2344 LegacySessionTokenAuthenticator, 2345 CustomAuthenticator, 2346 NoAuth, 2347 ], 2348 ] = Field( 2349 ..., 2350 description="Authenticators to select from.", 2351 examples=[ 2352 { 2353 "authenticators": { 2354 "token": "#/definitions/ApiKeyAuthenticator", 2355 "oauth": "#/definitions/OAuthAuthenticator", 2356 "jwt": "#/definitions/JwtAuthenticator", 2357 } 2358 } 2359 ], 2360 title="Authenticators", 2361 ) 2362 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2363 2364 2365class ConditionalStreams(BaseModel): 2366 type: Literal["ConditionalStreams"] 2367 condition: str = Field( 2368 ..., 2369 description="Condition that will be evaluated to determine if a set of streams should be available.", 2370 examples=["{{ config['is_sandbox'] }}"], 2371 title="Condition", 2372 ) 2373 streams: List[DeclarativeStream] = Field( 2374 ..., 2375 description="Streams that will be used during an operation based on the condition.", 2376 title="Streams", 2377 ) 2378 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2379 2380 2381class FileUploader(BaseModel): 2382 type: Literal["FileUploader"] 2383 requester: Union[HttpRequester, CustomRequester] = Field( 2384 ..., 2385 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2386 ) 2387 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2388 ..., 2389 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2390 ) 2391 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2392 None, 2393 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2394 ) 2395 filename_extractor: Optional[str] = Field( 2396 None, 2397 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2398 examples=[ 2399 "{{ record.id }}/{{ record.file_name }}/", 2400 "{{ record.id }}_{{ record.file_name }}/", 2401 ], 2402 ) 2403 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2404 2405 2406class DeclarativeStream(BaseModel): 2407 class Config: 2408 extra = Extra.allow 2409 2410 type: Literal["DeclarativeStream"] 2411 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2412 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2413 ..., 2414 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2415 title="Retriever", 2416 ) 2417 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2418 None, 2419 description="Component used to fetch data incrementally based on a time field in the data.", 2420 title="Incremental Sync", 2421 ) 2422 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2423 schema_loader: Optional[ 2424 Union[ 2425 InlineSchemaLoader, 2426 DynamicSchemaLoader, 2427 JsonFileSchemaLoader, 2428 List[ 2429 Union[ 2430 InlineSchemaLoader, 2431 DynamicSchemaLoader, 2432 JsonFileSchemaLoader, 2433 CustomSchemaLoader, 2434 ] 2435 ], 2436 CustomSchemaLoader, 2437 ] 2438 ] = Field( 2439 None, 2440 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2441 title="Schema Loader", 2442 ) 2443 transformations: Optional[ 2444 List[ 2445 Union[ 2446 AddFields, 2447 RemoveFields, 2448 KeysToLower, 2449 KeysToSnakeCase, 2450 FlattenFields, 2451 DpathFlattenFields, 2452 KeysReplace, 2453 CustomTransformation, 2454 ] 2455 ] 2456 ] = Field( 2457 None, 2458 description="A list of transformations to be applied to each output record.", 2459 title="Transformations", 2460 ) 2461 state_migrations: Optional[ 2462 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2463 ] = Field( 2464 [], 2465 description="Array of state migrations to be applied on the input state", 2466 title="State Migrations", 2467 ) 2468 file_uploader: Optional[FileUploader] = Field( 2469 None, 2470 description="(experimental) Describes how to fetch a file", 2471 title="File Uploader", 2472 ) 2473 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2474 2475 2476class SessionTokenAuthenticator(BaseModel): 2477 type: Literal["SessionTokenAuthenticator"] 2478 login_requester: HttpRequester = Field( 2479 ..., 2480 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2481 examples=[ 2482 { 2483 "type": "HttpRequester", 2484 "url_base": "https://my_api.com", 2485 "path": "/login", 2486 "authenticator": { 2487 "type": "BasicHttpAuthenticator", 2488 "username": "{{ config.username }}", 2489 "password": "{{ config.password }}", 2490 }, 2491 } 2492 ], 2493 title="Login Requester", 2494 ) 2495 session_token_path: List[str] = Field( 2496 ..., 2497 description="The path in the response body returned from the login requester to the session token.", 2498 examples=[["access_token"], ["result", "token"]], 2499 title="Session Token Path", 2500 ) 2501 expiration_duration: Optional[str] = Field( 2502 None, 2503 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2504 examples=["PT1H", "P1D"], 2505 title="Expiration Duration", 2506 ) 2507 request_authentication: Union[ 2508 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2509 ] = Field( 2510 ..., 2511 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2512 title="Data Request Authentication", 2513 ) 2514 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2515 None, description="Component used to decode the response.", title="Decoder" 2516 ) 2517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2518 2519 2520class HttpRequester(BaseModelWithDeprecations): 2521 type: Literal["HttpRequester"] 2522 url_base: Optional[str] = Field( 2523 None, 2524 deprecated=True, 2525 deprecation_message="Use `url` field instead.", 2526 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2527 examples=[ 2528 "https://connect.squareup.com/v2", 2529 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2530 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2531 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2532 ], 2533 title="API Base URL", 2534 ) 2535 url: Optional[str] = Field( 2536 None, 2537 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2538 examples=[ 2539 "https://connect.squareup.com/v2", 2540 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2541 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2542 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2543 ], 2544 title="API Endpoint URL", 2545 ) 2546 path: Optional[str] = Field( 2547 None, 2548 deprecated=True, 2549 deprecation_message="Use `url` field instead.", 2550 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2551 examples=[ 2552 "/products", 2553 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2554 "/trades/{{ config['symbol_id'] }}/history", 2555 ], 2556 title="URL Path", 2557 ) 2558 http_method: Optional[HttpMethod] = Field( 2559 HttpMethod.GET, 2560 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2561 examples=["GET", "POST"], 2562 title="HTTP Method", 2563 ) 2564 authenticator: Optional[ 2565 Union[ 2566 ApiKeyAuthenticator, 2567 BasicHttpAuthenticator, 2568 BearerAuthenticator, 2569 OAuthAuthenticator, 2570 JwtAuthenticator, 2571 SessionTokenAuthenticator, 2572 SelectiveAuthenticator, 2573 CustomAuthenticator, 2574 NoAuth, 2575 LegacySessionTokenAuthenticator, 2576 ] 2577 ] = Field( 2578 None, 2579 description="Authentication method to use for requests sent to the API.", 2580 title="Authenticator", 2581 ) 2582 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2583 None, 2584 deprecated=True, 2585 deprecation_message="Use `query_properties` field instead.", 2586 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2587 title="Fetch Properties from Endpoint", 2588 ) 2589 query_properties: Optional[QueryProperties] = Field( 2590 None, 2591 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2592 title="Query Properties", 2593 ) 2594 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2595 None, 2596 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2597 examples=[ 2598 {"unit": "day"}, 2599 { 2600 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2601 }, 2602 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2603 {"sort_by[asc]": "updated_at"}, 2604 ], 2605 title="Query Parameters", 2606 ) 2607 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2608 None, 2609 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2610 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2611 title="Request Headers", 2612 ) 2613 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2614 None, 2615 deprecated=True, 2616 deprecation_message="Use `request_body` field instead.", 2617 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2618 examples=[ 2619 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2620 ], 2621 title="Request Body Payload (Non-JSON)", 2622 ) 2623 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2624 None, 2625 deprecated=True, 2626 deprecation_message="Use `request_body` field instead.", 2627 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2628 examples=[ 2629 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2630 {"key": "{{ config['value'] }}"}, 2631 {"sort": {"field": "updated_at", "order": "ascending"}}, 2632 ], 2633 title="Request Body JSON Payload", 2634 ) 2635 request_body: Optional[ 2636 Union[ 2637 RequestBodyPlainText, 2638 RequestBodyUrlEncodedForm, 2639 RequestBodyJsonObject, 2640 RequestBodyGraphQL, 2641 ] 2642 ] = Field( 2643 None, 2644 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2645 title="Request Body", 2646 ) 2647 error_handler: Optional[ 2648 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2649 ] = Field( 2650 None, 2651 description="Error handler component that defines how to handle errors.", 2652 title="Error Handler", 2653 ) 2654 use_cache: Optional[bool] = Field( 2655 False, 2656 description="Enables stream requests caching. This field is automatically set by the CDK.", 2657 title="Use Cache", 2658 ) 2659 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2660 2661 2662class DynamicSchemaLoader(BaseModel): 2663 type: Literal["DynamicSchemaLoader"] 2664 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2665 ..., 2666 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2667 title="Retriever", 2668 ) 2669 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2670 None, 2671 description="Responsible for filtering fields to be added to json schema.", 2672 title="Schema Filter", 2673 ) 2674 schema_transformations: Optional[ 2675 List[ 2676 Union[ 2677 AddFields, 2678 RemoveFields, 2679 KeysToLower, 2680 KeysToSnakeCase, 2681 FlattenFields, 2682 DpathFlattenFields, 2683 KeysReplace, 2684 CustomTransformation, 2685 ] 2686 ] 2687 ] = Field( 2688 None, 2689 description="A list of transformations to be applied to the schema.", 2690 title="Schema Transformations", 2691 ) 2692 schema_type_identifier: SchemaTypeIdentifier 2693 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2694 2695 2696class ParentStreamConfig(BaseModel): 2697 type: Literal["ParentStreamConfig"] 2698 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2699 ..., description="Reference to the parent stream.", title="Parent Stream" 2700 ) 2701 parent_key: str = Field( 2702 ..., 2703 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2704 examples=["id", "{{ config['parent_record_id'] }}"], 2705 title="Parent Key", 2706 ) 2707 partition_field: str = Field( 2708 ..., 2709 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2710 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2711 title="Current Parent Key Value Identifier", 2712 ) 2713 request_option: Optional[RequestOption] = Field( 2714 None, 2715 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2716 title="Request Option", 2717 ) 2718 incremental_dependency: Optional[bool] = Field( 2719 False, 2720 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2721 title="Incremental Dependency", 2722 ) 2723 lazy_read_pointer: Optional[List[str]] = Field( 2724 [], 2725 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2726 title="Lazy Read Pointer", 2727 ) 2728 extra_fields: Optional[List[List[str]]] = Field( 2729 None, 2730 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2731 title="Extra Fields", 2732 ) 2733 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2734 2735 2736class PropertiesFromEndpoint(BaseModel): 2737 type: Literal["PropertiesFromEndpoint"] 2738 property_field_path: List[str] = Field( 2739 ..., 2740 description="Describes the path to the field that should be extracted", 2741 examples=[["name"]], 2742 ) 2743 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2744 ..., 2745 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2746 ) 2747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2748 2749 2750class QueryProperties(BaseModel): 2751 type: Literal["QueryProperties"] 2752 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2753 ..., 2754 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2755 title="Property List", 2756 ) 2757 always_include_properties: Optional[List[str]] = Field( 2758 None, 2759 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2760 title="Always Include Properties", 2761 ) 2762 property_chunking: Optional[PropertyChunking] = Field( 2763 None, 2764 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2765 title="Property Chunking", 2766 ) 2767 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2768 2769 2770class StateDelegatingStream(BaseModel): 2771 type: Literal["StateDelegatingStream"] 2772 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2773 full_refresh_stream: DeclarativeStream = Field( 2774 ..., 2775 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2776 title="Full Refresh Stream", 2777 ) 2778 incremental_stream: DeclarativeStream = Field( 2779 ..., 2780 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2781 title="Incremental Stream", 2782 ) 2783 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2784 2785 2786class SimpleRetriever(BaseModel): 2787 type: Literal["SimpleRetriever"] 2788 requester: Union[HttpRequester, CustomRequester] = Field( 2789 ..., 2790 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2791 ) 2792 decoder: Optional[ 2793 Union[ 2794 JsonDecoder, 2795 XmlDecoder, 2796 CsvDecoder, 2797 JsonlDecoder, 2798 GzipDecoder, 2799 IterableDecoder, 2800 ZipfileDecoder, 2801 CustomDecoder, 2802 ] 2803 ] = Field( 2804 None, 2805 description="Component decoding the response so records can be extracted.", 2806 title="HTTP Response Format", 2807 ) 2808 record_selector: RecordSelector = Field( 2809 ..., 2810 description="Component that describes how to extract records from a HTTP response.", 2811 ) 2812 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2813 None, 2814 description="Paginator component that describes how to navigate through the API's pages.", 2815 ) 2816 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2817 False, 2818 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2819 ) 2820 partition_router: Optional[ 2821 Union[ 2822 SubstreamPartitionRouter, 2823 ListPartitionRouter, 2824 GroupingPartitionRouter, 2825 CustomPartitionRouter, 2826 List[ 2827 Union[ 2828 SubstreamPartitionRouter, 2829 ListPartitionRouter, 2830 GroupingPartitionRouter, 2831 CustomPartitionRouter, 2832 ] 2833 ], 2834 ] 2835 ] = Field( 2836 None, 2837 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2838 title="Partition Router", 2839 ) 2840 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2841 2842 2843class AsyncRetriever(BaseModel): 2844 type: Literal["AsyncRetriever"] 2845 record_selector: RecordSelector = Field( 2846 ..., 2847 description="Component that describes how to extract records from a HTTP response.", 2848 ) 2849 status_mapping: AsyncJobStatusMap = Field( 2850 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2851 ) 2852 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2853 ..., description="Responsible for fetching the actual status of the async job." 2854 ) 2855 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2856 None, 2857 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2858 ) 2859 download_extractor: Optional[ 2860 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2861 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2862 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2863 ..., 2864 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2865 ) 2866 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2867 ..., 2868 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2869 ) 2870 polling_job_timeout: Optional[Union[int, str]] = Field( 2871 None, 2872 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2873 ) 2874 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2875 None, 2876 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2877 ) 2878 download_requester: Union[HttpRequester, CustomRequester] = Field( 2879 ..., 2880 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2881 ) 2882 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2883 None, 2884 description="Paginator component that describes how to navigate through the API's pages during download.", 2885 ) 2886 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2887 None, 2888 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2889 ) 2890 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2891 None, 2892 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2893 ) 2894 partition_router: Optional[ 2895 Union[ 2896 ListPartitionRouter, 2897 SubstreamPartitionRouter, 2898 GroupingPartitionRouter, 2899 CustomPartitionRouter, 2900 List[ 2901 Union[ 2902 ListPartitionRouter, 2903 SubstreamPartitionRouter, 2904 GroupingPartitionRouter, 2905 CustomPartitionRouter, 2906 ] 2907 ], 2908 ] 2909 ] = Field( 2910 [], 2911 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2912 title="Partition Router", 2913 ) 2914 decoder: Optional[ 2915 Union[ 2916 CsvDecoder, 2917 GzipDecoder, 2918 JsonDecoder, 2919 JsonlDecoder, 2920 IterableDecoder, 2921 XmlDecoder, 2922 ZipfileDecoder, 2923 CustomDecoder, 2924 ] 2925 ] = Field( 2926 None, 2927 description="Component decoding the response so records can be extracted.", 2928 title="HTTP Response Format", 2929 ) 2930 download_decoder: Optional[ 2931 Union[ 2932 CsvDecoder, 2933 GzipDecoder, 2934 JsonDecoder, 2935 JsonlDecoder, 2936 IterableDecoder, 2937 XmlDecoder, 2938 ZipfileDecoder, 2939 CustomDecoder, 2940 ] 2941 ] = Field( 2942 None, 2943 description="Component decoding the download response so records can be extracted.", 2944 title="Download HTTP Response Format", 2945 ) 2946 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2947 2948 2949class SubstreamPartitionRouter(BaseModel): 2950 type: Literal["SubstreamPartitionRouter"] 2951 parent_stream_configs: List[ParentStreamConfig] = Field( 2952 ..., 2953 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2954 title="Parent Stream Configs", 2955 ) 2956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2957 2958 2959class GroupingPartitionRouter(BaseModel): 2960 type: Literal["GroupingPartitionRouter"] 2961 group_size: int = Field( 2962 ..., 2963 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2964 examples=[10, 50], 2965 title="Group Size", 2966 ) 2967 underlying_partition_router: Union[ 2968 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2969 ] = Field( 2970 ..., 2971 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2972 title="Underlying Partition Router", 2973 ) 2974 deduplicate: Optional[bool] = Field( 2975 True, 2976 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2977 title="Deduplicate Partitions", 2978 ) 2979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2980 2981 2982class HttpComponentsResolver(BaseModel): 2983 type: Literal["HttpComponentsResolver"] 2984 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2985 ..., 2986 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2987 title="Retriever", 2988 ) 2989 components_mapping: List[ComponentMappingDefinition] 2990 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2991 2992 2993class DynamicDeclarativeStream(BaseModel): 2994 type: Literal["DynamicDeclarativeStream"] 2995 name: Optional[str] = Field( 2996 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2997 ) 2998 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 2999 ..., description="Reference to the stream template.", title="Stream Template" 3000 ) 3001 components_resolver: Union[ 3002 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3003 ] = Field( 3004 ..., 3005 description="Component resolve and populates stream templates with components values.", 3006 title="Components Resolver", 3007 ) 3008 use_parent_parameters: Optional[bool] = Field( 3009 True, 3010 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3011 title="Use Parent Parameters", 3012 ) 3013 3014 3015ComplexFieldType.update_forward_refs() 3016GzipDecoder.update_forward_refs() 3017CompositeErrorHandler.update_forward_refs() 3018DeclarativeSource1.update_forward_refs() 3019DeclarativeSource2.update_forward_refs() 3020SelectiveAuthenticator.update_forward_refs() 3021ConditionalStreams.update_forward_refs() 3022FileUploader.update_forward_refs() 3023DeclarativeStream.update_forward_refs() 3024SessionTokenAuthenticator.update_forward_refs() 3025HttpRequester.update_forward_refs() 3026DynamicSchemaLoader.update_forward_refs() 3027ParentStreamConfig.update_forward_refs() 3028PropertiesFromEndpoint.update_forward_refs() 3029SimpleRetriever.update_forward_refs() 3030AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[int] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
178class CustomPaginationStrategy(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomPaginationStrategy"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomPaginationStrategy"], 187 title="Class Name", 188 ) 189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
192class CustomRecordExtractor(BaseModel): 193 class Config: 194 extra = Extra.allow 195 196 type: Literal["CustomRecordExtractor"] 197 class_name: str = Field( 198 ..., 199 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 200 examples=["source_railz.components.MyCustomRecordExtractor"], 201 title="Class Name", 202 ) 203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
206class CustomRecordFilter(BaseModel): 207 class Config: 208 extra = Extra.allow 209 210 type: Literal["CustomRecordFilter"] 211 class_name: str = Field( 212 ..., 213 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 214 examples=["source_railz.components.MyCustomCustomRecordFilter"], 215 title="Class Name", 216 ) 217 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
220class CustomRequester(BaseModel): 221 class Config: 222 extra = Extra.allow 223 224 type: Literal["CustomRequester"] 225 class_name: str = Field( 226 ..., 227 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 228 examples=["source_railz.components.MyCustomRecordExtractor"], 229 title="Class Name", 230 ) 231 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
234class CustomRetriever(BaseModel): 235 class Config: 236 extra = Extra.allow 237 238 type: Literal["CustomRetriever"] 239 class_name: str = Field( 240 ..., 241 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 242 examples=["source_railz.components.MyCustomRetriever"], 243 title="Class Name", 244 ) 245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
248class CustomPartitionRouter(BaseModel): 249 class Config: 250 extra = Extra.allow 251 252 type: Literal["CustomPartitionRouter"] 253 class_name: str = Field( 254 ..., 255 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 256 examples=["source_railz.components.MyCustomPartitionRouter"], 257 title="Class Name", 258 ) 259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
262class CustomSchemaLoader(BaseModel): 263 class Config: 264 extra = Extra.allow 265 266 type: Literal["CustomSchemaLoader"] 267 class_name: str = Field( 268 ..., 269 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 270 examples=["source_railz.components.MyCustomSchemaLoader"], 271 title="Class Name", 272 ) 273 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
276class CustomSchemaNormalization(BaseModel): 277 class Config: 278 extra = Extra.allow 279 280 type: Literal["CustomSchemaNormalization"] 281 class_name: str = Field( 282 ..., 283 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 284 examples=[ 285 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 286 ], 287 title="Class Name", 288 ) 289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
292class CustomStateMigration(BaseModel): 293 class Config: 294 extra = Extra.allow 295 296 type: Literal["CustomStateMigration"] 297 class_name: str = Field( 298 ..., 299 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 300 examples=["source_railz.components.MyCustomStateMigration"], 301 title="Class Name", 302 ) 303 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
306class CustomTransformation(BaseModel): 307 class Config: 308 extra = Extra.allow 309 310 type: Literal["CustomTransformation"] 311 class_name: str = Field( 312 ..., 313 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 314 examples=["source_railz.components.MyCustomTransformation"], 315 title="Class Name", 316 ) 317 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
337class Algorithm(Enum): 338 HS256 = "HS256" 339 HS384 = "HS384" 340 HS512 = "HS512" 341 ES256 = "ES256" 342 ES256K = "ES256K" 343 ES384 = "ES384" 344 ES512 = "ES512" 345 RS256 = "RS256" 346 RS384 = "RS384" 347 RS512 = "RS512" 348 PS256 = "PS256" 349 PS384 = "PS384" 350 PS512 = "PS512" 351 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
354class JwtHeaders(BaseModel): 355 class Config: 356 extra = Extra.forbid 357 358 kid: Optional[str] = Field( 359 None, 360 description="Private key ID for user account.", 361 examples=["{{ config['kid'] }}"], 362 title="Key Identifier", 363 ) 364 typ: Optional[str] = Field( 365 "JWT", 366 description="The media type of the complete JWT.", 367 examples=["JWT"], 368 title="Type", 369 ) 370 cty: Optional[str] = Field( 371 None, 372 description="Content type of JWT header.", 373 examples=["JWT"], 374 title="Content Type", 375 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
378class JwtPayload(BaseModel): 379 class Config: 380 extra = Extra.forbid 381 382 iss: Optional[str] = Field( 383 None, 384 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 385 examples=["{{ config['iss'] }}"], 386 title="Issuer", 387 ) 388 sub: Optional[str] = Field( 389 None, 390 description="The subject of the JWT. Commonly defined by the API.", 391 title="Subject", 392 ) 393 aud: Optional[str] = Field( 394 None, 395 description="The recipient that the JWT is intended for. Commonly defined by the API.", 396 examples=["appstoreconnect-v1"], 397 title="Audience", 398 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
401class JwtAuthenticator(BaseModel): 402 type: Literal["JwtAuthenticator"] 403 secret_key: str = Field( 404 ..., 405 description="Secret used to sign the JSON web token.", 406 examples=["{{ config['secret_key'] }}"], 407 title="Secret Key", 408 ) 409 base64_encode_secret_key: Optional[bool] = Field( 410 False, 411 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 412 title="Base64-encode Secret Key", 413 ) 414 algorithm: Algorithm = Field( 415 ..., 416 description="Algorithm used to sign the JSON web token.", 417 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 418 title="Algorithm", 419 ) 420 token_duration: Optional[int] = Field( 421 1200, 422 description="The amount of time in seconds a JWT token can be valid after being issued.", 423 examples=[1200, 3600], 424 title="Token Duration", 425 ) 426 header_prefix: Optional[str] = Field( 427 None, 428 description="The prefix to be used within the Authentication header.", 429 examples=["Bearer", "Basic"], 430 title="Header Prefix", 431 ) 432 jwt_headers: Optional[JwtHeaders] = Field( 433 None, 434 description="JWT headers used when signing JSON web token.", 435 title="JWT Headers", 436 ) 437 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 438 None, 439 description="Additional headers to be included with the JWT headers object.", 440 title="Additional JWT Headers", 441 ) 442 jwt_payload: Optional[JwtPayload] = Field( 443 None, 444 description="JWT Payload used when signing JSON web token.", 445 title="JWT Payload", 446 ) 447 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 448 None, 449 description="Additional properties to be added to the JWT payload.", 450 title="Additional JWT Payload Properties", 451 ) 452 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
455class RefreshTokenUpdater(BaseModel): 456 refresh_token_name: Optional[str] = Field( 457 "refresh_token", 458 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 459 examples=["refresh_token"], 460 title="Refresh Token Property Name", 461 ) 462 access_token_config_path: Optional[List[str]] = Field( 463 ["credentials", "access_token"], 464 description="Config path to the access token. Make sure the field actually exists in the config.", 465 examples=[["credentials", "access_token"], ["access_token"]], 466 title="Config Path To Access Token", 467 ) 468 refresh_token_config_path: Optional[List[str]] = Field( 469 ["credentials", "refresh_token"], 470 description="Config path to the access token. Make sure the field actually exists in the config.", 471 examples=[["credentials", "refresh_token"], ["refresh_token"]], 472 title="Config Path To Refresh Token", 473 ) 474 token_expiry_date_config_path: Optional[List[str]] = Field( 475 ["credentials", "token_expiry_date"], 476 description="Config path to the expiry date. Make sure actually exists in the config.", 477 examples=[["credentials", "token_expiry_date"]], 478 title="Config Path To Expiry Date", 479 ) 480 refresh_token_error_status_codes: Optional[List[int]] = Field( 481 [], 482 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 483 examples=[[400, 500]], 484 title="Refresh Token Error Status Codes", 485 ) 486 refresh_token_error_key: Optional[str] = Field( 487 "", 488 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 489 examples=["error"], 490 title="Refresh Token Error Key", 491 ) 492 refresh_token_error_values: Optional[List[str]] = Field( 493 [], 494 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 495 examples=[["invalid_grant", "invalid_permissions"]], 496 title="Refresh Token Error Values", 497 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
500class OAuthAuthenticator(BaseModel): 501 type: Literal["OAuthAuthenticator"] 502 client_id_name: Optional[str] = Field( 503 "client_id", 504 description="The name of the property to use to refresh the `access_token`.", 505 examples=["custom_app_id"], 506 title="Client ID Property Name", 507 ) 508 client_id: Optional[str] = Field( 509 None, 510 description="The OAuth client ID. Fill it in the user inputs.", 511 examples=[ 512 "{{ config['client_id'] }}", 513 "{{ config['credentials']['client_id }}", 514 ], 515 title="Client ID", 516 ) 517 client_secret_name: Optional[str] = Field( 518 "client_secret", 519 description="The name of the property to use to refresh the `access_token`.", 520 examples=["custom_app_secret"], 521 title="Client Secret Property Name", 522 ) 523 client_secret: Optional[str] = Field( 524 None, 525 description="The OAuth client secret. Fill it in the user inputs.", 526 examples=[ 527 "{{ config['client_secret'] }}", 528 "{{ config['credentials']['client_secret }}", 529 ], 530 title="Client Secret", 531 ) 532 refresh_token_name: Optional[str] = Field( 533 "refresh_token", 534 description="The name of the property to use to refresh the `access_token`.", 535 examples=["custom_app_refresh_value"], 536 title="Refresh Token Property Name", 537 ) 538 refresh_token: Optional[str] = Field( 539 None, 540 description="Credential artifact used to get a new access token.", 541 examples=[ 542 "{{ config['refresh_token'] }}", 543 "{{ config['credentials]['refresh_token'] }}", 544 ], 545 title="Refresh Token", 546 ) 547 token_refresh_endpoint: Optional[str] = Field( 548 None, 549 description="The full URL to call to obtain a new access token.", 550 examples=["https://connect.squareup.com/oauth2/token"], 551 title="Token Refresh Endpoint", 552 ) 553 access_token_name: Optional[str] = Field( 554 "access_token", 555 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 556 examples=["access_token"], 557 title="Access Token Property Name", 558 ) 559 access_token_value: Optional[str] = Field( 560 None, 561 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 562 examples=["secret_access_token_value"], 563 title="Access Token Value", 564 ) 565 expires_in_name: Optional[str] = Field( 566 "expires_in", 567 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 568 examples=["expires_in"], 569 title="Token Expiry Property Name", 570 ) 571 grant_type_name: Optional[str] = Field( 572 "grant_type", 573 description="The name of the property to use to refresh the `access_token`.", 574 examples=["custom_grant_type"], 575 title="Grant Type Property Name", 576 ) 577 grant_type: Optional[str] = Field( 578 "refresh_token", 579 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 580 examples=["refresh_token", "client_credentials"], 581 title="Grant Type", 582 ) 583 refresh_request_body: Optional[Dict[str, Any]] = Field( 584 None, 585 description="Body of the request sent to get a new access token.", 586 examples=[ 587 { 588 "applicationId": "{{ config['application_id'] }}", 589 "applicationSecret": "{{ config['application_secret'] }}", 590 "token": "{{ config['token'] }}", 591 } 592 ], 593 title="Refresh Request Body", 594 ) 595 refresh_request_headers: Optional[Dict[str, Any]] = Field( 596 None, 597 description="Headers of the request sent to get a new access token.", 598 examples=[ 599 { 600 "Authorization": "<AUTH_TOKEN>", 601 "Content-Type": "application/x-www-form-urlencoded", 602 } 603 ], 604 title="Refresh Request Headers", 605 ) 606 scopes: Optional[List[str]] = Field( 607 None, 608 description="List of scopes that should be granted to the access token.", 609 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 610 title="Scopes", 611 ) 612 token_expiry_date: Optional[str] = Field( 613 None, 614 description="The access token expiry date.", 615 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 616 title="Token Expiry Date", 617 ) 618 token_expiry_date_format: Optional[str] = Field( 619 None, 620 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 621 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 622 title="Token Expiry Date Format", 623 ) 624 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 625 None, 626 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 627 title="Refresh Token Updater", 628 ) 629 profile_assertion: Optional[JwtAuthenticator] = Field( 630 None, 631 description="The authenticator being used to authenticate the client authenticator.", 632 title="Profile Assertion", 633 ) 634 use_profile_assertion: Optional[bool] = Field( 635 False, 636 description="Enable using profile assertion as a flow for OAuth authorization.", 637 title="Use Profile Assertion", 638 ) 639 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
642class Rate(BaseModel): 643 class Config: 644 extra = Extra.allow 645 646 limit: Union[int, str] = Field( 647 ..., 648 description="The maximum number of calls allowed within the interval.", 649 title="Limit", 650 ) 651 interval: str = Field( 652 ..., 653 description="The time interval for the rate limit.", 654 examples=["PT1H", "P1D"], 655 title="Interval", 656 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
659class HttpRequestRegexMatcher(BaseModel): 660 class Config: 661 extra = Extra.allow 662 663 method: Optional[str] = Field( 664 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 665 ) 666 url_base: Optional[str] = Field( 667 None, 668 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 669 title="URL Base", 670 ) 671 url_path_pattern: Optional[str] = Field( 672 None, 673 description="A regular expression pattern to match the URL path.", 674 title="URL Path Pattern", 675 ) 676 params: Optional[Dict[str, Any]] = Field( 677 None, description="The query parameters to match.", title="Parameters" 678 ) 679 headers: Optional[Dict[str, Any]] = Field( 680 None, description="The headers to match.", title="Headers" 681 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
684class DpathExtractor(BaseModel): 685 type: Literal["DpathExtractor"] 686 field_path: List[str] = Field( 687 ..., 688 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 689 examples=[ 690 ["data"], 691 ["data", "records"], 692 ["data", "{{ parameters.name }}"], 693 ["data", "*", "record"], 694 ], 695 title="Field Path", 696 ) 697 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
705class ExponentialBackoffStrategy(BaseModel): 706 type: Literal["ExponentialBackoffStrategy"] 707 factor: Optional[Union[float, str]] = Field( 708 5, 709 description="Multiplicative constant applied on each retry.", 710 examples=[5, 5.5, "10"], 711 title="Factor", 712 ) 713 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
716class GroupByKeyMergeStrategy(BaseModel): 717 type: Literal["GroupByKeyMergeStrategy"] 718 key: Union[str, List[str]] = Field( 719 ..., 720 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 721 examples=["id", ["parent_id", "end_date"]], 722 title="Key", 723 ) 724 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
736class Action(Enum): 737 SUCCESS = "SUCCESS" 738 FAIL = "FAIL" 739 RETRY = "RETRY" 740 IGNORE = "IGNORE" 741 RATE_LIMITED = "RATE_LIMITED"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
744class FailureType(Enum): 745 system_error = "system_error" 746 config_error = "config_error" 747 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
750class HttpResponseFilter(BaseModel): 751 type: Literal["HttpResponseFilter"] 752 action: Optional[Action] = Field( 753 None, 754 description="Action to execute if a response matches the filter.", 755 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 756 title="Action", 757 ) 758 failure_type: Optional[FailureType] = Field( 759 None, 760 description="Failure type of traced exception if a response matches the filter.", 761 examples=["system_error", "config_error", "transient_error"], 762 title="Failure Type", 763 ) 764 error_message: Optional[str] = Field( 765 None, 766 description="Error Message to display if the response matches the filter.", 767 title="Error Message", 768 ) 769 error_message_contains: Optional[str] = Field( 770 None, 771 description="Match the response if its error message contains the substring.", 772 example=["This API operation is not enabled for this site"], 773 title="Error Message Substring", 774 ) 775 http_codes: Optional[List[int]] = Field( 776 None, 777 description="Match the response if its HTTP code is included in this list.", 778 examples=[[420, 429], [500]], 779 title="HTTP Codes", 780 unique_items=True, 781 ) 782 predicate: Optional[str] = Field( 783 None, 784 description="Match the response if the predicate evaluates to true.", 785 examples=[ 786 "{{ 'Too much requests' in response }}", 787 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 788 ], 789 title="Predicate", 790 ) 791 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
794class ComplexFieldType(BaseModel): 795 field_type: str 796 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
799class TypesMap(BaseModel): 800 target_type: Union[str, List[str], ComplexFieldType] 801 current_type: Union[str, List[str]] 802 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
805class SchemaTypeIdentifier(BaseModel): 806 type: Optional[Literal["SchemaTypeIdentifier"]] = None 807 schema_pointer: Optional[List[str]] = Field( 808 [], 809 description="List of nested fields defining the schema field path to extract. Defaults to [].", 810 title="Schema Path", 811 ) 812 key_pointer: List[str] = Field( 813 ..., 814 description="List of potentially nested fields describing the full path of the field key to extract.", 815 title="Key Path", 816 ) 817 type_pointer: Optional[List[str]] = Field( 818 None, 819 description="List of potentially nested fields describing the full path of the field type to extract.", 820 title="Type Path", 821 ) 822 types_mapping: Optional[List[TypesMap]] = None 823 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
826class InlineSchemaLoader(BaseModel): 827 type: Literal["InlineSchemaLoader"] 828 schema_: Optional[Dict[str, Any]] = Field( 829 None, 830 alias="schema", 831 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 832 title="Schema", 833 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
836class JsonFileSchemaLoader(BaseModel): 837 type: Literal["JsonFileSchemaLoader"] 838 file_path: Optional[str] = Field( 839 None, 840 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 841 example=["./schemas/users.json"], 842 title="File Path", 843 ) 844 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
865class FlattenFields(BaseModel): 866 type: Literal["FlattenFields"] 867 flatten_lists: Optional[bool] = Field( 868 True, 869 description="Whether to flatten lists or leave it as is. Default is True.", 870 title="Flatten Lists", 871 ) 872 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
875class KeyTransformation(BaseModel): 876 type: Literal["KeyTransformation"] 877 prefix: Optional[str] = Field( 878 None, 879 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 880 examples=["flattened_"], 881 title="Key Prefix", 882 ) 883 suffix: Optional[str] = Field( 884 None, 885 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 886 examples=["_flattened"], 887 title="Key Suffix", 888 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
891class DpathFlattenFields(BaseModel): 892 type: Literal["DpathFlattenFields"] 893 field_path: List[str] = Field( 894 ..., 895 description="A path to field that needs to be flattened.", 896 examples=[["data"], ["data", "*", "field"]], 897 title="Field Path", 898 ) 899 delete_origin_value: Optional[bool] = Field( 900 None, 901 description="Whether to delete the origin value or keep it. Default is False.", 902 title="Delete Origin Value", 903 ) 904 replace_record: Optional[bool] = Field( 905 None, 906 description="Whether to replace the origin record or not. Default is False.", 907 title="Replace Origin Record", 908 ) 909 key_transformation: Optional[KeyTransformation] = Field( 910 None, 911 description="Transformation for object keys. If not provided, original key will be used.", 912 title="Key transformation", 913 ) 914 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
917class KeysReplace(BaseModel): 918 type: Literal["KeysReplace"] 919 old: str = Field( 920 ..., 921 description="Old value to replace.", 922 examples=[ 923 " ", 924 "{{ record.id }}", 925 "{{ config['id'] }}", 926 "{{ stream_slice['id'] }}", 927 ], 928 title="Old value", 929 ) 930 new: str = Field( 931 ..., 932 description="New value to set.", 933 examples=[ 934 "_", 935 "{{ record.id }}", 936 "{{ config['id'] }}", 937 "{{ stream_slice['id'] }}", 938 ], 939 title="New value", 940 ) 941 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
952class CustomDecoder(BaseModel): 953 class Config: 954 extra = Extra.allow 955 956 type: Literal["CustomDecoder"] 957 class_name: str = Field( 958 ..., 959 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 960 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 961 title="Class Name", 962 ) 963 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
966class MinMaxDatetime(BaseModel): 967 type: Literal["MinMaxDatetime"] 968 datetime: str = Field( 969 ..., 970 description="Datetime value.", 971 examples=[ 972 "2021-01-01", 973 "2021-01-01T00:00:00Z", 974 "{{ config['start_time'] }}", 975 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 976 ], 977 title="Datetime", 978 ) 979 datetime_format: Optional[str] = Field( 980 "", 981 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 982 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 983 title="Datetime Format", 984 ) 985 max_datetime: Optional[str] = Field( 986 None, 987 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 988 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 989 title="Max Datetime", 990 ) 991 min_datetime: Optional[str] = Field( 992 None, 993 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 994 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 995 title="Min Datetime", 996 ) 997 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1017class OauthConnectorInputSpecification(BaseModel): 1018 class Config: 1019 extra = Extra.allow 1020 1021 consent_url: str = Field( 1022 ..., 1023 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1024 examples=[ 1025 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1026 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1027 ], 1028 title="Consent URL", 1029 ) 1030 scope: Optional[str] = Field( 1031 None, 1032 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1033 examples=["user:read user:read_orders workspaces:read"], 1034 title="Scopes", 1035 ) 1036 access_token_url: str = Field( 1037 ..., 1038 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1039 examples=[ 1040 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1041 ], 1042 title="Access Token URL", 1043 ) 1044 access_token_headers: Optional[Dict[str, Any]] = Field( 1045 None, 1046 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1047 examples=[ 1048 { 1049 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1050 } 1051 ], 1052 title="Access Token Headers", 1053 ) 1054 access_token_params: Optional[Dict[str, Any]] = Field( 1055 None, 1056 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1057 examples=[ 1058 { 1059 "{{ auth_code_key }}": "{{ auth_code_value }}", 1060 "{{ client_id_key }}": "{{ client_id_value }}", 1061 "{{ client_secret_key }}": "{{ client_secret_value }}", 1062 } 1063 ], 1064 title="Access Token Query Params (Json Encoded)", 1065 ) 1066 extract_output: Optional[List[str]] = Field( 1067 None, 1068 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1069 examples=[["access_token", "refresh_token", "other_field"]], 1070 title="Extract Output", 1071 ) 1072 state: Optional[State] = Field( 1073 None, 1074 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1075 examples=[{"min": 7, "max": 128}], 1076 title="Configurable State Query Param", 1077 ) 1078 client_id_key: Optional[str] = Field( 1079 None, 1080 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1081 examples=["my_custom_client_id_key_name"], 1082 title="Client ID Key Override", 1083 ) 1084 client_secret_key: Optional[str] = Field( 1085 None, 1086 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1087 examples=["my_custom_client_secret_key_name"], 1088 title="Client Secret Key Override", 1089 ) 1090 scope_key: Optional[str] = Field( 1091 None, 1092 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1093 examples=["my_custom_scope_key_key_name"], 1094 title="Scopes Key Override", 1095 ) 1096 state_key: Optional[str] = Field( 1097 None, 1098 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1099 examples=["my_custom_state_key_key_name"], 1100 title="State Key Override", 1101 ) 1102 auth_code_key: Optional[str] = Field( 1103 None, 1104 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1105 examples=["my_custom_auth_code_key_name"], 1106 title="Auth Code Key Override", 1107 ) 1108 redirect_uri_key: Optional[str] = Field( 1109 None, 1110 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1111 examples=["my_custom_redirect_uri_key_name"], 1112 title="Redirect URI Key Override", 1113 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1116class OAuthConfigSpecification(BaseModel): 1117 class Config: 1118 extra = Extra.allow 1119 1120 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1121 None, 1122 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1123 examples=[ 1124 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1125 { 1126 "app_id": { 1127 "type": "string", 1128 "path_in_connector_config": ["info", "app_id"], 1129 } 1130 }, 1131 ], 1132 title="OAuth user input", 1133 ) 1134 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1135 None, 1136 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1137 title="DeclarativeOAuth Connector Specification", 1138 ) 1139 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1140 None, 1141 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1142 examples=[ 1143 { 1144 "refresh_token": { 1145 "type": "string,", 1146 "path_in_connector_config": ["credentials", "refresh_token"], 1147 } 1148 } 1149 ], 1150 title="OAuth output specification", 1151 ) 1152 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1153 None, 1154 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1155 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1156 title="OAuth input specification", 1157 ) 1158 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1159 None, 1160 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1161 examples=[ 1162 { 1163 "client_id": { 1164 "type": "string,", 1165 "path_in_connector_config": ["credentials", "client_id"], 1166 }, 1167 "client_secret": { 1168 "type": "string,", 1169 "path_in_connector_config": ["credentials", "client_secret"], 1170 }, 1171 } 1172 ], 1173 title="OAuth server output specification", 1174 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1177class OffsetIncrement(BaseModel): 1178 type: Literal["OffsetIncrement"] 1179 page_size: Optional[Union[int, str]] = Field( 1180 None, 1181 description="The number of records to include in each pages.", 1182 examples=[100, "{{ config['page_size'] }}"], 1183 title="Limit", 1184 ) 1185 inject_on_first_request: Optional[bool] = Field( 1186 False, 1187 description="Using the `offset` with value `0` during the first request", 1188 title="Inject Offset on First Request", 1189 ) 1190 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1193class PageIncrement(BaseModel): 1194 type: Literal["PageIncrement"] 1195 page_size: Optional[Union[int, str]] = Field( 1196 None, 1197 description="The number of records to include in each pages.", 1198 examples=[100, "100", "{{ config['page_size'] }}"], 1199 title="Page Size", 1200 ) 1201 start_from_page: Optional[int] = Field( 1202 0, 1203 description="Index of the first page to request.", 1204 examples=[0, 1], 1205 title="Start From Page", 1206 ) 1207 inject_on_first_request: Optional[bool] = Field( 1208 False, 1209 description="Using the `page number` with value defined by `start_from_page` during the first request", 1210 title="Inject Page Number on First Request", 1211 ) 1212 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1215class PrimaryKey(BaseModel): 1216 __root__: Union[str, List[str], List[List[str]]] = Field( 1217 ..., 1218 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1219 examples=["id", ["code", "type"]], 1220 title="Primary Key", 1221 )
class
PropertyLimitType(enum.Enum):
1224class PropertyLimitType(Enum): 1225 characters = "characters" 1226 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1229class PropertyChunking(BaseModel): 1230 type: Literal["PropertyChunking"] 1231 property_limit_type: PropertyLimitType = Field( 1232 ..., 1233 description="The type used to determine the maximum number of properties per chunk", 1234 title="Property Limit Type", 1235 ) 1236 property_limit: Optional[int] = Field( 1237 None, 1238 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1239 title="Property Limit", 1240 ) 1241 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1242 None, 1243 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1244 title="Record Merge Strategy", 1245 ) 1246 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1249class RecordFilter(BaseModel): 1250 type: Literal["RecordFilter"] 1251 condition: Optional[str] = Field( 1252 "", 1253 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1254 examples=[ 1255 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1256 "{{ record.status in ['active', 'expired'] }}", 1257 ], 1258 title="Condition", 1259 ) 1260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1268class RemoveFields(BaseModel): 1269 type: Literal["RemoveFields"] 1270 condition: Optional[str] = Field( 1271 "", 1272 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1273 examples=[ 1274 "{{ property|string == '' }}", 1275 "{{ property is integer }}", 1276 "{{ property|length > 5 }}", 1277 "{{ property == 'some_string_to_match' }}", 1278 ], 1279 ) 1280 field_pointers: List[List[str]] = Field( 1281 ..., 1282 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1283 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1284 title="Field Paths", 1285 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1292class InjectInto(Enum): 1293 request_parameter = "request_parameter" 1294 header = "header" 1295 body_data = "body_data" 1296 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1299class RequestOption(BaseModel): 1300 type: Literal["RequestOption"] 1301 inject_into: InjectInto = Field( 1302 ..., 1303 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1304 examples=["request_parameter", "header", "body_data", "body_json"], 1305 title="Inject Into", 1306 ) 1307 field_name: Optional[str] = Field( 1308 None, 1309 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1310 examples=["segment_id"], 1311 title="Field Name", 1312 ) 1313 field_path: Optional[List[str]] = Field( 1314 None, 1315 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1316 examples=[["data", "viewer", "id"]], 1317 title="Field Path", 1318 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1328class LegacySessionTokenAuthenticator(BaseModel): 1329 type: Literal["LegacySessionTokenAuthenticator"] 1330 header: str = Field( 1331 ..., 1332 description="The name of the session token header that will be injected in the request", 1333 examples=["X-Session"], 1334 title="Session Request Header", 1335 ) 1336 login_url: str = Field( 1337 ..., 1338 description="Path of the login URL (do not include the base URL)", 1339 examples=["session"], 1340 title="Login Path", 1341 ) 1342 session_token: Optional[str] = Field( 1343 None, 1344 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1345 example=["{{ config['session_token'] }}"], 1346 title="Session Token", 1347 ) 1348 session_token_response_key: str = Field( 1349 ..., 1350 description="Name of the key of the session token to be extracted from the response", 1351 examples=["id"], 1352 title="Response Token Response Key", 1353 ) 1354 username: Optional[str] = Field( 1355 None, 1356 description="Username used to authenticate and obtain a session token", 1357 examples=[" {{ config['username'] }}"], 1358 title="Username", 1359 ) 1360 password: Optional[str] = Field( 1361 "", 1362 description="Password used to authenticate and obtain a session token", 1363 examples=["{{ config['password'] }}", ""], 1364 title="Password", 1365 ) 1366 validate_session_url: str = Field( 1367 ..., 1368 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1369 examples=["user/current"], 1370 title="Validate Session Path", 1371 ) 1372 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1390class ValueType(Enum): 1391 string = "string" 1392 number = "number" 1393 integer = "integer" 1394 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1397class WaitTimeFromHeader(BaseModel): 1398 type: Literal["WaitTimeFromHeader"] 1399 header: str = Field( 1400 ..., 1401 description="The name of the response header defining how long to wait before retrying.", 1402 examples=["Retry-After"], 1403 title="Response Header Name", 1404 ) 1405 regex: Optional[str] = Field( 1406 None, 1407 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1408 examples=["([-+]?\\d+)"], 1409 title="Extraction Regex", 1410 ) 1411 max_waiting_time_in_seconds: Optional[float] = Field( 1412 None, 1413 description="Given the value extracted from the header is greater than this value, stop the stream.", 1414 examples=[3600], 1415 title="Max Waiting Time in Seconds", 1416 ) 1417 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1420class WaitUntilTimeFromHeader(BaseModel): 1421 type: Literal["WaitUntilTimeFromHeader"] 1422 header: str = Field( 1423 ..., 1424 description="The name of the response header defining how long to wait before retrying.", 1425 examples=["wait_time"], 1426 title="Response Header", 1427 ) 1428 min_wait: Optional[Union[float, str]] = Field( 1429 None, 1430 description="Minimum time to wait before retrying.", 1431 examples=[10, "60"], 1432 title="Minimum Wait Time", 1433 ) 1434 regex: Optional[str] = Field( 1435 None, 1436 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1437 examples=["([-+]?\\d+)"], 1438 title="Extraction Regex", 1439 ) 1440 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1443class ComponentMappingDefinition(BaseModel): 1444 type: Literal["ComponentMappingDefinition"] 1445 field_path: List[str] = Field( 1446 ..., 1447 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1448 examples=[ 1449 ["name"], 1450 ["retriever", "requester", "url"], 1451 ["retriever", "requester", "{{ components_values.field }}"], 1452 ["*", "**", "name"], 1453 ], 1454 title="Field Path", 1455 ) 1456 value: str = Field( 1457 ..., 1458 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1459 examples=[ 1460 "{{ components_values['updates'] }}", 1461 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1462 "{{ config['segment_id'] }}", 1463 "{{ stream_slice['parent_id'] }}", 1464 "{{ stream_slice['extra_fields']['name'] }}", 1465 ], 1466 title="Value", 1467 ) 1468 value_type: Optional[ValueType] = Field( 1469 None, 1470 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1471 title="Value Type", 1472 ) 1473 create_or_update: Optional[bool] = Field( 1474 False, 1475 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1476 title="Create or Update", 1477 ) 1478 condition: Optional[str] = Field( 1479 None, 1480 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1481 examples=[ 1482 "{{ components_values.get('cursor_field', None) }}", 1483 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1484 ], 1485 title="Condition", 1486 ) 1487 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1490class StreamConfig(BaseModel): 1491 type: Literal["StreamConfig"] 1492 configs_pointer: List[str] = Field( 1493 ..., 1494 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1495 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1496 title="Configs Pointer", 1497 ) 1498 default_values: Optional[List[Dict[str, Any]]] = Field( 1499 None, 1500 description="A list of default values, each matching the structure expected from the parsed component value.", 1501 title="Default Values", 1502 ) 1503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1506class ConfigComponentsResolver(BaseModel): 1507 type: Literal["ConfigComponentsResolver"] 1508 stream_config: Union[List[StreamConfig], StreamConfig] 1509 components_mapping: List[ComponentMappingDefinition] 1510 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1513class StreamParametersDefinition(BaseModel): 1514 type: Literal["StreamParametersDefinition"] 1515 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1516 ..., 1517 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1518 examples=[ 1519 [ 1520 { 1521 "name": "test stream", 1522 "$parameters": {"entity": "test entity"}, 1523 "primary_key": "test key", 1524 } 1525 ] 1526 ], 1527 title="Stream Parameters", 1528 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1531class ParametrizedComponentsResolver(BaseModel): 1532 type: Literal["ParametrizedComponentsResolver"] 1533 stream_parameters: StreamParametersDefinition 1534 components_mapping: List[ComponentMappingDefinition] 1535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1560class ValidateAdheresToSchema(BaseModel): 1561 type: Literal["ValidateAdheresToSchema"] 1562 base_schema: Union[str, Dict[str, Any]] = Field( 1563 ..., 1564 description="The base JSON schema against which the user-provided schema will be validated.", 1565 examples=[ 1566 "{{ config['report_validation_schema'] }}", 1567 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1568 { 1569 "$schema": "http://json-schema.org/draft-07/schema#", 1570 "title": "Person", 1571 "type": "object", 1572 "properties": { 1573 "name": {"type": "string", "description": "The person's name"}, 1574 "age": { 1575 "type": "integer", 1576 "minimum": 0, 1577 "description": "The person's age", 1578 }, 1579 }, 1580 "required": ["name", "age"], 1581 }, 1582 ], 1583 title="Base JSON Schema", 1584 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1587class CustomValidationStrategy(BaseModel): 1588 class Config: 1589 extra = Extra.allow 1590 1591 type: Literal["CustomValidationStrategy"] 1592 class_name: str = Field( 1593 ..., 1594 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1595 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1596 title="Class Name", 1597 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1600class ConfigRemapField(BaseModel): 1601 type: Literal["ConfigRemapField"] 1602 map: Union[Dict[str, Any], str] = Field( 1603 ..., 1604 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1605 examples=[ 1606 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1607 "{{ config['status_mapping'] }}", 1608 ], 1609 title="Value Mapping", 1610 ) 1611 field_path: List[str] = Field( 1612 ..., 1613 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1614 examples=[ 1615 ["status"], 1616 ["data", "status"], 1617 ["data", "{{ config.name }}", "status"], 1618 ["data", "*", "status"], 1619 ], 1620 title="Field Path", 1621 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1624class ConfigRemoveFields(BaseModel): 1625 type: Literal["ConfigRemoveFields"] 1626 field_pointers: List[List[str]] = Field( 1627 ..., 1628 description="A list of field pointers to be removed from the config.", 1629 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1630 title="Field Pointers", 1631 ) 1632 condition: Optional[str] = Field( 1633 "", 1634 description="Fields will be removed if expression is evaluated to True.", 1635 examples=[ 1636 "{{ config['environemnt'] == 'sandbox' }}", 1637 "{{ property is integer }}", 1638 "{{ property|length > 5 }}", 1639 "{{ property == 'some_string_to_match' }}", 1640 ], 1641 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1644class CustomConfigTransformation(BaseModel): 1645 type: Literal["CustomConfigTransformation"] 1646 class_name: str = Field( 1647 ..., 1648 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1649 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1650 ) 1651 parameters: Optional[Dict[str, Any]] = Field( 1652 None, 1653 alias="$parameters", 1654 description="Additional parameters to be passed to the custom config transformation.", 1655 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1658class AddedFieldDefinition(BaseModel): 1659 type: Literal["AddedFieldDefinition"] 1660 path: List[str] = Field( 1661 ..., 1662 description="List of strings defining the path where to add the value on the record.", 1663 examples=[["segment_id"], ["metadata", "segment_id"]], 1664 title="Path", 1665 ) 1666 value: str = Field( 1667 ..., 1668 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1669 examples=[ 1670 "{{ record['updates'] }}", 1671 "{{ record['MetaData']['LastUpdatedTime'] }}", 1672 "{{ stream_partition['segment_id'] }}", 1673 ], 1674 title="Value", 1675 ) 1676 value_type: Optional[ValueType] = Field( 1677 None, 1678 description="Type of the value. If not specified, the type will be inferred from the value.", 1679 title="Value Type", 1680 ) 1681 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1684class AddFields(BaseModel): 1685 type: Literal["AddFields"] 1686 fields: List[AddedFieldDefinition] = Field( 1687 ..., 1688 description="List of transformations (path and corresponding value) that will be added to the record.", 1689 title="Fields", 1690 ) 1691 condition: Optional[str] = Field( 1692 "", 1693 description="Fields will be added if expression is evaluated to True.", 1694 examples=[ 1695 "{{ property|string == '' }}", 1696 "{{ property is integer }}", 1697 "{{ property|length > 5 }}", 1698 "{{ property == 'some_string_to_match' }}", 1699 ], 1700 ) 1701 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1704class ApiKeyAuthenticator(BaseModel): 1705 type: Literal["ApiKeyAuthenticator"] 1706 api_token: Optional[str] = Field( 1707 None, 1708 description="The API key to inject in the request. Fill it in the user inputs.", 1709 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1710 title="API Key", 1711 ) 1712 header: Optional[str] = Field( 1713 None, 1714 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1715 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1716 title="Header Name", 1717 ) 1718 inject_into: Optional[RequestOption] = Field( 1719 None, 1720 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1721 examples=[ 1722 {"inject_into": "header", "field_name": "Authorization"}, 1723 {"inject_into": "request_parameter", "field_name": "authKey"}, 1724 ], 1725 title="Inject API Key Into Outgoing HTTP Request", 1726 ) 1727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1730class AuthFlow(BaseModel): 1731 auth_flow_type: Optional[AuthFlowType] = Field( 1732 None, description="The type of auth to use", title="Auth flow type" 1733 ) 1734 predicate_key: Optional[List[str]] = Field( 1735 None, 1736 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1737 examples=[["credentials", "auth_type"]], 1738 title="Predicate key", 1739 ) 1740 predicate_value: Optional[str] = Field( 1741 None, 1742 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1743 examples=["Oauth"], 1744 title="Predicate value", 1745 ) 1746 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1749class CheckStream(BaseModel): 1750 type: Literal["CheckStream"] 1751 stream_names: Optional[List[str]] = Field( 1752 None, 1753 description="Names of the streams to try reading from when running a check operation.", 1754 examples=[["users"], ["users", "contacts"]], 1755 title="Stream Names", 1756 ) 1757 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1760class IncrementingCountCursor(BaseModel): 1761 type: Literal["IncrementingCountCursor"] 1762 cursor_field: str = Field( 1763 ..., 1764 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1765 examples=["created_at", "{{ config['record_cursor'] }}"], 1766 title="Cursor Field", 1767 ) 1768 start_value: Optional[Union[str, int]] = Field( 1769 None, 1770 description="The value that determines the earliest record that should be synced.", 1771 examples=[0, "{{ config['start_value'] }}"], 1772 title="Start Value", 1773 ) 1774 start_value_option: Optional[RequestOption] = Field( 1775 None, 1776 description="Optionally configures how the start value will be sent in requests to the source API.", 1777 title="Inject Start Value Into Outgoing HTTP Request", 1778 ) 1779 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1782class DatetimeBasedCursor(BaseModel): 1783 type: Literal["DatetimeBasedCursor"] 1784 clamping: Optional[Clamping] = Field( 1785 None, 1786 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1787 title="Date Range Clamping", 1788 ) 1789 cursor_field: str = Field( 1790 ..., 1791 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1792 examples=["created_at", "{{ config['record_cursor'] }}"], 1793 title="Cursor Field", 1794 ) 1795 cursor_datetime_formats: Optional[List[str]] = Field( 1796 None, 1797 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1798 examples=[ 1799 "%Y-%m-%d", 1800 "%Y-%m-%d %H:%M:%S", 1801 "%Y-%m-%dT%H:%M:%S", 1802 "%Y-%m-%dT%H:%M:%SZ", 1803 "%Y-%m-%dT%H:%M:%S%z", 1804 "%Y-%m-%dT%H:%M:%S.%fZ", 1805 "%Y-%m-%dT%H:%M:%S.%f%z", 1806 "%Y-%m-%d %H:%M:%S.%f+00:00", 1807 "%s", 1808 "%ms", 1809 ], 1810 title="Cursor Datetime Formats", 1811 ) 1812 start_datetime: Union[MinMaxDatetime, str] = Field( 1813 ..., 1814 description="The datetime that determines the earliest record that should be synced.", 1815 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1816 title="Start Datetime", 1817 ) 1818 start_time_option: Optional[RequestOption] = Field( 1819 None, 1820 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1821 title="Inject Start Time Into Outgoing HTTP Request", 1822 ) 1823 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1824 None, 1825 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1826 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1827 title="End Datetime", 1828 ) 1829 end_time_option: Optional[RequestOption] = Field( 1830 None, 1831 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1832 title="Inject End Time Into Outgoing HTTP Request", 1833 ) 1834 datetime_format: str = Field( 1835 ..., 1836 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1837 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1838 title="Outgoing Datetime Format", 1839 ) 1840 cursor_granularity: Optional[str] = Field( 1841 None, 1842 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1843 examples=["PT1S"], 1844 title="Cursor Granularity", 1845 ) 1846 is_data_feed: Optional[bool] = Field( 1847 None, 1848 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1849 title="Data Feed API", 1850 ) 1851 is_client_side_incremental: Optional[bool] = Field( 1852 None, 1853 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1854 title="Client-side Incremental Filtering", 1855 ) 1856 is_compare_strictly: Optional[bool] = Field( 1857 False, 1858 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1859 title="Strict Start-End Time Comparison", 1860 ) 1861 global_substream_cursor: Optional[bool] = Field( 1862 False, 1863 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1864 title="Global Substream Cursor", 1865 ) 1866 lookback_window: Optional[str] = Field( 1867 None, 1868 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1869 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1870 title="Lookback Window", 1871 ) 1872 partition_field_end: Optional[str] = Field( 1873 None, 1874 description="Name of the partition start time field.", 1875 examples=["ending_time"], 1876 title="Partition Field End", 1877 ) 1878 partition_field_start: Optional[str] = Field( 1879 None, 1880 description="Name of the partition end time field.", 1881 examples=["starting_time"], 1882 title="Partition Field Start", 1883 ) 1884 step: Optional[str] = Field( 1885 None, 1886 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1887 examples=["P1W", "{{ config['step_increment'] }}"], 1888 title="Step", 1889 ) 1890 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1893class FixedWindowCallRatePolicy(BaseModel): 1894 class Config: 1895 extra = Extra.allow 1896 1897 type: Literal["FixedWindowCallRatePolicy"] 1898 period: str = Field( 1899 ..., description="The time interval for the rate limit window.", title="Period" 1900 ) 1901 call_limit: int = Field( 1902 ..., 1903 description="The maximum number of calls allowed within the period.", 1904 title="Call Limit", 1905 ) 1906 matchers: List[HttpRequestRegexMatcher] = Field( 1907 ..., 1908 description="List of matchers that define which requests this policy applies to.", 1909 title="Matchers", 1910 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1913class MovingWindowCallRatePolicy(BaseModel): 1914 class Config: 1915 extra = Extra.allow 1916 1917 type: Literal["MovingWindowCallRatePolicy"] 1918 rates: List[Rate] = Field( 1919 ..., 1920 description="List of rates that define the call limits for different time intervals.", 1921 title="Rates", 1922 ) 1923 matchers: List[HttpRequestRegexMatcher] = Field( 1924 ..., 1925 description="List of matchers that define which requests this policy applies to.", 1926 title="Matchers", 1927 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1930class UnlimitedCallRatePolicy(BaseModel): 1931 class Config: 1932 extra = Extra.allow 1933 1934 type: Literal["UnlimitedCallRatePolicy"] 1935 matchers: List[HttpRequestRegexMatcher] = Field( 1936 ..., 1937 description="List of matchers that define which requests this policy applies to.", 1938 title="Matchers", 1939 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1942class DefaultErrorHandler(BaseModel): 1943 type: Literal["DefaultErrorHandler"] 1944 backoff_strategies: Optional[ 1945 List[ 1946 Union[ 1947 ConstantBackoffStrategy, 1948 ExponentialBackoffStrategy, 1949 WaitTimeFromHeader, 1950 WaitUntilTimeFromHeader, 1951 CustomBackoffStrategy, 1952 ] 1953 ] 1954 ] = Field( 1955 None, 1956 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1957 title="Backoff Strategies", 1958 ) 1959 max_retries: Optional[int] = Field( 1960 5, 1961 description="The maximum number of time to retry a retryable request before giving up and failing.", 1962 examples=[5, 0, 10], 1963 title="Max Retry Count", 1964 ) 1965 response_filters: Optional[List[HttpResponseFilter]] = Field( 1966 None, 1967 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1968 title="Response Filters", 1969 ) 1970 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1973class DefaultPaginator(BaseModel): 1974 type: Literal["DefaultPaginator"] 1975 pagination_strategy: Union[ 1976 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1977 ] = Field( 1978 ..., 1979 description="Strategy defining how records are paginated.", 1980 title="Pagination Strategy", 1981 ) 1982 page_size_option: Optional[RequestOption] = Field( 1983 None, title="Inject Page Size Into Outgoing HTTP Request" 1984 ) 1985 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1986 None, 1987 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 1988 title="Inject Page Token Into Outgoing HTTP Request", 1989 ) 1990 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1993class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1994 type: Literal["ApiKey"] 1995 inject_into: RequestOption = Field( 1996 ..., 1997 description="Configure how the API Key will be sent in requests to the source API.", 1998 examples=[ 1999 {"inject_into": "header", "field_name": "Authorization"}, 2000 {"inject_into": "request_parameter", "field_name": "authKey"}, 2001 ], 2002 title="Inject API Key Into Outgoing HTTP Request", 2003 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2006class ListPartitionRouter(BaseModel): 2007 type: Literal["ListPartitionRouter"] 2008 cursor_field: str = Field( 2009 ..., 2010 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2011 examples=["section", "{{ config['section_key'] }}"], 2012 title="Current Partition Value Identifier", 2013 ) 2014 values: Union[str, List[str]] = Field( 2015 ..., 2016 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2017 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2018 title="Partition Values", 2019 ) 2020 request_option: Optional[RequestOption] = Field( 2021 None, 2022 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2023 title="Inject Partition Value Into Outgoing HTTP Request", 2024 ) 2025 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2028class RecordSelector(BaseModel): 2029 type: Literal["RecordSelector"] 2030 extractor: Union[DpathExtractor, CustomRecordExtractor] 2031 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2032 None, 2033 description="Responsible for filtering records to be emitted by the Source.", 2034 title="Record Filter", 2035 ) 2036 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2037 None, 2038 description="Responsible for normalization according to the schema.", 2039 title="Schema Normalization", 2040 ) 2041 transform_before_filtering: Optional[bool] = Field( 2042 None, 2043 description="If true, transformation will be applied before record filtering.", 2044 title="Transform Before Filtering", 2045 ) 2046 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2049class GzipDecoder(BaseModel): 2050 type: Literal["GzipDecoder"] 2051 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2054class RequestBodyGraphQL(BaseModel): 2055 type: Literal["RequestBodyGraphQL"] 2056 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2059class DpathValidator(BaseModel): 2060 type: Literal["DpathValidator"] 2061 field_path: List[str] = Field( 2062 ..., 2063 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2064 examples=[ 2065 ["data"], 2066 ["data", "records"], 2067 ["data", "{{ parameters.name }}"], 2068 ["data", "*", "record"], 2069 ], 2070 title="Field Path", 2071 ) 2072 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2073 ..., 2074 description="The condition that the specified config value will be evaluated against", 2075 title="Validation Strategy", 2076 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2079class PredicateValidator(BaseModel): 2080 type: Literal["PredicateValidator"] 2081 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2082 ..., 2083 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2084 examples=[ 2085 "test-value", 2086 "{{ config['api_version'] }}", 2087 "{{ config['tenant_id'] }}", 2088 123, 2089 ], 2090 title="Value", 2091 ) 2092 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2093 ..., 2094 description="The validation strategy to apply to the value.", 2095 title="Validation Strategy", 2096 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2099class ConfigAddFields(BaseModel): 2100 type: Literal["ConfigAddFields"] 2101 fields: List[AddedFieldDefinition] = Field( 2102 ..., 2103 description="A list of transformations (path and corresponding value) that will be added to the config.", 2104 title="Fields", 2105 ) 2106 condition: Optional[str] = Field( 2107 "", 2108 description="Fields will be added if expression is evaluated to True.", 2109 examples=[ 2110 "{{ config['environemnt'] == 'sandbox' }}", 2111 "{{ property is integer }}", 2112 "{{ property|length > 5 }}", 2113 "{{ property == 'some_string_to_match' }}", 2114 ], 2115 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2118class CompositeErrorHandler(BaseModel): 2119 type: Literal["CompositeErrorHandler"] 2120 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2121 ..., 2122 description="List of error handlers to iterate on to determine how to handle a failed response.", 2123 title="Error Handlers", 2124 ) 2125 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2128class HTTPAPIBudget(BaseModel): 2129 class Config: 2130 extra = Extra.allow 2131 2132 type: Literal["HTTPAPIBudget"] 2133 policies: List[ 2134 Union[ 2135 FixedWindowCallRatePolicy, 2136 MovingWindowCallRatePolicy, 2137 UnlimitedCallRatePolicy, 2138 ] 2139 ] = Field( 2140 ..., 2141 description="List of call rate policies that define how many calls are allowed.", 2142 title="Policies", 2143 ) 2144 ratelimit_reset_header: Optional[str] = Field( 2145 "ratelimit-reset", 2146 description="The HTTP response header name that indicates when the rate limit resets.", 2147 title="Rate Limit Reset Header", 2148 ) 2149 ratelimit_remaining_header: Optional[str] = Field( 2150 "ratelimit-remaining", 2151 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2152 title="Rate Limit Remaining Header", 2153 ) 2154 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2155 [429], 2156 description="List of HTTP status codes that indicate a rate limit has been hit.", 2157 title="Status Codes for Rate Limit Hit", 2158 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2161class ZipfileDecoder(BaseModel): 2162 class Config: 2163 extra = Extra.allow 2164 2165 type: Literal["ZipfileDecoder"] 2166 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2167 ..., 2168 description="Parser to parse the decompressed data from the zipfile(s).", 2169 title="Parser", 2170 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2173class ConfigMigration(BaseModel): 2174 type: Literal["ConfigMigration"] 2175 description: Optional[str] = Field( 2176 None, description="The description/purpose of the config migration." 2177 ) 2178 transformations: List[ 2179 Union[ 2180 ConfigRemapField, 2181 ConfigAddFields, 2182 ConfigRemoveFields, 2183 CustomConfigTransformation, 2184 ] 2185 ] = Field( 2186 ..., 2187 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2188 title="Transformations", 2189 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2192class ConfigNormalizationRules(BaseModel): 2193 class Config: 2194 extra = Extra.forbid 2195 2196 type: Literal["ConfigNormalizationRules"] 2197 config_migrations: Optional[List[ConfigMigration]] = Field( 2198 [], 2199 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2200 title="Config Migrations", 2201 ) 2202 transformations: Optional[ 2203 List[ 2204 Union[ 2205 ConfigRemapField, 2206 ConfigAddFields, 2207 ConfigRemoveFields, 2208 CustomConfigTransformation, 2209 ] 2210 ] 2211 ] = Field( 2212 [], 2213 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2214 title="Transformations", 2215 ) 2216 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2217 [], 2218 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2219 title="Validations", 2220 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2223class Spec(BaseModel): 2224 type: Literal["Spec"] 2225 connection_specification: Dict[str, Any] = Field( 2226 ..., 2227 description="A connection specification describing how a the connector can be configured.", 2228 title="Connection Specification", 2229 ) 2230 documentation_url: Optional[str] = Field( 2231 None, 2232 description="URL of the connector's documentation page.", 2233 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2234 title="Documentation URL", 2235 ) 2236 advanced_auth: Optional[AuthFlow] = Field( 2237 None, 2238 description="Advanced specification for configuring the authentication flow.", 2239 title="Advanced Auth", 2240 ) 2241 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2242 None, title="Config Normalization Rules" 2243 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2246class DeclarativeSource1(BaseModel): 2247 class Config: 2248 extra = Extra.forbid 2249 2250 type: Literal["DeclarativeSource"] 2251 check: Union[CheckStream, CheckDynamicStream] 2252 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2253 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2254 version: str = Field( 2255 ..., 2256 description="The version of the Airbyte CDK used to build and test the source.", 2257 ) 2258 schemas: Optional[Schemas] = None 2259 definitions: Optional[Dict[str, Any]] = None 2260 spec: Optional[Spec] = None 2261 concurrency_level: Optional[ConcurrencyLevel] = None 2262 api_budget: Optional[HTTPAPIBudget] = None 2263 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2264 None, 2265 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2266 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2267 title="Maximum Concurrent Asynchronous Jobs", 2268 ) 2269 metadata: Optional[Dict[str, Any]] = Field( 2270 None, 2271 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2272 ) 2273 description: Optional[str] = Field( 2274 None, 2275 description="A description of the connector. It will be presented on the Source documentation page.", 2276 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2279class DeclarativeSource2(BaseModel): 2280 class Config: 2281 extra = Extra.forbid 2282 2283 type: Literal["DeclarativeSource"] 2284 check: Union[CheckStream, CheckDynamicStream] 2285 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2286 None 2287 ) 2288 dynamic_streams: List[DynamicDeclarativeStream] 2289 version: str = Field( 2290 ..., 2291 description="The version of the Airbyte CDK used to build and test the source.", 2292 ) 2293 schemas: Optional[Schemas] = None 2294 definitions: Optional[Dict[str, Any]] = None 2295 spec: Optional[Spec] = None 2296 concurrency_level: Optional[ConcurrencyLevel] = None 2297 api_budget: Optional[HTTPAPIBudget] = None 2298 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2299 None, 2300 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2301 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2302 title="Maximum Concurrent Asynchronous Jobs", 2303 ) 2304 metadata: Optional[Dict[str, Any]] = Field( 2305 None, 2306 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2307 ) 2308 description: Optional[str] = Field( 2309 None, 2310 description="A description of the connector. It will be presented on the Source documentation page.", 2311 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2314class DeclarativeSource(BaseModel): 2315 class Config: 2316 extra = Extra.forbid 2317 2318 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2319 ..., 2320 description="An API source that extracts data according to its declarative components.", 2321 title="DeclarativeSource", 2322 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2325class SelectiveAuthenticator(BaseModel): 2326 class Config: 2327 extra = Extra.allow 2328 2329 type: Literal["SelectiveAuthenticator"] 2330 authenticator_selection_path: List[str] = Field( 2331 ..., 2332 description="Path of the field in config with selected authenticator name", 2333 examples=[["auth"], ["auth", "type"]], 2334 title="Authenticator Selection Path", 2335 ) 2336 authenticators: Dict[ 2337 str, 2338 Union[ 2339 ApiKeyAuthenticator, 2340 BasicHttpAuthenticator, 2341 BearerAuthenticator, 2342 OAuthAuthenticator, 2343 JwtAuthenticator, 2344 SessionTokenAuthenticator, 2345 LegacySessionTokenAuthenticator, 2346 CustomAuthenticator, 2347 NoAuth, 2348 ], 2349 ] = Field( 2350 ..., 2351 description="Authenticators to select from.", 2352 examples=[ 2353 { 2354 "authenticators": { 2355 "token": "#/definitions/ApiKeyAuthenticator", 2356 "oauth": "#/definitions/OAuthAuthenticator", 2357 "jwt": "#/definitions/JwtAuthenticator", 2358 } 2359 } 2360 ], 2361 title="Authenticators", 2362 ) 2363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2366class ConditionalStreams(BaseModel): 2367 type: Literal["ConditionalStreams"] 2368 condition: str = Field( 2369 ..., 2370 description="Condition that will be evaluated to determine if a set of streams should be available.", 2371 examples=["{{ config['is_sandbox'] }}"], 2372 title="Condition", 2373 ) 2374 streams: List[DeclarativeStream] = Field( 2375 ..., 2376 description="Streams that will be used during an operation based on the condition.", 2377 title="Streams", 2378 ) 2379 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2382class FileUploader(BaseModel): 2383 type: Literal["FileUploader"] 2384 requester: Union[HttpRequester, CustomRequester] = Field( 2385 ..., 2386 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2387 ) 2388 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2389 ..., 2390 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2391 ) 2392 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2393 None, 2394 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2395 ) 2396 filename_extractor: Optional[str] = Field( 2397 None, 2398 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2399 examples=[ 2400 "{{ record.id }}/{{ record.file_name }}/", 2401 "{{ record.id }}_{{ record.file_name }}/", 2402 ], 2403 ) 2404 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2407class DeclarativeStream(BaseModel): 2408 class Config: 2409 extra = Extra.allow 2410 2411 type: Literal["DeclarativeStream"] 2412 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2413 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2414 ..., 2415 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2416 title="Retriever", 2417 ) 2418 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2419 None, 2420 description="Component used to fetch data incrementally based on a time field in the data.", 2421 title="Incremental Sync", 2422 ) 2423 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2424 schema_loader: Optional[ 2425 Union[ 2426 InlineSchemaLoader, 2427 DynamicSchemaLoader, 2428 JsonFileSchemaLoader, 2429 List[ 2430 Union[ 2431 InlineSchemaLoader, 2432 DynamicSchemaLoader, 2433 JsonFileSchemaLoader, 2434 CustomSchemaLoader, 2435 ] 2436 ], 2437 CustomSchemaLoader, 2438 ] 2439 ] = Field( 2440 None, 2441 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2442 title="Schema Loader", 2443 ) 2444 transformations: Optional[ 2445 List[ 2446 Union[ 2447 AddFields, 2448 RemoveFields, 2449 KeysToLower, 2450 KeysToSnakeCase, 2451 FlattenFields, 2452 DpathFlattenFields, 2453 KeysReplace, 2454 CustomTransformation, 2455 ] 2456 ] 2457 ] = Field( 2458 None, 2459 description="A list of transformations to be applied to each output record.", 2460 title="Transformations", 2461 ) 2462 state_migrations: Optional[ 2463 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2464 ] = Field( 2465 [], 2466 description="Array of state migrations to be applied on the input state", 2467 title="State Migrations", 2468 ) 2469 file_uploader: Optional[FileUploader] = Field( 2470 None, 2471 description="(experimental) Describes how to fetch a file", 2472 title="File Uploader", 2473 ) 2474 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2477class SessionTokenAuthenticator(BaseModel): 2478 type: Literal["SessionTokenAuthenticator"] 2479 login_requester: HttpRequester = Field( 2480 ..., 2481 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2482 examples=[ 2483 { 2484 "type": "HttpRequester", 2485 "url_base": "https://my_api.com", 2486 "path": "/login", 2487 "authenticator": { 2488 "type": "BasicHttpAuthenticator", 2489 "username": "{{ config.username }}", 2490 "password": "{{ config.password }}", 2491 }, 2492 } 2493 ], 2494 title="Login Requester", 2495 ) 2496 session_token_path: List[str] = Field( 2497 ..., 2498 description="The path in the response body returned from the login requester to the session token.", 2499 examples=[["access_token"], ["result", "token"]], 2500 title="Session Token Path", 2501 ) 2502 expiration_duration: Optional[str] = Field( 2503 None, 2504 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2505 examples=["PT1H", "P1D"], 2506 title="Expiration Duration", 2507 ) 2508 request_authentication: Union[ 2509 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2510 ] = Field( 2511 ..., 2512 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2513 title="Data Request Authentication", 2514 ) 2515 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2516 None, description="Component used to decode the response.", title="Decoder" 2517 ) 2518 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2521class HttpRequester(BaseModelWithDeprecations): 2522 type: Literal["HttpRequester"] 2523 url_base: Optional[str] = Field( 2524 None, 2525 deprecated=True, 2526 deprecation_message="Use `url` field instead.", 2527 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2528 examples=[ 2529 "https://connect.squareup.com/v2", 2530 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2531 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2532 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2533 ], 2534 title="API Base URL", 2535 ) 2536 url: Optional[str] = Field( 2537 None, 2538 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2539 examples=[ 2540 "https://connect.squareup.com/v2", 2541 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2542 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2543 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2544 ], 2545 title="API Endpoint URL", 2546 ) 2547 path: Optional[str] = Field( 2548 None, 2549 deprecated=True, 2550 deprecation_message="Use `url` field instead.", 2551 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2552 examples=[ 2553 "/products", 2554 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2555 "/trades/{{ config['symbol_id'] }}/history", 2556 ], 2557 title="URL Path", 2558 ) 2559 http_method: Optional[HttpMethod] = Field( 2560 HttpMethod.GET, 2561 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2562 examples=["GET", "POST"], 2563 title="HTTP Method", 2564 ) 2565 authenticator: Optional[ 2566 Union[ 2567 ApiKeyAuthenticator, 2568 BasicHttpAuthenticator, 2569 BearerAuthenticator, 2570 OAuthAuthenticator, 2571 JwtAuthenticator, 2572 SessionTokenAuthenticator, 2573 SelectiveAuthenticator, 2574 CustomAuthenticator, 2575 NoAuth, 2576 LegacySessionTokenAuthenticator, 2577 ] 2578 ] = Field( 2579 None, 2580 description="Authentication method to use for requests sent to the API.", 2581 title="Authenticator", 2582 ) 2583 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2584 None, 2585 deprecated=True, 2586 deprecation_message="Use `query_properties` field instead.", 2587 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2588 title="Fetch Properties from Endpoint", 2589 ) 2590 query_properties: Optional[QueryProperties] = Field( 2591 None, 2592 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2593 title="Query Properties", 2594 ) 2595 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2596 None, 2597 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2598 examples=[ 2599 {"unit": "day"}, 2600 { 2601 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2602 }, 2603 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2604 {"sort_by[asc]": "updated_at"}, 2605 ], 2606 title="Query Parameters", 2607 ) 2608 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2609 None, 2610 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2611 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2612 title="Request Headers", 2613 ) 2614 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2615 None, 2616 deprecated=True, 2617 deprecation_message="Use `request_body` field instead.", 2618 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2619 examples=[ 2620 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2621 ], 2622 title="Request Body Payload (Non-JSON)", 2623 ) 2624 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2625 None, 2626 deprecated=True, 2627 deprecation_message="Use `request_body` field instead.", 2628 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2629 examples=[ 2630 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2631 {"key": "{{ config['value'] }}"}, 2632 {"sort": {"field": "updated_at", "order": "ascending"}}, 2633 ], 2634 title="Request Body JSON Payload", 2635 ) 2636 request_body: Optional[ 2637 Union[ 2638 RequestBodyPlainText, 2639 RequestBodyUrlEncodedForm, 2640 RequestBodyJsonObject, 2641 RequestBodyGraphQL, 2642 ] 2643 ] = Field( 2644 None, 2645 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2646 title="Request Body", 2647 ) 2648 error_handler: Optional[ 2649 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2650 ] = Field( 2651 None, 2652 description="Error handler component that defines how to handle errors.", 2653 title="Error Handler", 2654 ) 2655 use_cache: Optional[bool] = Field( 2656 False, 2657 description="Enables stream requests caching. This field is automatically set by the CDK.", 2658 title="Use Cache", 2659 ) 2660 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2663class DynamicSchemaLoader(BaseModel): 2664 type: Literal["DynamicSchemaLoader"] 2665 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2666 ..., 2667 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2668 title="Retriever", 2669 ) 2670 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2671 None, 2672 description="Responsible for filtering fields to be added to json schema.", 2673 title="Schema Filter", 2674 ) 2675 schema_transformations: Optional[ 2676 List[ 2677 Union[ 2678 AddFields, 2679 RemoveFields, 2680 KeysToLower, 2681 KeysToSnakeCase, 2682 FlattenFields, 2683 DpathFlattenFields, 2684 KeysReplace, 2685 CustomTransformation, 2686 ] 2687 ] 2688 ] = Field( 2689 None, 2690 description="A list of transformations to be applied to the schema.", 2691 title="Schema Transformations", 2692 ) 2693 schema_type_identifier: SchemaTypeIdentifier 2694 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2697class ParentStreamConfig(BaseModel): 2698 type: Literal["ParentStreamConfig"] 2699 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2700 ..., description="Reference to the parent stream.", title="Parent Stream" 2701 ) 2702 parent_key: str = Field( 2703 ..., 2704 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2705 examples=["id", "{{ config['parent_record_id'] }}"], 2706 title="Parent Key", 2707 ) 2708 partition_field: str = Field( 2709 ..., 2710 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2711 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2712 title="Current Parent Key Value Identifier", 2713 ) 2714 request_option: Optional[RequestOption] = Field( 2715 None, 2716 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2717 title="Request Option", 2718 ) 2719 incremental_dependency: Optional[bool] = Field( 2720 False, 2721 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2722 title="Incremental Dependency", 2723 ) 2724 lazy_read_pointer: Optional[List[str]] = Field( 2725 [], 2726 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2727 title="Lazy Read Pointer", 2728 ) 2729 extra_fields: Optional[List[List[str]]] = Field( 2730 None, 2731 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2732 title="Extra Fields", 2733 ) 2734 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2737class PropertiesFromEndpoint(BaseModel): 2738 type: Literal["PropertiesFromEndpoint"] 2739 property_field_path: List[str] = Field( 2740 ..., 2741 description="Describes the path to the field that should be extracted", 2742 examples=[["name"]], 2743 ) 2744 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2745 ..., 2746 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2747 ) 2748 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2751class QueryProperties(BaseModel): 2752 type: Literal["QueryProperties"] 2753 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2754 ..., 2755 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2756 title="Property List", 2757 ) 2758 always_include_properties: Optional[List[str]] = Field( 2759 None, 2760 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2761 title="Always Include Properties", 2762 ) 2763 property_chunking: Optional[PropertyChunking] = Field( 2764 None, 2765 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2766 title="Property Chunking", 2767 ) 2768 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2771class StateDelegatingStream(BaseModel): 2772 type: Literal["StateDelegatingStream"] 2773 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2774 full_refresh_stream: DeclarativeStream = Field( 2775 ..., 2776 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2777 title="Full Refresh Stream", 2778 ) 2779 incremental_stream: DeclarativeStream = Field( 2780 ..., 2781 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2782 title="Incremental Stream", 2783 ) 2784 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2787class SimpleRetriever(BaseModel): 2788 type: Literal["SimpleRetriever"] 2789 requester: Union[HttpRequester, CustomRequester] = Field( 2790 ..., 2791 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2792 ) 2793 decoder: Optional[ 2794 Union[ 2795 JsonDecoder, 2796 XmlDecoder, 2797 CsvDecoder, 2798 JsonlDecoder, 2799 GzipDecoder, 2800 IterableDecoder, 2801 ZipfileDecoder, 2802 CustomDecoder, 2803 ] 2804 ] = Field( 2805 None, 2806 description="Component decoding the response so records can be extracted.", 2807 title="HTTP Response Format", 2808 ) 2809 record_selector: RecordSelector = Field( 2810 ..., 2811 description="Component that describes how to extract records from a HTTP response.", 2812 ) 2813 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2814 None, 2815 description="Paginator component that describes how to navigate through the API's pages.", 2816 ) 2817 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2818 False, 2819 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2820 ) 2821 partition_router: Optional[ 2822 Union[ 2823 SubstreamPartitionRouter, 2824 ListPartitionRouter, 2825 GroupingPartitionRouter, 2826 CustomPartitionRouter, 2827 List[ 2828 Union[ 2829 SubstreamPartitionRouter, 2830 ListPartitionRouter, 2831 GroupingPartitionRouter, 2832 CustomPartitionRouter, 2833 ] 2834 ], 2835 ] 2836 ] = Field( 2837 None, 2838 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2839 title="Partition Router", 2840 ) 2841 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2844class AsyncRetriever(BaseModel): 2845 type: Literal["AsyncRetriever"] 2846 record_selector: RecordSelector = Field( 2847 ..., 2848 description="Component that describes how to extract records from a HTTP response.", 2849 ) 2850 status_mapping: AsyncJobStatusMap = Field( 2851 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2852 ) 2853 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2854 ..., description="Responsible for fetching the actual status of the async job." 2855 ) 2856 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2857 None, 2858 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2859 ) 2860 download_extractor: Optional[ 2861 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2862 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2863 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2864 ..., 2865 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2866 ) 2867 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2868 ..., 2869 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2870 ) 2871 polling_job_timeout: Optional[Union[int, str]] = Field( 2872 None, 2873 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2874 ) 2875 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2876 None, 2877 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2878 ) 2879 download_requester: Union[HttpRequester, CustomRequester] = Field( 2880 ..., 2881 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2882 ) 2883 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2884 None, 2885 description="Paginator component that describes how to navigate through the API's pages during download.", 2886 ) 2887 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2888 None, 2889 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2890 ) 2891 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2892 None, 2893 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2894 ) 2895 partition_router: Optional[ 2896 Union[ 2897 ListPartitionRouter, 2898 SubstreamPartitionRouter, 2899 GroupingPartitionRouter, 2900 CustomPartitionRouter, 2901 List[ 2902 Union[ 2903 ListPartitionRouter, 2904 SubstreamPartitionRouter, 2905 GroupingPartitionRouter, 2906 CustomPartitionRouter, 2907 ] 2908 ], 2909 ] 2910 ] = Field( 2911 [], 2912 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2913 title="Partition Router", 2914 ) 2915 decoder: Optional[ 2916 Union[ 2917 CsvDecoder, 2918 GzipDecoder, 2919 JsonDecoder, 2920 JsonlDecoder, 2921 IterableDecoder, 2922 XmlDecoder, 2923 ZipfileDecoder, 2924 CustomDecoder, 2925 ] 2926 ] = Field( 2927 None, 2928 description="Component decoding the response so records can be extracted.", 2929 title="HTTP Response Format", 2930 ) 2931 download_decoder: Optional[ 2932 Union[ 2933 CsvDecoder, 2934 GzipDecoder, 2935 JsonDecoder, 2936 JsonlDecoder, 2937 IterableDecoder, 2938 XmlDecoder, 2939 ZipfileDecoder, 2940 CustomDecoder, 2941 ] 2942 ] = Field( 2943 None, 2944 description="Component decoding the download response so records can be extracted.", 2945 title="Download HTTP Response Format", 2946 ) 2947 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2950class SubstreamPartitionRouter(BaseModel): 2951 type: Literal["SubstreamPartitionRouter"] 2952 parent_stream_configs: List[ParentStreamConfig] = Field( 2953 ..., 2954 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2955 title="Parent Stream Configs", 2956 ) 2957 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2960class GroupingPartitionRouter(BaseModel): 2961 type: Literal["GroupingPartitionRouter"] 2962 group_size: int = Field( 2963 ..., 2964 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2965 examples=[10, 50], 2966 title="Group Size", 2967 ) 2968 underlying_partition_router: Union[ 2969 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2970 ] = Field( 2971 ..., 2972 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2973 title="Underlying Partition Router", 2974 ) 2975 deduplicate: Optional[bool] = Field( 2976 True, 2977 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2978 title="Deduplicate Partitions", 2979 ) 2980 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2983class HttpComponentsResolver(BaseModel): 2984 type: Literal["HttpComponentsResolver"] 2985 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2986 ..., 2987 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2988 title="Retriever", 2989 ) 2990 components_mapping: List[ComponentMappingDefinition] 2991 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2994class DynamicDeclarativeStream(BaseModel): 2995 type: Literal["DynamicDeclarativeStream"] 2996 name: Optional[str] = Field( 2997 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2998 ) 2999 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3000 ..., description="Reference to the stream template.", title="Stream Template" 3001 ) 3002 components_resolver: Union[ 3003 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3004 ] = Field( 3005 ..., 3006 description="Component resolve and populates stream templates with components values.", 3007 title="Components Resolver", 3008 ) 3009 use_parent_parameters: Optional[bool] = Field( 3010 True, 3011 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3012 title="Use Parent Parameters", 3013 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]