airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[int] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomIncrementalSync(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomIncrementalSync"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomIncrementalSync"], 186 title="Class Name", 187 ) 188 cursor_field: str = Field( 189 ..., 190 description="The location of the value on a record that will be used as a bookmark during sync.", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomPaginationStrategy(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomPaginationStrategy"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomPaginationStrategy"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordExtractor(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordExtractor"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomRecordExtractor"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRecordFilter(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRecordFilter"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomCustomRecordFilter"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRequester(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRequester"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRecordExtractor"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomRetriever(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomRetriever"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomRetriever"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomPartitionRouter(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomPartitionRouter"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomPartitionRouter"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaLoader(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaLoader"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 287 examples=["source_railz.components.MyCustomSchemaLoader"], 288 title="Class Name", 289 ) 290 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 291 292 293class CustomSchemaNormalization(BaseModel): 294 class Config: 295 extra = Extra.allow 296 297 type: Literal["CustomSchemaNormalization"] 298 class_name: str = Field( 299 ..., 300 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 301 examples=[ 302 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 303 ], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomStateMigration(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomStateMigration"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomStateMigration"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class CustomTransformation(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Literal["CustomTransformation"] 328 class_name: str = Field( 329 ..., 330 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 331 examples=["source_railz.components.MyCustomTransformation"], 332 title="Class Name", 333 ) 334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 335 336 337class LegacyToPerPartitionStateMigration(BaseModel): 338 class Config: 339 extra = Extra.allow 340 341 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 342 343 344class Clamping(BaseModel): 345 target: str = Field( 346 ..., 347 description="The period of time that datetime windows will be clamped by", 348 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 349 title="Target", 350 ) 351 target_details: Optional[Dict[str, Any]] = None 352 353 354class Algorithm(Enum): 355 HS256 = "HS256" 356 HS384 = "HS384" 357 HS512 = "HS512" 358 ES256 = "ES256" 359 ES256K = "ES256K" 360 ES384 = "ES384" 361 ES512 = "ES512" 362 RS256 = "RS256" 363 RS384 = "RS384" 364 RS512 = "RS512" 365 PS256 = "PS256" 366 PS384 = "PS384" 367 PS512 = "PS512" 368 EdDSA = "EdDSA" 369 370 371class JwtHeaders(BaseModel): 372 class Config: 373 extra = Extra.forbid 374 375 kid: Optional[str] = Field( 376 None, 377 description="Private key ID for user account.", 378 examples=["{{ config['kid'] }}"], 379 title="Key Identifier", 380 ) 381 typ: Optional[str] = Field( 382 "JWT", 383 description="The media type of the complete JWT.", 384 examples=["JWT"], 385 title="Type", 386 ) 387 cty: Optional[str] = Field( 388 None, 389 description="Content type of JWT header.", 390 examples=["JWT"], 391 title="Content Type", 392 ) 393 394 395class JwtPayload(BaseModel): 396 class Config: 397 extra = Extra.forbid 398 399 iss: Optional[str] = Field( 400 None, 401 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 402 examples=["{{ config['iss'] }}"], 403 title="Issuer", 404 ) 405 sub: Optional[str] = Field( 406 None, 407 description="The subject of the JWT. Commonly defined by the API.", 408 title="Subject", 409 ) 410 aud: Optional[str] = Field( 411 None, 412 description="The recipient that the JWT is intended for. Commonly defined by the API.", 413 examples=["appstoreconnect-v1"], 414 title="Audience", 415 ) 416 417 418class JwtAuthenticator(BaseModel): 419 type: Literal["JwtAuthenticator"] 420 secret_key: str = Field( 421 ..., 422 description="Secret used to sign the JSON web token.", 423 examples=["{{ config['secret_key'] }}"], 424 title="Secret Key", 425 ) 426 base64_encode_secret_key: Optional[bool] = Field( 427 False, 428 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 429 title="Base64-encode Secret Key", 430 ) 431 algorithm: Algorithm = Field( 432 ..., 433 description="Algorithm used to sign the JSON web token.", 434 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 435 title="Algorithm", 436 ) 437 token_duration: Optional[int] = Field( 438 1200, 439 description="The amount of time in seconds a JWT token can be valid after being issued.", 440 examples=[1200, 3600], 441 title="Token Duration", 442 ) 443 header_prefix: Optional[str] = Field( 444 None, 445 description="The prefix to be used within the Authentication header.", 446 examples=["Bearer", "Basic"], 447 title="Header Prefix", 448 ) 449 jwt_headers: Optional[JwtHeaders] = Field( 450 None, 451 description="JWT headers used when signing JSON web token.", 452 title="JWT Headers", 453 ) 454 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 455 None, 456 description="Additional headers to be included with the JWT headers object.", 457 title="Additional JWT Headers", 458 ) 459 jwt_payload: Optional[JwtPayload] = Field( 460 None, 461 description="JWT Payload used when signing JSON web token.", 462 title="JWT Payload", 463 ) 464 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 465 None, 466 description="Additional properties to be added to the JWT payload.", 467 title="Additional JWT Payload Properties", 468 ) 469 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 470 471 472class RefreshTokenUpdater(BaseModel): 473 refresh_token_name: Optional[str] = Field( 474 "refresh_token", 475 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 476 examples=["refresh_token"], 477 title="Refresh Token Property Name", 478 ) 479 access_token_config_path: Optional[List[str]] = Field( 480 ["credentials", "access_token"], 481 description="Config path to the access token. Make sure the field actually exists in the config.", 482 examples=[["credentials", "access_token"], ["access_token"]], 483 title="Config Path To Access Token", 484 ) 485 refresh_token_config_path: Optional[List[str]] = Field( 486 ["credentials", "refresh_token"], 487 description="Config path to the access token. Make sure the field actually exists in the config.", 488 examples=[["credentials", "refresh_token"], ["refresh_token"]], 489 title="Config Path To Refresh Token", 490 ) 491 token_expiry_date_config_path: Optional[List[str]] = Field( 492 ["credentials", "token_expiry_date"], 493 description="Config path to the expiry date. Make sure actually exists in the config.", 494 examples=[["credentials", "token_expiry_date"]], 495 title="Config Path To Expiry Date", 496 ) 497 refresh_token_error_status_codes: Optional[List[int]] = Field( 498 [], 499 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 500 examples=[[400, 500]], 501 title="Refresh Token Error Status Codes", 502 ) 503 refresh_token_error_key: Optional[str] = Field( 504 "", 505 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 506 examples=["error"], 507 title="Refresh Token Error Key", 508 ) 509 refresh_token_error_values: Optional[List[str]] = Field( 510 [], 511 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 512 examples=[["invalid_grant", "invalid_permissions"]], 513 title="Refresh Token Error Values", 514 ) 515 516 517class OAuthAuthenticator(BaseModel): 518 type: Literal["OAuthAuthenticator"] 519 client_id_name: Optional[str] = Field( 520 "client_id", 521 description="The name of the property to use to refresh the `access_token`.", 522 examples=["custom_app_id"], 523 title="Client ID Property Name", 524 ) 525 client_id: Optional[str] = Field( 526 None, 527 description="The OAuth client ID. Fill it in the user inputs.", 528 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 529 title="Client ID", 530 ) 531 client_secret_name: Optional[str] = Field( 532 "client_secret", 533 description="The name of the property to use to refresh the `access_token`.", 534 examples=["custom_app_secret"], 535 title="Client Secret Property Name", 536 ) 537 client_secret: Optional[str] = Field( 538 None, 539 description="The OAuth client secret. Fill it in the user inputs.", 540 examples=[ 541 "{{ config['client_secret }}", 542 "{{ config['credentials']['client_secret }}", 543 ], 544 title="Client Secret", 545 ) 546 refresh_token_name: Optional[str] = Field( 547 "refresh_token", 548 description="The name of the property to use to refresh the `access_token`.", 549 examples=["custom_app_refresh_value"], 550 title="Refresh Token Property Name", 551 ) 552 refresh_token: Optional[str] = Field( 553 None, 554 description="Credential artifact used to get a new access token.", 555 examples=[ 556 "{{ config['refresh_token'] }}", 557 "{{ config['credentials]['refresh_token'] }}", 558 ], 559 title="Refresh Token", 560 ) 561 token_refresh_endpoint: Optional[str] = Field( 562 None, 563 description="The full URL to call to obtain a new access token.", 564 examples=["https://connect.squareup.com/oauth2/token"], 565 title="Token Refresh Endpoint", 566 ) 567 access_token_name: Optional[str] = Field( 568 "access_token", 569 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 570 examples=["access_token"], 571 title="Access Token Property Name", 572 ) 573 access_token_value: Optional[str] = Field( 574 None, 575 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 576 examples=["secret_access_token_value"], 577 title="Access Token Value", 578 ) 579 expires_in_name: Optional[str] = Field( 580 "expires_in", 581 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 582 examples=["expires_in"], 583 title="Token Expiry Property Name", 584 ) 585 grant_type_name: Optional[str] = Field( 586 "grant_type", 587 description="The name of the property to use to refresh the `access_token`.", 588 examples=["custom_grant_type"], 589 title="Grant Type Property Name", 590 ) 591 grant_type: Optional[str] = Field( 592 "refresh_token", 593 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 594 examples=["refresh_token", "client_credentials"], 595 title="Grant Type", 596 ) 597 refresh_request_body: Optional[Dict[str, Any]] = Field( 598 None, 599 description="Body of the request sent to get a new access token.", 600 examples=[ 601 { 602 "applicationId": "{{ config['application_id'] }}", 603 "applicationSecret": "{{ config['application_secret'] }}", 604 "token": "{{ config['token'] }}", 605 } 606 ], 607 title="Refresh Request Body", 608 ) 609 refresh_request_headers: Optional[Dict[str, Any]] = Field( 610 None, 611 description="Headers of the request sent to get a new access token.", 612 examples=[ 613 { 614 "Authorization": "<AUTH_TOKEN>", 615 "Content-Type": "application/x-www-form-urlencoded", 616 } 617 ], 618 title="Refresh Request Headers", 619 ) 620 scopes: Optional[List[str]] = Field( 621 None, 622 description="List of scopes that should be granted to the access token.", 623 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 624 title="Scopes", 625 ) 626 token_expiry_date: Optional[str] = Field( 627 None, 628 description="The access token expiry date.", 629 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 630 title="Token Expiry Date", 631 ) 632 token_expiry_date_format: Optional[str] = Field( 633 None, 634 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 635 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 636 title="Token Expiry Date Format", 637 ) 638 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 639 None, 640 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 641 title="Token Updater", 642 ) 643 profile_assertion: Optional[JwtAuthenticator] = Field( 644 None, 645 description="The authenticator being used to authenticate the client authenticator.", 646 title="Profile Assertion", 647 ) 648 use_profile_assertion: Optional[bool] = Field( 649 False, 650 description="Enable using profile assertion as a flow for OAuth authorization.", 651 title="Use Profile Assertion", 652 ) 653 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 654 655 656class Rate(BaseModel): 657 class Config: 658 extra = Extra.allow 659 660 limit: Union[int, str] = Field( 661 ..., 662 description="The maximum number of calls allowed within the interval.", 663 title="Limit", 664 ) 665 interval: str = Field( 666 ..., 667 description="The time interval for the rate limit.", 668 examples=["PT1H", "P1D"], 669 title="Interval", 670 ) 671 672 673class HttpRequestRegexMatcher(BaseModel): 674 class Config: 675 extra = Extra.allow 676 677 method: Optional[str] = Field( 678 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 679 ) 680 url_base: Optional[str] = Field( 681 None, 682 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 683 title="URL Base", 684 ) 685 url_path_pattern: Optional[str] = Field( 686 None, 687 description="A regular expression pattern to match the URL path.", 688 title="URL Path Pattern", 689 ) 690 params: Optional[Dict[str, Any]] = Field( 691 None, description="The query parameters to match.", title="Parameters" 692 ) 693 headers: Optional[Dict[str, Any]] = Field( 694 None, description="The headers to match.", title="Headers" 695 ) 696 697 698class DpathExtractor(BaseModel): 699 type: Literal["DpathExtractor"] 700 field_path: List[str] = Field( 701 ..., 702 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 703 examples=[ 704 ["data"], 705 ["data", "records"], 706 ["data", "{{ parameters.name }}"], 707 ["data", "*", "record"], 708 ], 709 title="Field Path", 710 ) 711 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 712 713 714class ResponseToFileExtractor(BaseModel): 715 type: Literal["ResponseToFileExtractor"] 716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 717 718 719class ExponentialBackoffStrategy(BaseModel): 720 type: Literal["ExponentialBackoffStrategy"] 721 factor: Optional[Union[float, str]] = Field( 722 5, 723 description="Multiplicative constant applied on each retry.", 724 examples=[5, 5.5, "10"], 725 title="Factor", 726 ) 727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 728 729 730class GroupByKeyMergeStrategy(BaseModel): 731 type: Literal["GroupByKeyMergeStrategy"] 732 key: Union[str, List[str]] = Field( 733 ..., 734 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 735 examples=["id", ["parent_id", "end_date"]], 736 title="Key", 737 ) 738 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 739 740 741class SessionTokenRequestBearerAuthenticator(BaseModel): 742 type: Literal["Bearer"] 743 744 745class HttpMethod(Enum): 746 GET = "GET" 747 POST = "POST" 748 749 750class Action(Enum): 751 SUCCESS = "SUCCESS" 752 FAIL = "FAIL" 753 RETRY = "RETRY" 754 IGNORE = "IGNORE" 755 RATE_LIMITED = "RATE_LIMITED" 756 757 758class FailureType(Enum): 759 system_error = "system_error" 760 config_error = "config_error" 761 transient_error = "transient_error" 762 763 764class HttpResponseFilter(BaseModel): 765 type: Literal["HttpResponseFilter"] 766 action: Optional[Action] = Field( 767 None, 768 description="Action to execute if a response matches the filter.", 769 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 770 title="Action", 771 ) 772 failure_type: Optional[FailureType] = Field( 773 None, 774 description="Failure type of traced exception if a response matches the filter.", 775 examples=["system_error", "config_error", "transient_error"], 776 title="Failure Type", 777 ) 778 error_message: Optional[str] = Field( 779 None, 780 description="Error Message to display if the response matches the filter.", 781 title="Error Message", 782 ) 783 error_message_contains: Optional[str] = Field( 784 None, 785 description="Match the response if its error message contains the substring.", 786 example=["This API operation is not enabled for this site"], 787 title="Error Message Substring", 788 ) 789 http_codes: Optional[List[int]] = Field( 790 None, 791 description="Match the response if its HTTP code is included in this list.", 792 examples=[[420, 429], [500]], 793 title="HTTP Codes", 794 unique_items=True, 795 ) 796 predicate: Optional[str] = Field( 797 None, 798 description="Match the response if the predicate evaluates to true.", 799 examples=[ 800 "{{ 'Too much requests' in response }}", 801 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 802 ], 803 title="Predicate", 804 ) 805 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 806 807 808class ComplexFieldType(BaseModel): 809 field_type: str 810 items: Optional[Union[str, ComplexFieldType]] = None 811 812 813class TypesMap(BaseModel): 814 target_type: Union[str, List[str], ComplexFieldType] 815 current_type: Union[str, List[str]] 816 condition: Optional[str] = None 817 818 819class SchemaTypeIdentifier(BaseModel): 820 type: Optional[Literal["SchemaTypeIdentifier"]] = None 821 schema_pointer: Optional[List[str]] = Field( 822 [], 823 description="List of nested fields defining the schema field path to extract. Defaults to [].", 824 title="Schema Path", 825 ) 826 key_pointer: List[str] = Field( 827 ..., 828 description="List of potentially nested fields describing the full path of the field key to extract.", 829 title="Key Path", 830 ) 831 type_pointer: Optional[List[str]] = Field( 832 None, 833 description="List of potentially nested fields describing the full path of the field type to extract.", 834 title="Type Path", 835 ) 836 types_mapping: Optional[List[TypesMap]] = None 837 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 838 839 840class InlineSchemaLoader(BaseModel): 841 type: Literal["InlineSchemaLoader"] 842 schema_: Optional[Dict[str, Any]] = Field( 843 None, 844 alias="schema", 845 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 846 title="Schema", 847 ) 848 849 850class JsonFileSchemaLoader(BaseModel): 851 type: Literal["JsonFileSchemaLoader"] 852 file_path: Optional[str] = Field( 853 None, 854 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 855 example=["./schemas/users.json"], 856 title="File Path", 857 ) 858 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 859 860 861class JsonDecoder(BaseModel): 862 type: Literal["JsonDecoder"] 863 864 865class JsonlDecoder(BaseModel): 866 type: Literal["JsonlDecoder"] 867 868 869class KeysToLower(BaseModel): 870 type: Literal["KeysToLower"] 871 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 872 873 874class KeysToSnakeCase(BaseModel): 875 type: Literal["KeysToSnakeCase"] 876 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 877 878 879class FlattenFields(BaseModel): 880 type: Literal["FlattenFields"] 881 flatten_lists: Optional[bool] = Field( 882 True, 883 description="Whether to flatten lists or leave it as is. Default is True.", 884 title="Flatten Lists", 885 ) 886 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 887 888 889class KeyTransformation(BaseModel): 890 type: Literal["KeyTransformation"] 891 prefix: Optional[str] = Field( 892 None, 893 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 894 examples=["flattened_"], 895 title="Key Prefix", 896 ) 897 suffix: Optional[str] = Field( 898 None, 899 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 900 examples=["_flattened"], 901 title="Key Suffix", 902 ) 903 904 905class DpathFlattenFields(BaseModel): 906 type: Literal["DpathFlattenFields"] 907 field_path: List[str] = Field( 908 ..., 909 description="A path to field that needs to be flattened.", 910 examples=[["data"], ["data", "*", "field"]], 911 title="Field Path", 912 ) 913 delete_origin_value: Optional[bool] = Field( 914 None, 915 description="Whether to delete the origin value or keep it. Default is False.", 916 title="Delete Origin Value", 917 ) 918 replace_record: Optional[bool] = Field( 919 None, 920 description="Whether to replace the origin record or not. Default is False.", 921 title="Replace Origin Record", 922 ) 923 key_transformation: Optional[KeyTransformation] = Field( 924 None, 925 description="Transformation for object keys. If not provided, original key will be used.", 926 title="Key transformation", 927 ) 928 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 929 930 931class KeysReplace(BaseModel): 932 type: Literal["KeysReplace"] 933 old: str = Field( 934 ..., 935 description="Old value to replace.", 936 examples=[ 937 " ", 938 "{{ record.id }}", 939 "{{ config['id'] }}", 940 "{{ stream_slice['id'] }}", 941 ], 942 title="Old value", 943 ) 944 new: str = Field( 945 ..., 946 description="New value to set.", 947 examples=[ 948 "_", 949 "{{ record.id }}", 950 "{{ config['id'] }}", 951 "{{ stream_slice['id'] }}", 952 ], 953 title="New value", 954 ) 955 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 956 957 958class IterableDecoder(BaseModel): 959 type: Literal["IterableDecoder"] 960 961 962class XmlDecoder(BaseModel): 963 type: Literal["XmlDecoder"] 964 965 966class CustomDecoder(BaseModel): 967 class Config: 968 extra = Extra.allow 969 970 type: Literal["CustomDecoder"] 971 class_name: str = Field( 972 ..., 973 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 974 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 975 title="Class Name", 976 ) 977 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 978 979 980class MinMaxDatetime(BaseModel): 981 type: Literal["MinMaxDatetime"] 982 datetime: str = Field( 983 ..., 984 description="Datetime value.", 985 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 986 title="Datetime", 987 ) 988 datetime_format: Optional[str] = Field( 989 "", 990 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 991 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 992 title="Datetime Format", 993 ) 994 max_datetime: Optional[str] = Field( 995 None, 996 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 997 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 998 title="Max Datetime", 999 ) 1000 min_datetime: Optional[str] = Field( 1001 None, 1002 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1003 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1004 title="Min Datetime", 1005 ) 1006 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1007 1008 1009class NoAuth(BaseModel): 1010 type: Literal["NoAuth"] 1011 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1012 1013 1014class NoPagination(BaseModel): 1015 type: Literal["NoPagination"] 1016 1017 1018class State(BaseModel): 1019 class Config: 1020 extra = Extra.allow 1021 1022 min: int 1023 max: int 1024 1025 1026class OauthConnectorInputSpecification(BaseModel): 1027 class Config: 1028 extra = Extra.allow 1029 1030 consent_url: str = Field( 1031 ..., 1032 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1033 examples=[ 1034 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1035 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1036 ], 1037 title="Consent URL", 1038 ) 1039 scope: Optional[str] = Field( 1040 None, 1041 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1042 examples=["user:read user:read_orders workspaces:read"], 1043 title="Scopes", 1044 ) 1045 access_token_url: str = Field( 1046 ..., 1047 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1048 examples=[ 1049 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1050 ], 1051 title="Access Token URL", 1052 ) 1053 access_token_headers: Optional[Dict[str, Any]] = Field( 1054 None, 1055 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1056 examples=[ 1057 { 1058 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1059 } 1060 ], 1061 title="Access Token Headers", 1062 ) 1063 access_token_params: Optional[Dict[str, Any]] = Field( 1064 None, 1065 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1066 examples=[ 1067 { 1068 "{{ auth_code_key }}": "{{ auth_code_value }}", 1069 "{{ client_id_key }}": "{{ client_id_value }}", 1070 "{{ client_secret_key }}": "{{ client_secret_value }}", 1071 } 1072 ], 1073 title="Access Token Query Params (Json Encoded)", 1074 ) 1075 extract_output: Optional[List[str]] = Field( 1076 None, 1077 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1078 examples=[["access_token", "refresh_token", "other_field"]], 1079 title="Extract Output", 1080 ) 1081 state: Optional[State] = Field( 1082 None, 1083 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1084 examples=[{"min": 7, "max": 128}], 1085 title="Configurable State Query Param", 1086 ) 1087 client_id_key: Optional[str] = Field( 1088 None, 1089 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1090 examples=["my_custom_client_id_key_name"], 1091 title="Client ID Key Override", 1092 ) 1093 client_secret_key: Optional[str] = Field( 1094 None, 1095 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1096 examples=["my_custom_client_secret_key_name"], 1097 title="Client Secret Key Override", 1098 ) 1099 scope_key: Optional[str] = Field( 1100 None, 1101 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1102 examples=["my_custom_scope_key_key_name"], 1103 title="Scopes Key Override", 1104 ) 1105 state_key: Optional[str] = Field( 1106 None, 1107 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1108 examples=["my_custom_state_key_key_name"], 1109 title="State Key Override", 1110 ) 1111 auth_code_key: Optional[str] = Field( 1112 None, 1113 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1114 examples=["my_custom_auth_code_key_name"], 1115 title="Auth Code Key Override", 1116 ) 1117 redirect_uri_key: Optional[str] = Field( 1118 None, 1119 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1120 examples=["my_custom_redirect_uri_key_name"], 1121 title="Redirect URI Key Override", 1122 ) 1123 1124 1125class OAuthConfigSpecification(BaseModel): 1126 class Config: 1127 extra = Extra.allow 1128 1129 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1130 None, 1131 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1132 examples=[ 1133 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1134 { 1135 "app_id": { 1136 "type": "string", 1137 "path_in_connector_config": ["info", "app_id"], 1138 } 1139 }, 1140 ], 1141 title="OAuth user input", 1142 ) 1143 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1144 None, 1145 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1146 title="DeclarativeOAuth Connector Specification", 1147 ) 1148 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1149 None, 1150 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1151 examples=[ 1152 { 1153 "refresh_token": { 1154 "type": "string,", 1155 "path_in_connector_config": ["credentials", "refresh_token"], 1156 } 1157 } 1158 ], 1159 title="OAuth output specification", 1160 ) 1161 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1162 None, 1163 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1164 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1165 title="OAuth input specification", 1166 ) 1167 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1168 None, 1169 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1170 examples=[ 1171 { 1172 "client_id": { 1173 "type": "string,", 1174 "path_in_connector_config": ["credentials", "client_id"], 1175 }, 1176 "client_secret": { 1177 "type": "string,", 1178 "path_in_connector_config": ["credentials", "client_secret"], 1179 }, 1180 } 1181 ], 1182 title="OAuth server output specification", 1183 ) 1184 1185 1186class OffsetIncrement(BaseModel): 1187 type: Literal["OffsetIncrement"] 1188 page_size: Optional[Union[int, str]] = Field( 1189 None, 1190 description="The number of records to include in each pages.", 1191 examples=[100, "{{ config['page_size'] }}"], 1192 title="Limit", 1193 ) 1194 inject_on_first_request: Optional[bool] = Field( 1195 False, 1196 description="Using the `offset` with value `0` during the first request", 1197 title="Inject Offset", 1198 ) 1199 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1200 1201 1202class PageIncrement(BaseModel): 1203 type: Literal["PageIncrement"] 1204 page_size: Optional[Union[int, str]] = Field( 1205 None, 1206 description="The number of records to include in each pages.", 1207 examples=[100, "100", "{{ config['page_size'] }}"], 1208 title="Page Size", 1209 ) 1210 start_from_page: Optional[int] = Field( 1211 0, 1212 description="Index of the first page to request.", 1213 examples=[0, 1], 1214 title="Start From Page", 1215 ) 1216 inject_on_first_request: Optional[bool] = Field( 1217 False, 1218 description="Using the `page number` with value defined by `start_from_page` during the first request", 1219 title="Inject Page Number", 1220 ) 1221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1222 1223 1224class PrimaryKey(BaseModel): 1225 __root__: Union[str, List[str], List[List[str]]] = Field( 1226 ..., 1227 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1228 examples=["id", ["code", "type"]], 1229 title="Primary Key", 1230 ) 1231 1232 1233class PropertyLimitType(Enum): 1234 characters = "characters" 1235 property_count = "property_count" 1236 1237 1238class PropertyChunking(BaseModel): 1239 type: Literal["PropertyChunking"] 1240 property_limit_type: PropertyLimitType = Field( 1241 ..., 1242 description="The type used to determine the maximum number of properties per chunk", 1243 title="Property Limit Type", 1244 ) 1245 property_limit: Optional[int] = Field( 1246 None, 1247 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1248 title="Property Limit", 1249 ) 1250 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1251 None, 1252 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1253 title="Record Merge Strategy", 1254 ) 1255 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1256 1257 1258class RecordFilter(BaseModel): 1259 type: Literal["RecordFilter"] 1260 condition: Optional[str] = Field( 1261 "", 1262 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1263 examples=[ 1264 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1265 "{{ record.status in ['active', 'expired'] }}", 1266 ], 1267 title="Condition", 1268 ) 1269 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1270 1271 1272class SchemaNormalization(Enum): 1273 Default = "Default" 1274 None_ = "None" 1275 1276 1277class RemoveFields(BaseModel): 1278 type: Literal["RemoveFields"] 1279 condition: Optional[str] = Field( 1280 "", 1281 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1282 examples=[ 1283 "{{ property|string == '' }}", 1284 "{{ property is integer }}", 1285 "{{ property|length > 5 }}", 1286 "{{ property == 'some_string_to_match' }}", 1287 ], 1288 ) 1289 field_pointers: List[List[str]] = Field( 1290 ..., 1291 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1292 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1293 title="Field Paths", 1294 ) 1295 1296 1297class RequestPath(BaseModel): 1298 type: Literal["RequestPath"] 1299 1300 1301class InjectInto(Enum): 1302 request_parameter = "request_parameter" 1303 header = "header" 1304 body_data = "body_data" 1305 body_json = "body_json" 1306 1307 1308class RequestOption(BaseModel): 1309 type: Literal["RequestOption"] 1310 field_name: Optional[str] = Field( 1311 None, 1312 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1313 examples=["segment_id"], 1314 title="Field Name", 1315 ) 1316 field_path: Optional[List[str]] = Field( 1317 None, 1318 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1319 examples=[["data", "viewer", "id"]], 1320 title="Field Path", 1321 ) 1322 inject_into: InjectInto = Field( 1323 ..., 1324 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1325 examples=["request_parameter", "header", "body_data", "body_json"], 1326 title="Inject Into", 1327 ) 1328 1329 1330class Schemas(BaseModel): 1331 pass 1332 1333 class Config: 1334 extra = Extra.allow 1335 1336 1337class LegacySessionTokenAuthenticator(BaseModel): 1338 type: Literal["LegacySessionTokenAuthenticator"] 1339 header: str = Field( 1340 ..., 1341 description="The name of the session token header that will be injected in the request", 1342 examples=["X-Session"], 1343 title="Session Request Header", 1344 ) 1345 login_url: str = Field( 1346 ..., 1347 description="Path of the login URL (do not include the base URL)", 1348 examples=["session"], 1349 title="Login Path", 1350 ) 1351 session_token: Optional[str] = Field( 1352 None, 1353 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1354 example=["{{ config['session_token'] }}"], 1355 title="Session Token", 1356 ) 1357 session_token_response_key: str = Field( 1358 ..., 1359 description="Name of the key of the session token to be extracted from the response", 1360 examples=["id"], 1361 title="Response Token Response Key", 1362 ) 1363 username: Optional[str] = Field( 1364 None, 1365 description="Username used to authenticate and obtain a session token", 1366 examples=[" {{ config['username'] }}"], 1367 title="Username", 1368 ) 1369 password: Optional[str] = Field( 1370 "", 1371 description="Password used to authenticate and obtain a session token", 1372 examples=["{{ config['password'] }}", ""], 1373 title="Password", 1374 ) 1375 validate_session_url: str = Field( 1376 ..., 1377 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1378 examples=["user/current"], 1379 title="Validate Session Path", 1380 ) 1381 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1382 1383 1384class CsvDecoder(BaseModel): 1385 type: Literal["CsvDecoder"] 1386 encoding: Optional[str] = "utf-8" 1387 delimiter: Optional[str] = "," 1388 1389 1390class AsyncJobStatusMap(BaseModel): 1391 type: Optional[Literal["AsyncJobStatusMap"]] = None 1392 running: List[str] 1393 completed: List[str] 1394 failed: List[str] 1395 timeout: List[str] 1396 1397 1398class ValueType(Enum): 1399 string = "string" 1400 number = "number" 1401 integer = "integer" 1402 boolean = "boolean" 1403 1404 1405class WaitTimeFromHeader(BaseModel): 1406 type: Literal["WaitTimeFromHeader"] 1407 header: str = Field( 1408 ..., 1409 description="The name of the response header defining how long to wait before retrying.", 1410 examples=["Retry-After"], 1411 title="Response Header Name", 1412 ) 1413 regex: Optional[str] = Field( 1414 None, 1415 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1416 examples=["([-+]?\\d+)"], 1417 title="Extraction Regex", 1418 ) 1419 max_waiting_time_in_seconds: Optional[float] = Field( 1420 None, 1421 description="Given the value extracted from the header is greater than this value, stop the stream.", 1422 examples=[3600], 1423 title="Max Waiting Time in Seconds", 1424 ) 1425 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1426 1427 1428class WaitUntilTimeFromHeader(BaseModel): 1429 type: Literal["WaitUntilTimeFromHeader"] 1430 header: str = Field( 1431 ..., 1432 description="The name of the response header defining how long to wait before retrying.", 1433 examples=["wait_time"], 1434 title="Response Header", 1435 ) 1436 min_wait: Optional[Union[float, str]] = Field( 1437 None, 1438 description="Minimum time to wait before retrying.", 1439 examples=[10, "60"], 1440 title="Minimum Wait Time", 1441 ) 1442 regex: Optional[str] = Field( 1443 None, 1444 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1445 examples=["([-+]?\\d+)"], 1446 title="Extraction Regex", 1447 ) 1448 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1449 1450 1451class ComponentMappingDefinition(BaseModel): 1452 type: Literal["ComponentMappingDefinition"] 1453 field_path: List[str] = Field( 1454 ..., 1455 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1456 examples=[ 1457 ["data"], 1458 ["data", "records"], 1459 ["data", 1, "name"], 1460 ["data", "{{ components_values.name }}"], 1461 ["data", "*", "record"], 1462 ["*", "**", "name"], 1463 ], 1464 title="Field Path", 1465 ) 1466 value: str = Field( 1467 ..., 1468 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1469 examples=[ 1470 "{{ components_values['updates'] }}", 1471 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1472 "{{ config['segment_id'] }}", 1473 "{{ stream_slice['parent_id'] }}", 1474 "{{ stream_slice['extra_fields']['name'] }}", 1475 ], 1476 title="Value", 1477 ) 1478 value_type: Optional[ValueType] = Field( 1479 None, 1480 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1481 title="Value Type", 1482 ) 1483 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1484 1485 1486class StreamConfig(BaseModel): 1487 type: Literal["StreamConfig"] 1488 configs_pointer: List[str] = Field( 1489 ..., 1490 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1491 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1492 title="Configs Pointer", 1493 ) 1494 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1495 1496 1497class ConfigComponentsResolver(BaseModel): 1498 type: Literal["ConfigComponentsResolver"] 1499 stream_config: StreamConfig 1500 components_mapping: List[ComponentMappingDefinition] 1501 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1502 1503 1504class RequestBodyPlainText(BaseModel): 1505 type: Literal["RequestBodyPlainText"] 1506 value: str 1507 1508 1509class RequestBodyUrlEncodedForm(BaseModel): 1510 type: Literal["RequestBodyUrlEncodedForm"] 1511 value: Dict[str, str] 1512 1513 1514class RequestBodyJsonObject(BaseModel): 1515 type: Literal["RequestBodyJsonObject"] 1516 value: Dict[str, Any] 1517 1518 1519class RequestBodyGraphQlQuery(BaseModel): 1520 class Config: 1521 extra = Extra.allow 1522 1523 query: Dict[str, Any] = Field(..., description="The GraphQL query to be executed") 1524 1525 1526class AddedFieldDefinition(BaseModel): 1527 type: Literal["AddedFieldDefinition"] 1528 path: List[str] = Field( 1529 ..., 1530 description="List of strings defining the path where to add the value on the record.", 1531 examples=[["segment_id"], ["metadata", "segment_id"]], 1532 title="Path", 1533 ) 1534 value: str = Field( 1535 ..., 1536 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1537 examples=[ 1538 "{{ record['updates'] }}", 1539 "{{ record['MetaData']['LastUpdatedTime'] }}", 1540 "{{ stream_partition['segment_id'] }}", 1541 ], 1542 title="Value", 1543 ) 1544 value_type: Optional[ValueType] = Field( 1545 None, 1546 description="Type of the value. If not specified, the type will be inferred from the value.", 1547 title="Value Type", 1548 ) 1549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1550 1551 1552class AddFields(BaseModel): 1553 type: Literal["AddFields"] 1554 fields: List[AddedFieldDefinition] = Field( 1555 ..., 1556 description="List of transformations (path and corresponding value) that will be added to the record.", 1557 title="Fields", 1558 ) 1559 condition: Optional[str] = Field( 1560 "", 1561 description="Fields will be added if expression is evaluated to True.", 1562 examples=[ 1563 "{{ property|string == '' }}", 1564 "{{ property is integer }}", 1565 "{{ property|length > 5 }}", 1566 "{{ property == 'some_string_to_match' }}", 1567 ], 1568 ) 1569 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1570 1571 1572class ApiKeyAuthenticator(BaseModel): 1573 type: Literal["ApiKeyAuthenticator"] 1574 api_token: Optional[str] = Field( 1575 None, 1576 description="The API key to inject in the request. Fill it in the user inputs.", 1577 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1578 title="API Key", 1579 ) 1580 header: Optional[str] = Field( 1581 None, 1582 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1583 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1584 title="Header Name", 1585 ) 1586 inject_into: Optional[RequestOption] = Field( 1587 None, 1588 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1589 examples=[ 1590 {"inject_into": "header", "field_name": "Authorization"}, 1591 {"inject_into": "request_parameter", "field_name": "authKey"}, 1592 ], 1593 title="Inject API Key Into Outgoing HTTP Request", 1594 ) 1595 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1596 1597 1598class AuthFlow(BaseModel): 1599 auth_flow_type: Optional[AuthFlowType] = Field( 1600 None, description="The type of auth to use", title="Auth flow type" 1601 ) 1602 predicate_key: Optional[List[str]] = Field( 1603 None, 1604 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1605 examples=[["credentials", "auth_type"]], 1606 title="Predicate key", 1607 ) 1608 predicate_value: Optional[str] = Field( 1609 None, 1610 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1611 examples=["Oauth"], 1612 title="Predicate value", 1613 ) 1614 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1615 1616 1617class CheckStream(BaseModel): 1618 type: Literal["CheckStream"] 1619 stream_names: Optional[List[str]] = Field( 1620 None, 1621 description="Names of the streams to try reading from when running a check operation.", 1622 examples=[["users"], ["users", "contacts"]], 1623 title="Stream Names", 1624 ) 1625 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1626 1627 1628class IncrementingCountCursor(BaseModel): 1629 type: Literal["IncrementingCountCursor"] 1630 cursor_field: str = Field( 1631 ..., 1632 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1633 examples=["created_at", "{{ config['record_cursor'] }}"], 1634 title="Cursor Field", 1635 ) 1636 start_value: Optional[Union[str, int]] = Field( 1637 None, 1638 description="The value that determines the earliest record that should be synced.", 1639 examples=[0, "{{ config['start_value'] }}"], 1640 title="Start Value", 1641 ) 1642 start_value_option: Optional[RequestOption] = Field( 1643 None, 1644 description="Optionally configures how the start value will be sent in requests to the source API.", 1645 title="Inject Start Value Into Outgoing HTTP Request", 1646 ) 1647 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1648 1649 1650class DatetimeBasedCursor(BaseModel): 1651 type: Literal["DatetimeBasedCursor"] 1652 clamping: Optional[Clamping] = Field( 1653 None, 1654 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1655 title="Date Range Clamping", 1656 ) 1657 cursor_field: str = Field( 1658 ..., 1659 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1660 examples=["created_at", "{{ config['record_cursor'] }}"], 1661 title="Cursor Field", 1662 ) 1663 datetime_format: str = Field( 1664 ..., 1665 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1666 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1667 title="Outgoing Datetime Format", 1668 ) 1669 start_datetime: Union[str, MinMaxDatetime] = Field( 1670 ..., 1671 description="The datetime that determines the earliest record that should be synced.", 1672 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1673 title="Start Datetime", 1674 ) 1675 cursor_datetime_formats: Optional[List[str]] = Field( 1676 None, 1677 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1678 title="Cursor Datetime Formats", 1679 ) 1680 cursor_granularity: Optional[str] = Field( 1681 None, 1682 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1683 examples=["PT1S"], 1684 title="Cursor Granularity", 1685 ) 1686 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1687 None, 1688 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1689 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1690 title="End Datetime", 1691 ) 1692 end_time_option: Optional[RequestOption] = Field( 1693 None, 1694 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1695 title="Inject End Time Into Outgoing HTTP Request", 1696 ) 1697 is_data_feed: Optional[bool] = Field( 1698 None, 1699 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1700 title="Whether the target API is formatted as a data feed", 1701 ) 1702 is_client_side_incremental: Optional[bool] = Field( 1703 None, 1704 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1705 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1706 ) 1707 is_compare_strictly: Optional[bool] = Field( 1708 False, 1709 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1710 title="Whether to skip requests if the start time equals the end time", 1711 ) 1712 global_substream_cursor: Optional[bool] = Field( 1713 False, 1714 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1715 title="Whether to store cursor as one value instead of per partition", 1716 ) 1717 lookback_window: Optional[str] = Field( 1718 None, 1719 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1720 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1721 title="Lookback Window", 1722 ) 1723 partition_field_end: Optional[str] = Field( 1724 None, 1725 description="Name of the partition start time field.", 1726 examples=["ending_time"], 1727 title="Partition Field End", 1728 ) 1729 partition_field_start: Optional[str] = Field( 1730 None, 1731 description="Name of the partition end time field.", 1732 examples=["starting_time"], 1733 title="Partition Field Start", 1734 ) 1735 start_time_option: Optional[RequestOption] = Field( 1736 None, 1737 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1738 title="Inject Start Time Into Outgoing HTTP Request", 1739 ) 1740 step: Optional[str] = Field( 1741 None, 1742 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1743 examples=["P1W", "{{ config['step_increment'] }}"], 1744 title="Step", 1745 ) 1746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1747 1748 1749class FixedWindowCallRatePolicy(BaseModel): 1750 class Config: 1751 extra = Extra.allow 1752 1753 type: Literal["FixedWindowCallRatePolicy"] 1754 period: str = Field( 1755 ..., description="The time interval for the rate limit window.", title="Period" 1756 ) 1757 call_limit: int = Field( 1758 ..., 1759 description="The maximum number of calls allowed within the period.", 1760 title="Call Limit", 1761 ) 1762 matchers: List[HttpRequestRegexMatcher] = Field( 1763 ..., 1764 description="List of matchers that define which requests this policy applies to.", 1765 title="Matchers", 1766 ) 1767 1768 1769class MovingWindowCallRatePolicy(BaseModel): 1770 class Config: 1771 extra = Extra.allow 1772 1773 type: Literal["MovingWindowCallRatePolicy"] 1774 rates: List[Rate] = Field( 1775 ..., 1776 description="List of rates that define the call limits for different time intervals.", 1777 title="Rates", 1778 ) 1779 matchers: List[HttpRequestRegexMatcher] = Field( 1780 ..., 1781 description="List of matchers that define which requests this policy applies to.", 1782 title="Matchers", 1783 ) 1784 1785 1786class UnlimitedCallRatePolicy(BaseModel): 1787 class Config: 1788 extra = Extra.allow 1789 1790 type: Literal["UnlimitedCallRatePolicy"] 1791 matchers: List[HttpRequestRegexMatcher] = Field( 1792 ..., 1793 description="List of matchers that define which requests this policy applies to.", 1794 title="Matchers", 1795 ) 1796 1797 1798class DefaultErrorHandler(BaseModel): 1799 type: Literal["DefaultErrorHandler"] 1800 backoff_strategies: Optional[ 1801 List[ 1802 Union[ 1803 ConstantBackoffStrategy, 1804 CustomBackoffStrategy, 1805 ExponentialBackoffStrategy, 1806 WaitTimeFromHeader, 1807 WaitUntilTimeFromHeader, 1808 ] 1809 ] 1810 ] = Field( 1811 None, 1812 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1813 title="Backoff Strategies", 1814 ) 1815 max_retries: Optional[int] = Field( 1816 5, 1817 description="The maximum number of time to retry a retryable request before giving up and failing.", 1818 examples=[5, 0, 10], 1819 title="Max Retry Count", 1820 ) 1821 response_filters: Optional[List[HttpResponseFilter]] = Field( 1822 None, 1823 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1824 title="Response Filters", 1825 ) 1826 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1827 1828 1829class DefaultPaginator(BaseModel): 1830 type: Literal["DefaultPaginator"] 1831 pagination_strategy: Union[ 1832 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1833 ] = Field( 1834 ..., 1835 description="Strategy defining how records are paginated.", 1836 title="Pagination Strategy", 1837 ) 1838 page_size_option: Optional[RequestOption] = Field( 1839 None, title="Inject Page Size Into Outgoing HTTP Request" 1840 ) 1841 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1842 None, title="Inject Page Token Into Outgoing HTTP Request" 1843 ) 1844 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1845 1846 1847class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1848 type: Literal["ApiKey"] 1849 inject_into: RequestOption = Field( 1850 ..., 1851 description="Configure how the API Key will be sent in requests to the source API.", 1852 examples=[ 1853 {"inject_into": "header", "field_name": "Authorization"}, 1854 {"inject_into": "request_parameter", "field_name": "authKey"}, 1855 ], 1856 title="Inject API Key Into Outgoing HTTP Request", 1857 ) 1858 1859 1860class ListPartitionRouter(BaseModel): 1861 type: Literal["ListPartitionRouter"] 1862 cursor_field: str = Field( 1863 ..., 1864 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1865 examples=["section", "{{ config['section_key'] }}"], 1866 title="Current Partition Value Identifier", 1867 ) 1868 values: Union[str, List[str]] = Field( 1869 ..., 1870 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1871 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1872 title="Partition Values", 1873 ) 1874 request_option: Optional[RequestOption] = Field( 1875 None, 1876 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1877 title="Inject Partition Value Into Outgoing HTTP Request", 1878 ) 1879 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1880 1881 1882class RecordSelector(BaseModel): 1883 type: Literal["RecordSelector"] 1884 extractor: Union[DpathExtractor, CustomRecordExtractor] 1885 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 1886 None, 1887 description="Responsible for filtering records to be emitted by the Source.", 1888 title="Record Filter", 1889 ) 1890 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1891 None, 1892 description="Responsible for normalization according to the schema.", 1893 title="Schema Normalization", 1894 ) 1895 transform_before_filtering: Optional[bool] = Field( 1896 None, 1897 description="If true, transformation will be applied before record filtering.", 1898 title="Transform Before Filtering", 1899 ) 1900 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1901 1902 1903class GzipDecoder(BaseModel): 1904 type: Literal["GzipDecoder"] 1905 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 1906 1907 1908class Spec(BaseModel): 1909 type: Literal["Spec"] 1910 connection_specification: Dict[str, Any] = Field( 1911 ..., 1912 description="A connection specification describing how a the connector can be configured.", 1913 title="Connection Specification", 1914 ) 1915 documentation_url: Optional[str] = Field( 1916 None, 1917 description="URL of the connector's documentation page.", 1918 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1919 title="Documentation URL", 1920 ) 1921 advanced_auth: Optional[AuthFlow] = Field( 1922 None, 1923 description="Advanced specification for configuring the authentication flow.", 1924 title="Advanced Auth", 1925 ) 1926 1927 1928class RequestBodyGraphQL(BaseModel): 1929 type: Literal["RequestBodyGraphQL"] 1930 value: RequestBodyGraphQlQuery 1931 1932 1933class CompositeErrorHandler(BaseModel): 1934 type: Literal["CompositeErrorHandler"] 1935 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1936 ..., 1937 description="List of error handlers to iterate on to determine how to handle a failed response.", 1938 title="Error Handlers", 1939 ) 1940 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1941 1942 1943class HTTPAPIBudget(BaseModel): 1944 class Config: 1945 extra = Extra.allow 1946 1947 type: Literal["HTTPAPIBudget"] 1948 policies: List[ 1949 Union[ 1950 FixedWindowCallRatePolicy, 1951 MovingWindowCallRatePolicy, 1952 UnlimitedCallRatePolicy, 1953 ] 1954 ] = Field( 1955 ..., 1956 description="List of call rate policies that define how many calls are allowed.", 1957 title="Policies", 1958 ) 1959 ratelimit_reset_header: Optional[str] = Field( 1960 "ratelimit-reset", 1961 description="The HTTP response header name that indicates when the rate limit resets.", 1962 title="Rate Limit Reset Header", 1963 ) 1964 ratelimit_remaining_header: Optional[str] = Field( 1965 "ratelimit-remaining", 1966 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1967 title="Rate Limit Remaining Header", 1968 ) 1969 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1970 [429], 1971 description="List of HTTP status codes that indicate a rate limit has been hit.", 1972 title="Status Codes for Rate Limit Hit", 1973 ) 1974 1975 1976class ZipfileDecoder(BaseModel): 1977 class Config: 1978 extra = Extra.allow 1979 1980 type: Literal["ZipfileDecoder"] 1981 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1982 ..., 1983 description="Parser to parse the decompressed data from the zipfile(s).", 1984 title="Parser", 1985 ) 1986 1987 1988class DeclarativeSource1(BaseModel): 1989 class Config: 1990 extra = Extra.forbid 1991 1992 type: Literal["DeclarativeSource"] 1993 check: Union[CheckStream, CheckDynamicStream] 1994 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1995 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1996 version: str = Field( 1997 ..., 1998 description="The version of the Airbyte CDK used to build and test the source.", 1999 ) 2000 schemas: Optional[Schemas] = None 2001 definitions: Optional[Dict[str, Any]] = None 2002 spec: Optional[Spec] = None 2003 concurrency_level: Optional[ConcurrencyLevel] = None 2004 api_budget: Optional[HTTPAPIBudget] = None 2005 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2006 None, 2007 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2008 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2009 title="Maximum Concurrent Asynchronous Jobs", 2010 ) 2011 metadata: Optional[Dict[str, Any]] = Field( 2012 None, 2013 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2014 ) 2015 description: Optional[str] = Field( 2016 None, 2017 description="A description of the connector. It will be presented on the Source documentation page.", 2018 ) 2019 2020 2021class DeclarativeSource2(BaseModel): 2022 class Config: 2023 extra = Extra.forbid 2024 2025 type: Literal["DeclarativeSource"] 2026 check: Union[CheckStream, CheckDynamicStream] 2027 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 2028 dynamic_streams: List[DynamicDeclarativeStream] 2029 version: str = Field( 2030 ..., 2031 description="The version of the Airbyte CDK used to build and test the source.", 2032 ) 2033 schemas: Optional[Schemas] = None 2034 definitions: Optional[Dict[str, Any]] = None 2035 spec: Optional[Spec] = None 2036 concurrency_level: Optional[ConcurrencyLevel] = None 2037 api_budget: Optional[HTTPAPIBudget] = None 2038 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2039 None, 2040 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2041 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2042 title="Maximum Concurrent Asynchronous Jobs", 2043 ) 2044 metadata: Optional[Dict[str, Any]] = Field( 2045 None, 2046 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2047 ) 2048 description: Optional[str] = Field( 2049 None, 2050 description="A description of the connector. It will be presented on the Source documentation page.", 2051 ) 2052 2053 2054class DeclarativeSource(BaseModel): 2055 class Config: 2056 extra = Extra.forbid 2057 2058 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2059 ..., 2060 description="An API source that extracts data according to its declarative components.", 2061 title="DeclarativeSource", 2062 ) 2063 2064 2065class SelectiveAuthenticator(BaseModel): 2066 class Config: 2067 extra = Extra.allow 2068 2069 type: Literal["SelectiveAuthenticator"] 2070 authenticator_selection_path: List[str] = Field( 2071 ..., 2072 description="Path of the field in config with selected authenticator name", 2073 examples=[["auth"], ["auth", "type"]], 2074 title="Authenticator Selection Path", 2075 ) 2076 authenticators: Dict[ 2077 str, 2078 Union[ 2079 ApiKeyAuthenticator, 2080 BasicHttpAuthenticator, 2081 BearerAuthenticator, 2082 CustomAuthenticator, 2083 OAuthAuthenticator, 2084 JwtAuthenticator, 2085 SessionTokenAuthenticator, 2086 NoAuth, 2087 LegacySessionTokenAuthenticator, 2088 ], 2089 ] = Field( 2090 ..., 2091 description="Authenticators to select from.", 2092 examples=[ 2093 { 2094 "authenticators": { 2095 "token": "#/definitions/ApiKeyAuthenticator", 2096 "oauth": "#/definitions/OAuthAuthenticator", 2097 "jwt": "#/definitions/JwtAuthenticator", 2098 } 2099 } 2100 ], 2101 title="Authenticators", 2102 ) 2103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2104 2105 2106class FileUploader(BaseModel): 2107 type: Literal["FileUploader"] 2108 requester: Union[HttpRequester, CustomRequester] = Field( 2109 ..., 2110 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2111 ) 2112 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2113 ..., 2114 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2115 ) 2116 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2117 None, 2118 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2119 ) 2120 filename_extractor: Optional[str] = Field( 2121 None, 2122 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2123 examples=[ 2124 "{{ record.id }}/{{ record.file_name }}/", 2125 "{{ record.id }}_{{ record.file_name }}/", 2126 ], 2127 ) 2128 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2129 2130 2131class DeclarativeStream(BaseModel): 2132 class Config: 2133 extra = Extra.allow 2134 2135 type: Literal["DeclarativeStream"] 2136 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2137 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2138 ..., 2139 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2140 title="Retriever", 2141 ) 2142 incremental_sync: Optional[ 2143 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2144 ] = Field( 2145 None, 2146 description="Component used to fetch data incrementally based on a time field in the data.", 2147 title="Incremental Sync", 2148 ) 2149 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2150 schema_loader: Optional[ 2151 Union[ 2152 InlineSchemaLoader, 2153 DynamicSchemaLoader, 2154 JsonFileSchemaLoader, 2155 CustomSchemaLoader, 2156 List[ 2157 Union[ 2158 InlineSchemaLoader, 2159 DynamicSchemaLoader, 2160 JsonFileSchemaLoader, 2161 CustomSchemaLoader, 2162 ] 2163 ], 2164 ] 2165 ] = Field( 2166 None, 2167 description="Component used to retrieve the schema for the current stream.", 2168 title="Schema Loader", 2169 ) 2170 transformations: Optional[ 2171 List[ 2172 Union[ 2173 AddFields, 2174 CustomTransformation, 2175 RemoveFields, 2176 KeysToLower, 2177 KeysToSnakeCase, 2178 FlattenFields, 2179 DpathFlattenFields, 2180 KeysReplace, 2181 ] 2182 ] 2183 ] = Field( 2184 None, 2185 description="A list of transformations to be applied to each output record.", 2186 title="Transformations", 2187 ) 2188 state_migrations: Optional[ 2189 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2190 ] = Field( 2191 [], 2192 description="Array of state migrations to be applied on the input state", 2193 title="State Migrations", 2194 ) 2195 file_uploader: Optional[FileUploader] = Field( 2196 None, 2197 description="(experimental) Describes how to fetch a file", 2198 title="File Uploader", 2199 ) 2200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2201 2202 2203class SessionTokenAuthenticator(BaseModel): 2204 type: Literal["SessionTokenAuthenticator"] 2205 login_requester: HttpRequester = Field( 2206 ..., 2207 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2208 examples=[ 2209 { 2210 "type": "HttpRequester", 2211 "url_base": "https://my_api.com", 2212 "path": "/login", 2213 "authenticator": { 2214 "type": "BasicHttpAuthenticator", 2215 "username": "{{ config.username }}", 2216 "password": "{{ config.password }}", 2217 }, 2218 } 2219 ], 2220 title="Login Requester", 2221 ) 2222 session_token_path: List[str] = Field( 2223 ..., 2224 description="The path in the response body returned from the login requester to the session token.", 2225 examples=[["access_token"], ["result", "token"]], 2226 title="Session Token Path", 2227 ) 2228 expiration_duration: Optional[str] = Field( 2229 None, 2230 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2231 examples=["PT1H", "P1D"], 2232 title="Expiration Duration", 2233 ) 2234 request_authentication: Union[ 2235 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2236 ] = Field( 2237 ..., 2238 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2239 title="Data Request Authentication", 2240 ) 2241 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2242 None, description="Component used to decode the response.", title="Decoder" 2243 ) 2244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2245 2246 2247class HttpRequester(BaseModelWithDeprecations): 2248 type: Literal["HttpRequester"] 2249 url_base: Optional[str] = Field( 2250 None, 2251 deprecated=True, 2252 deprecation_message="Use `url` field instead.", 2253 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2254 examples=[ 2255 "https://connect.squareup.com/v2", 2256 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2257 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2258 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2259 ], 2260 title="API Base URL", 2261 ) 2262 url: Optional[str] = Field( 2263 None, 2264 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2265 examples=[ 2266 "https://connect.squareup.com/v2", 2267 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2268 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2269 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2270 ], 2271 title="API Endpoint URL", 2272 ) 2273 path: Optional[str] = Field( 2274 None, 2275 deprecated=True, 2276 deprecation_message="Use `url` field instead.", 2277 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2278 examples=[ 2279 "/products", 2280 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2281 "/trades/{{ config['symbol_id'] }}/history", 2282 ], 2283 title="URL Path", 2284 ) 2285 http_method: Optional[HttpMethod] = Field( 2286 HttpMethod.GET, 2287 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2288 examples=["GET", "POST"], 2289 title="HTTP Method", 2290 ) 2291 authenticator: Optional[ 2292 Union[ 2293 ApiKeyAuthenticator, 2294 BasicHttpAuthenticator, 2295 BearerAuthenticator, 2296 OAuthAuthenticator, 2297 JwtAuthenticator, 2298 SessionTokenAuthenticator, 2299 SelectiveAuthenticator, 2300 CustomAuthenticator, 2301 NoAuth, 2302 LegacySessionTokenAuthenticator, 2303 ] 2304 ] = Field( 2305 None, 2306 description="Authentication method to use for requests sent to the API.", 2307 title="Authenticator", 2308 ) 2309 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2310 None, 2311 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2312 title="Fetch Properties from Endpoint", 2313 ) 2314 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2315 None, 2316 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2317 examples=[ 2318 {"unit": "day"}, 2319 { 2320 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2321 }, 2322 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2323 {"sort_by[asc]": "updated_at"}, 2324 ], 2325 title="Query Parameters", 2326 ) 2327 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2328 None, 2329 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2330 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2331 title="Request Headers", 2332 ) 2333 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2334 None, 2335 deprecated=True, 2336 deprecation_message="Use `request_body` field instead.", 2337 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2338 examples=[ 2339 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2340 ], 2341 title="Request Body Payload (Non-JSON)", 2342 ) 2343 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2344 None, 2345 deprecated=True, 2346 deprecation_message="Use `request_body` field instead.", 2347 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2348 examples=[ 2349 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2350 {"key": "{{ config['value'] }}"}, 2351 {"sort": {"field": "updated_at", "order": "ascending"}}, 2352 ], 2353 title="Request Body JSON Payload", 2354 ) 2355 request_body: Optional[ 2356 Union[ 2357 RequestBodyPlainText, 2358 RequestBodyUrlEncodedForm, 2359 RequestBodyJsonObject, 2360 RequestBodyGraphQL, 2361 ] 2362 ] = Field( 2363 None, 2364 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2365 examples=[ 2366 { 2367 "type": "RequestBodyJsonObject", 2368 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2369 }, 2370 { 2371 "type": "RequestBodyJsonObject", 2372 "value": {"key": "{{ config['value'] }}"}, 2373 }, 2374 { 2375 "type": "RequestBodyJsonObject", 2376 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2377 }, 2378 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2379 { 2380 "type": "RequestBodyUrlEncodedForm", 2381 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2382 }, 2383 { 2384 "type": "RequestBodyGraphQL", 2385 "value": { 2386 "query": { 2387 "param1": "value1", 2388 "param2": "{{ config['param2_value'] }}", 2389 } 2390 }, 2391 }, 2392 ], 2393 title="Request Body", 2394 ) 2395 error_handler: Optional[ 2396 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2397 ] = Field( 2398 None, 2399 description="Error handler component that defines how to handle errors.", 2400 title="Error Handler", 2401 ) 2402 use_cache: Optional[bool] = Field( 2403 False, 2404 description="Enables stream requests caching. This field is automatically set by the CDK.", 2405 title="Use Cache", 2406 ) 2407 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2408 2409 2410class DynamicSchemaLoader(BaseModel): 2411 type: Literal["DynamicSchemaLoader"] 2412 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2413 ..., 2414 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2415 title="Retriever", 2416 ) 2417 schema_transformations: Optional[ 2418 List[ 2419 Union[ 2420 AddFields, 2421 CustomTransformation, 2422 RemoveFields, 2423 KeysToLower, 2424 KeysToSnakeCase, 2425 FlattenFields, 2426 DpathFlattenFields, 2427 KeysReplace, 2428 ] 2429 ] 2430 ] = Field( 2431 None, 2432 description="A list of transformations to be applied to the schema.", 2433 title="Schema Transformations", 2434 ) 2435 schema_type_identifier: SchemaTypeIdentifier 2436 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2437 2438 2439class ParentStreamConfig(BaseModel): 2440 type: Literal["ParentStreamConfig"] 2441 lazy_read_pointer: Optional[List[str]] = Field( 2442 [], 2443 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2444 title="Lazy Read Pointer", 2445 ) 2446 parent_key: str = Field( 2447 ..., 2448 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2449 examples=["id", "{{ config['parent_record_id'] }}"], 2450 title="Parent Key", 2451 ) 2452 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2453 ..., description="Reference to the parent stream.", title="Parent Stream" 2454 ) 2455 partition_field: str = Field( 2456 ..., 2457 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2458 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2459 title="Current Parent Key Value Identifier", 2460 ) 2461 request_option: Optional[RequestOption] = Field( 2462 None, 2463 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2464 title="Request Option", 2465 ) 2466 incremental_dependency: Optional[bool] = Field( 2467 False, 2468 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2469 title="Incremental Dependency", 2470 ) 2471 extra_fields: Optional[List[List[str]]] = Field( 2472 None, 2473 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2474 title="Extra Fields", 2475 ) 2476 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2477 2478 2479class PropertiesFromEndpoint(BaseModel): 2480 type: Literal["PropertiesFromEndpoint"] 2481 property_field_path: List[str] = Field( 2482 ..., 2483 description="Describes the path to the field that should be extracted", 2484 examples=[["name"]], 2485 ) 2486 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2487 ..., 2488 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2489 ) 2490 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2491 2492 2493class QueryProperties(BaseModel): 2494 type: Literal["QueryProperties"] 2495 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2496 ..., 2497 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2498 title="Property List", 2499 ) 2500 always_include_properties: Optional[List[str]] = Field( 2501 None, 2502 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2503 title="Always Include Properties", 2504 ) 2505 property_chunking: Optional[PropertyChunking] = Field( 2506 None, 2507 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2508 title="Property Chunking", 2509 ) 2510 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2511 2512 2513class StateDelegatingStream(BaseModel): 2514 type: Literal["StateDelegatingStream"] 2515 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2516 full_refresh_stream: DeclarativeStream = Field( 2517 ..., 2518 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2519 title="Full Refresh Stream", 2520 ) 2521 incremental_stream: DeclarativeStream = Field( 2522 ..., 2523 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2524 title="Incremental Stream", 2525 ) 2526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2527 2528 2529class SimpleRetriever(BaseModel): 2530 type: Literal["SimpleRetriever"] 2531 requester: Union[HttpRequester, CustomRequester] = Field( 2532 ..., 2533 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2534 ) 2535 decoder: Optional[ 2536 Union[ 2537 JsonDecoder, 2538 XmlDecoder, 2539 CsvDecoder, 2540 JsonlDecoder, 2541 GzipDecoder, 2542 IterableDecoder, 2543 ZipfileDecoder, 2544 CustomDecoder, 2545 ] 2546 ] = Field( 2547 None, 2548 description="Component decoding the response so records can be extracted.", 2549 title="HTTP Response Format", 2550 ) 2551 record_selector: RecordSelector = Field( 2552 ..., 2553 description="Component that describes how to extract records from a HTTP response.", 2554 ) 2555 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2556 None, 2557 description="Paginator component that describes how to navigate through the API's pages.", 2558 ) 2559 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2560 False, 2561 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2562 ) 2563 partition_router: Optional[ 2564 Union[ 2565 ListPartitionRouter, 2566 SubstreamPartitionRouter, 2567 GroupingPartitionRouter, 2568 CustomPartitionRouter, 2569 List[ 2570 Union[ 2571 ListPartitionRouter, 2572 SubstreamPartitionRouter, 2573 GroupingPartitionRouter, 2574 CustomPartitionRouter, 2575 ] 2576 ], 2577 ] 2578 ] = Field( 2579 [], 2580 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2581 title="Partition Router", 2582 ) 2583 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2584 2585 2586class AsyncRetriever(BaseModel): 2587 type: Literal["AsyncRetriever"] 2588 record_selector: RecordSelector = Field( 2589 ..., 2590 description="Component that describes how to extract records from a HTTP response.", 2591 ) 2592 status_mapping: AsyncJobStatusMap = Field( 2593 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2594 ) 2595 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2596 ..., description="Responsible for fetching the actual status of the async job." 2597 ) 2598 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2599 ..., 2600 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2601 ) 2602 download_extractor: Optional[ 2603 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2604 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2605 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2606 ..., 2607 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2608 ) 2609 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2610 ..., 2611 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2612 ) 2613 polling_job_timeout: Optional[Union[int, str]] = Field( 2614 None, 2615 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2616 ) 2617 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2618 None, 2619 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2620 ) 2621 download_requester: Union[HttpRequester, CustomRequester] = Field( 2622 ..., 2623 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2624 ) 2625 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2626 None, 2627 description="Paginator component that describes how to navigate through the API's pages during download.", 2628 ) 2629 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2630 None, 2631 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2632 ) 2633 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2634 None, 2635 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2636 ) 2637 partition_router: Optional[ 2638 Union[ 2639 ListPartitionRouter, 2640 SubstreamPartitionRouter, 2641 GroupingPartitionRouter, 2642 CustomPartitionRouter, 2643 List[ 2644 Union[ 2645 ListPartitionRouter, 2646 SubstreamPartitionRouter, 2647 GroupingPartitionRouter, 2648 CustomPartitionRouter, 2649 ] 2650 ], 2651 ] 2652 ] = Field( 2653 [], 2654 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2655 title="Partition Router", 2656 ) 2657 decoder: Optional[ 2658 Union[ 2659 CsvDecoder, 2660 GzipDecoder, 2661 JsonDecoder, 2662 JsonlDecoder, 2663 IterableDecoder, 2664 XmlDecoder, 2665 ZipfileDecoder, 2666 CustomDecoder, 2667 ] 2668 ] = Field( 2669 None, 2670 description="Component decoding the response so records can be extracted.", 2671 title="Decoder", 2672 ) 2673 download_decoder: Optional[ 2674 Union[ 2675 CsvDecoder, 2676 GzipDecoder, 2677 JsonDecoder, 2678 JsonlDecoder, 2679 IterableDecoder, 2680 XmlDecoder, 2681 ZipfileDecoder, 2682 CustomDecoder, 2683 ] 2684 ] = Field( 2685 None, 2686 description="Component decoding the download response so records can be extracted.", 2687 title="Download Decoder", 2688 ) 2689 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2690 2691 2692class SubstreamPartitionRouter(BaseModel): 2693 type: Literal["SubstreamPartitionRouter"] 2694 parent_stream_configs: List[ParentStreamConfig] = Field( 2695 ..., 2696 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2697 title="Parent Stream Configs", 2698 ) 2699 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2700 2701 2702class GroupingPartitionRouter(BaseModel): 2703 type: Literal["GroupingPartitionRouter"] 2704 group_size: int = Field( 2705 ..., 2706 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2707 examples=[10, 50], 2708 title="Group Size", 2709 ) 2710 underlying_partition_router: Union[ 2711 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2712 ] = Field( 2713 ..., 2714 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2715 title="Underlying Partition Router", 2716 ) 2717 deduplicate: Optional[bool] = Field( 2718 True, 2719 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2720 title="Deduplicate Partitions", 2721 ) 2722 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2723 2724 2725class HttpComponentsResolver(BaseModel): 2726 type: Literal["HttpComponentsResolver"] 2727 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2728 ..., 2729 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2730 title="Retriever", 2731 ) 2732 components_mapping: List[ComponentMappingDefinition] 2733 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2734 2735 2736class DynamicDeclarativeStream(BaseModel): 2737 type: Literal["DynamicDeclarativeStream"] 2738 name: Optional[str] = Field( 2739 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2740 ) 2741 stream_template: DeclarativeStream = Field( 2742 ..., description="Reference to the stream template.", title="Stream Template" 2743 ) 2744 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2745 ..., 2746 description="Component resolve and populates stream templates with components values.", 2747 title="Components Resolver", 2748 ) 2749 2750 2751ComplexFieldType.update_forward_refs() 2752GzipDecoder.update_forward_refs() 2753CompositeErrorHandler.update_forward_refs() 2754DeclarativeSource1.update_forward_refs() 2755DeclarativeSource2.update_forward_refs() 2756SelectiveAuthenticator.update_forward_refs() 2757FileUploader.update_forward_refs() 2758DeclarativeStream.update_forward_refs() 2759SessionTokenAuthenticator.update_forward_refs() 2760HttpRequester.update_forward_refs() 2761DynamicSchemaLoader.update_forward_refs() 2762ParentStreamConfig.update_forward_refs() 2763PropertiesFromEndpoint.update_forward_refs() 2764SimpleRetriever.update_forward_refs() 2765AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[int] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomIncrementalSync(pydantic.v1.main.BaseModel):
178class CustomIncrementalSync(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomIncrementalSync"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomIncrementalSync"], 187 title="Class Name", 188 ) 189 cursor_field: str = Field( 190 ..., 191 description="The location of the value on a record that will be used as a bookmark during sync.", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomIncrementalSync.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
196class CustomPaginationStrategy(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomPaginationStrategy"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomPaginationStrategy"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
210class CustomRecordExtractor(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordExtractor"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomRecordExtractor"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
224class CustomRecordFilter(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRecordFilter"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomCustomRecordFilter"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
238class CustomRequester(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRequester"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRecordExtractor"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
252class CustomRetriever(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomRetriever"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomRetriever"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
266class CustomPartitionRouter(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomPartitionRouter"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomPartitionRouter"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
280class CustomSchemaLoader(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaLoader"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 288 examples=["source_railz.components.MyCustomSchemaLoader"], 289 title="Class Name", 290 ) 291 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
294class CustomSchemaNormalization(BaseModel): 295 class Config: 296 extra = Extra.allow 297 298 type: Literal["CustomSchemaNormalization"] 299 class_name: str = Field( 300 ..., 301 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 302 examples=[ 303 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 304 ], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
310class CustomStateMigration(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomStateMigration"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomStateMigration"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
324class CustomTransformation(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomTransformation"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 332 examples=["source_railz.components.MyCustomTransformation"], 333 title="Class Name", 334 ) 335 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
355class Algorithm(Enum): 356 HS256 = "HS256" 357 HS384 = "HS384" 358 HS512 = "HS512" 359 ES256 = "ES256" 360 ES256K = "ES256K" 361 ES384 = "ES384" 362 ES512 = "ES512" 363 RS256 = "RS256" 364 RS384 = "RS384" 365 RS512 = "RS512" 366 PS256 = "PS256" 367 PS384 = "PS384" 368 PS512 = "PS512" 369 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
372class JwtHeaders(BaseModel): 373 class Config: 374 extra = Extra.forbid 375 376 kid: Optional[str] = Field( 377 None, 378 description="Private key ID for user account.", 379 examples=["{{ config['kid'] }}"], 380 title="Key Identifier", 381 ) 382 typ: Optional[str] = Field( 383 "JWT", 384 description="The media type of the complete JWT.", 385 examples=["JWT"], 386 title="Type", 387 ) 388 cty: Optional[str] = Field( 389 None, 390 description="Content type of JWT header.", 391 examples=["JWT"], 392 title="Content Type", 393 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
396class JwtPayload(BaseModel): 397 class Config: 398 extra = Extra.forbid 399 400 iss: Optional[str] = Field( 401 None, 402 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 403 examples=["{{ config['iss'] }}"], 404 title="Issuer", 405 ) 406 sub: Optional[str] = Field( 407 None, 408 description="The subject of the JWT. Commonly defined by the API.", 409 title="Subject", 410 ) 411 aud: Optional[str] = Field( 412 None, 413 description="The recipient that the JWT is intended for. Commonly defined by the API.", 414 examples=["appstoreconnect-v1"], 415 title="Audience", 416 )
class
JwtPayload.Config:
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
419class JwtAuthenticator(BaseModel): 420 type: Literal["JwtAuthenticator"] 421 secret_key: str = Field( 422 ..., 423 description="Secret used to sign the JSON web token.", 424 examples=["{{ config['secret_key'] }}"], 425 title="Secret Key", 426 ) 427 base64_encode_secret_key: Optional[bool] = Field( 428 False, 429 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 430 title="Base64-encode Secret Key", 431 ) 432 algorithm: Algorithm = Field( 433 ..., 434 description="Algorithm used to sign the JSON web token.", 435 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 436 title="Algorithm", 437 ) 438 token_duration: Optional[int] = Field( 439 1200, 440 description="The amount of time in seconds a JWT token can be valid after being issued.", 441 examples=[1200, 3600], 442 title="Token Duration", 443 ) 444 header_prefix: Optional[str] = Field( 445 None, 446 description="The prefix to be used within the Authentication header.", 447 examples=["Bearer", "Basic"], 448 title="Header Prefix", 449 ) 450 jwt_headers: Optional[JwtHeaders] = Field( 451 None, 452 description="JWT headers used when signing JSON web token.", 453 title="JWT Headers", 454 ) 455 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 456 None, 457 description="Additional headers to be included with the JWT headers object.", 458 title="Additional JWT Headers", 459 ) 460 jwt_payload: Optional[JwtPayload] = Field( 461 None, 462 description="JWT Payload used when signing JSON web token.", 463 title="JWT Payload", 464 ) 465 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 466 None, 467 description="Additional properties to be added to the JWT payload.", 468 title="Additional JWT Payload Properties", 469 ) 470 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
473class RefreshTokenUpdater(BaseModel): 474 refresh_token_name: Optional[str] = Field( 475 "refresh_token", 476 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 477 examples=["refresh_token"], 478 title="Refresh Token Property Name", 479 ) 480 access_token_config_path: Optional[List[str]] = Field( 481 ["credentials", "access_token"], 482 description="Config path to the access token. Make sure the field actually exists in the config.", 483 examples=[["credentials", "access_token"], ["access_token"]], 484 title="Config Path To Access Token", 485 ) 486 refresh_token_config_path: Optional[List[str]] = Field( 487 ["credentials", "refresh_token"], 488 description="Config path to the access token. Make sure the field actually exists in the config.", 489 examples=[["credentials", "refresh_token"], ["refresh_token"]], 490 title="Config Path To Refresh Token", 491 ) 492 token_expiry_date_config_path: Optional[List[str]] = Field( 493 ["credentials", "token_expiry_date"], 494 description="Config path to the expiry date. Make sure actually exists in the config.", 495 examples=[["credentials", "token_expiry_date"]], 496 title="Config Path To Expiry Date", 497 ) 498 refresh_token_error_status_codes: Optional[List[int]] = Field( 499 [], 500 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 501 examples=[[400, 500]], 502 title="Refresh Token Error Status Codes", 503 ) 504 refresh_token_error_key: Optional[str] = Field( 505 "", 506 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 507 examples=["error"], 508 title="Refresh Token Error Key", 509 ) 510 refresh_token_error_values: Optional[List[str]] = Field( 511 [], 512 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 513 examples=[["invalid_grant", "invalid_permissions"]], 514 title="Refresh Token Error Values", 515 )
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
518class OAuthAuthenticator(BaseModel): 519 type: Literal["OAuthAuthenticator"] 520 client_id_name: Optional[str] = Field( 521 "client_id", 522 description="The name of the property to use to refresh the `access_token`.", 523 examples=["custom_app_id"], 524 title="Client ID Property Name", 525 ) 526 client_id: Optional[str] = Field( 527 None, 528 description="The OAuth client ID. Fill it in the user inputs.", 529 examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], 530 title="Client ID", 531 ) 532 client_secret_name: Optional[str] = Field( 533 "client_secret", 534 description="The name of the property to use to refresh the `access_token`.", 535 examples=["custom_app_secret"], 536 title="Client Secret Property Name", 537 ) 538 client_secret: Optional[str] = Field( 539 None, 540 description="The OAuth client secret. Fill it in the user inputs.", 541 examples=[ 542 "{{ config['client_secret }}", 543 "{{ config['credentials']['client_secret }}", 544 ], 545 title="Client Secret", 546 ) 547 refresh_token_name: Optional[str] = Field( 548 "refresh_token", 549 description="The name of the property to use to refresh the `access_token`.", 550 examples=["custom_app_refresh_value"], 551 title="Refresh Token Property Name", 552 ) 553 refresh_token: Optional[str] = Field( 554 None, 555 description="Credential artifact used to get a new access token.", 556 examples=[ 557 "{{ config['refresh_token'] }}", 558 "{{ config['credentials]['refresh_token'] }}", 559 ], 560 title="Refresh Token", 561 ) 562 token_refresh_endpoint: Optional[str] = Field( 563 None, 564 description="The full URL to call to obtain a new access token.", 565 examples=["https://connect.squareup.com/oauth2/token"], 566 title="Token Refresh Endpoint", 567 ) 568 access_token_name: Optional[str] = Field( 569 "access_token", 570 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 571 examples=["access_token"], 572 title="Access Token Property Name", 573 ) 574 access_token_value: Optional[str] = Field( 575 None, 576 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 577 examples=["secret_access_token_value"], 578 title="Access Token Value", 579 ) 580 expires_in_name: Optional[str] = Field( 581 "expires_in", 582 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 583 examples=["expires_in"], 584 title="Token Expiry Property Name", 585 ) 586 grant_type_name: Optional[str] = Field( 587 "grant_type", 588 description="The name of the property to use to refresh the `access_token`.", 589 examples=["custom_grant_type"], 590 title="Grant Type Property Name", 591 ) 592 grant_type: Optional[str] = Field( 593 "refresh_token", 594 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 595 examples=["refresh_token", "client_credentials"], 596 title="Grant Type", 597 ) 598 refresh_request_body: Optional[Dict[str, Any]] = Field( 599 None, 600 description="Body of the request sent to get a new access token.", 601 examples=[ 602 { 603 "applicationId": "{{ config['application_id'] }}", 604 "applicationSecret": "{{ config['application_secret'] }}", 605 "token": "{{ config['token'] }}", 606 } 607 ], 608 title="Refresh Request Body", 609 ) 610 refresh_request_headers: Optional[Dict[str, Any]] = Field( 611 None, 612 description="Headers of the request sent to get a new access token.", 613 examples=[ 614 { 615 "Authorization": "<AUTH_TOKEN>", 616 "Content-Type": "application/x-www-form-urlencoded", 617 } 618 ], 619 title="Refresh Request Headers", 620 ) 621 scopes: Optional[List[str]] = Field( 622 None, 623 description="List of scopes that should be granted to the access token.", 624 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 625 title="Scopes", 626 ) 627 token_expiry_date: Optional[str] = Field( 628 None, 629 description="The access token expiry date.", 630 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 631 title="Token Expiry Date", 632 ) 633 token_expiry_date_format: Optional[str] = Field( 634 None, 635 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 636 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 637 title="Token Expiry Date Format", 638 ) 639 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 640 None, 641 description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 642 title="Token Updater", 643 ) 644 profile_assertion: Optional[JwtAuthenticator] = Field( 645 None, 646 description="The authenticator being used to authenticate the client authenticator.", 647 title="Profile Assertion", 648 ) 649 use_profile_assertion: Optional[bool] = Field( 650 False, 651 description="Enable using profile assertion as a flow for OAuth authorization.", 652 title="Use Profile Assertion", 653 ) 654 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
Rate(pydantic.v1.main.BaseModel):
657class Rate(BaseModel): 658 class Config: 659 extra = Extra.allow 660 661 limit: Union[int, str] = Field( 662 ..., 663 description="The maximum number of calls allowed within the interval.", 664 title="Limit", 665 ) 666 interval: str = Field( 667 ..., 668 description="The time interval for the rate limit.", 669 examples=["PT1H", "P1D"], 670 title="Interval", 671 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
674class HttpRequestRegexMatcher(BaseModel): 675 class Config: 676 extra = Extra.allow 677 678 method: Optional[str] = Field( 679 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 680 ) 681 url_base: Optional[str] = Field( 682 None, 683 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 684 title="URL Base", 685 ) 686 url_path_pattern: Optional[str] = Field( 687 None, 688 description="A regular expression pattern to match the URL path.", 689 title="URL Path Pattern", 690 ) 691 params: Optional[Dict[str, Any]] = Field( 692 None, description="The query parameters to match.", title="Parameters" 693 ) 694 headers: Optional[Dict[str, Any]] = Field( 695 None, description="The headers to match.", title="Headers" 696 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
699class DpathExtractor(BaseModel): 700 type: Literal["DpathExtractor"] 701 field_path: List[str] = Field( 702 ..., 703 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 704 examples=[ 705 ["data"], 706 ["data", "records"], 707 ["data", "{{ parameters.name }}"], 708 ["data", "*", "record"], 709 ], 710 title="Field Path", 711 ) 712 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
720class ExponentialBackoffStrategy(BaseModel): 721 type: Literal["ExponentialBackoffStrategy"] 722 factor: Optional[Union[float, str]] = Field( 723 5, 724 description="Multiplicative constant applied on each retry.", 725 examples=[5, 5.5, "10"], 726 title="Factor", 727 ) 728 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
731class GroupByKeyMergeStrategy(BaseModel): 732 type: Literal["GroupByKeyMergeStrategy"] 733 key: Union[str, List[str]] = Field( 734 ..., 735 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 736 examples=["id", ["parent_id", "end_date"]], 737 title="Key", 738 ) 739 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
751class Action(Enum): 752 SUCCESS = "SUCCESS" 753 FAIL = "FAIL" 754 RETRY = "RETRY" 755 IGNORE = "IGNORE" 756 RATE_LIMITED = "RATE_LIMITED"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
759class FailureType(Enum): 760 system_error = "system_error" 761 config_error = "config_error" 762 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
765class HttpResponseFilter(BaseModel): 766 type: Literal["HttpResponseFilter"] 767 action: Optional[Action] = Field( 768 None, 769 description="Action to execute if a response matches the filter.", 770 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 771 title="Action", 772 ) 773 failure_type: Optional[FailureType] = Field( 774 None, 775 description="Failure type of traced exception if a response matches the filter.", 776 examples=["system_error", "config_error", "transient_error"], 777 title="Failure Type", 778 ) 779 error_message: Optional[str] = Field( 780 None, 781 description="Error Message to display if the response matches the filter.", 782 title="Error Message", 783 ) 784 error_message_contains: Optional[str] = Field( 785 None, 786 description="Match the response if its error message contains the substring.", 787 example=["This API operation is not enabled for this site"], 788 title="Error Message Substring", 789 ) 790 http_codes: Optional[List[int]] = Field( 791 None, 792 description="Match the response if its HTTP code is included in this list.", 793 examples=[[420, 429], [500]], 794 title="HTTP Codes", 795 unique_items=True, 796 ) 797 predicate: Optional[str] = Field( 798 None, 799 description="Match the response if the predicate evaluates to true.", 800 examples=[ 801 "{{ 'Too much requests' in response }}", 802 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 803 ], 804 title="Predicate", 805 ) 806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
809class ComplexFieldType(BaseModel): 810 field_type: str 811 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
814class TypesMap(BaseModel): 815 target_type: Union[str, List[str], ComplexFieldType] 816 current_type: Union[str, List[str]] 817 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
820class SchemaTypeIdentifier(BaseModel): 821 type: Optional[Literal["SchemaTypeIdentifier"]] = None 822 schema_pointer: Optional[List[str]] = Field( 823 [], 824 description="List of nested fields defining the schema field path to extract. Defaults to [].", 825 title="Schema Path", 826 ) 827 key_pointer: List[str] = Field( 828 ..., 829 description="List of potentially nested fields describing the full path of the field key to extract.", 830 title="Key Path", 831 ) 832 type_pointer: Optional[List[str]] = Field( 833 None, 834 description="List of potentially nested fields describing the full path of the field type to extract.", 835 title="Type Path", 836 ) 837 types_mapping: Optional[List[TypesMap]] = None 838 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
841class InlineSchemaLoader(BaseModel): 842 type: Literal["InlineSchemaLoader"] 843 schema_: Optional[Dict[str, Any]] = Field( 844 None, 845 alias="schema", 846 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 847 title="Schema", 848 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
851class JsonFileSchemaLoader(BaseModel): 852 type: Literal["JsonFileSchemaLoader"] 853 file_path: Optional[str] = Field( 854 None, 855 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 856 example=["./schemas/users.json"], 857 title="File Path", 858 ) 859 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
880class FlattenFields(BaseModel): 881 type: Literal["FlattenFields"] 882 flatten_lists: Optional[bool] = Field( 883 True, 884 description="Whether to flatten lists or leave it as is. Default is True.", 885 title="Flatten Lists", 886 ) 887 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
890class KeyTransformation(BaseModel): 891 type: Literal["KeyTransformation"] 892 prefix: Optional[str] = Field( 893 None, 894 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 895 examples=["flattened_"], 896 title="Key Prefix", 897 ) 898 suffix: Optional[str] = Field( 899 None, 900 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 901 examples=["_flattened"], 902 title="Key Suffix", 903 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
906class DpathFlattenFields(BaseModel): 907 type: Literal["DpathFlattenFields"] 908 field_path: List[str] = Field( 909 ..., 910 description="A path to field that needs to be flattened.", 911 examples=[["data"], ["data", "*", "field"]], 912 title="Field Path", 913 ) 914 delete_origin_value: Optional[bool] = Field( 915 None, 916 description="Whether to delete the origin value or keep it. Default is False.", 917 title="Delete Origin Value", 918 ) 919 replace_record: Optional[bool] = Field( 920 None, 921 description="Whether to replace the origin record or not. Default is False.", 922 title="Replace Origin Record", 923 ) 924 key_transformation: Optional[KeyTransformation] = Field( 925 None, 926 description="Transformation for object keys. If not provided, original key will be used.", 927 title="Key transformation", 928 ) 929 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
932class KeysReplace(BaseModel): 933 type: Literal["KeysReplace"] 934 old: str = Field( 935 ..., 936 description="Old value to replace.", 937 examples=[ 938 " ", 939 "{{ record.id }}", 940 "{{ config['id'] }}", 941 "{{ stream_slice['id'] }}", 942 ], 943 title="Old value", 944 ) 945 new: str = Field( 946 ..., 947 description="New value to set.", 948 examples=[ 949 "_", 950 "{{ record.id }}", 951 "{{ config['id'] }}", 952 "{{ stream_slice['id'] }}", 953 ], 954 title="New value", 955 ) 956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
967class CustomDecoder(BaseModel): 968 class Config: 969 extra = Extra.allow 970 971 type: Literal["CustomDecoder"] 972 class_name: str = Field( 973 ..., 974 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 975 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 976 title="Class Name", 977 ) 978 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
981class MinMaxDatetime(BaseModel): 982 type: Literal["MinMaxDatetime"] 983 datetime: str = Field( 984 ..., 985 description="Datetime value.", 986 examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], 987 title="Datetime", 988 ) 989 datetime_format: Optional[str] = Field( 990 "", 991 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 992 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 993 title="Datetime Format", 994 ) 995 max_datetime: Optional[str] = Field( 996 None, 997 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 998 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 999 title="Max Datetime", 1000 ) 1001 min_datetime: Optional[str] = Field( 1002 None, 1003 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 1004 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 1005 title="Min Datetime", 1006 ) 1007 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1027class OauthConnectorInputSpecification(BaseModel): 1028 class Config: 1029 extra = Extra.allow 1030 1031 consent_url: str = Field( 1032 ..., 1033 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1034 examples=[ 1035 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1036 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1037 ], 1038 title="Consent URL", 1039 ) 1040 scope: Optional[str] = Field( 1041 None, 1042 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1043 examples=["user:read user:read_orders workspaces:read"], 1044 title="Scopes", 1045 ) 1046 access_token_url: str = Field( 1047 ..., 1048 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1049 examples=[ 1050 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1051 ], 1052 title="Access Token URL", 1053 ) 1054 access_token_headers: Optional[Dict[str, Any]] = Field( 1055 None, 1056 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1057 examples=[ 1058 { 1059 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1060 } 1061 ], 1062 title="Access Token Headers", 1063 ) 1064 access_token_params: Optional[Dict[str, Any]] = Field( 1065 None, 1066 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1067 examples=[ 1068 { 1069 "{{ auth_code_key }}": "{{ auth_code_value }}", 1070 "{{ client_id_key }}": "{{ client_id_value }}", 1071 "{{ client_secret_key }}": "{{ client_secret_value }}", 1072 } 1073 ], 1074 title="Access Token Query Params (Json Encoded)", 1075 ) 1076 extract_output: Optional[List[str]] = Field( 1077 None, 1078 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1079 examples=[["access_token", "refresh_token", "other_field"]], 1080 title="Extract Output", 1081 ) 1082 state: Optional[State] = Field( 1083 None, 1084 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1085 examples=[{"min": 7, "max": 128}], 1086 title="Configurable State Query Param", 1087 ) 1088 client_id_key: Optional[str] = Field( 1089 None, 1090 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1091 examples=["my_custom_client_id_key_name"], 1092 title="Client ID Key Override", 1093 ) 1094 client_secret_key: Optional[str] = Field( 1095 None, 1096 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1097 examples=["my_custom_client_secret_key_name"], 1098 title="Client Secret Key Override", 1099 ) 1100 scope_key: Optional[str] = Field( 1101 None, 1102 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1103 examples=["my_custom_scope_key_key_name"], 1104 title="Scopes Key Override", 1105 ) 1106 state_key: Optional[str] = Field( 1107 None, 1108 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1109 examples=["my_custom_state_key_key_name"], 1110 title="State Key Override", 1111 ) 1112 auth_code_key: Optional[str] = Field( 1113 None, 1114 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1115 examples=["my_custom_auth_code_key_name"], 1116 title="Auth Code Key Override", 1117 ) 1118 redirect_uri_key: Optional[str] = Field( 1119 None, 1120 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1121 examples=["my_custom_redirect_uri_key_name"], 1122 title="Redirect URI Key Override", 1123 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1126class OAuthConfigSpecification(BaseModel): 1127 class Config: 1128 extra = Extra.allow 1129 1130 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1131 None, 1132 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1133 examples=[ 1134 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1135 { 1136 "app_id": { 1137 "type": "string", 1138 "path_in_connector_config": ["info", "app_id"], 1139 } 1140 }, 1141 ], 1142 title="OAuth user input", 1143 ) 1144 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1145 None, 1146 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1147 title="DeclarativeOAuth Connector Specification", 1148 ) 1149 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1150 None, 1151 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1152 examples=[ 1153 { 1154 "refresh_token": { 1155 "type": "string,", 1156 "path_in_connector_config": ["credentials", "refresh_token"], 1157 } 1158 } 1159 ], 1160 title="OAuth output specification", 1161 ) 1162 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1163 None, 1164 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1165 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1166 title="OAuth input specification", 1167 ) 1168 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1169 None, 1170 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1171 examples=[ 1172 { 1173 "client_id": { 1174 "type": "string,", 1175 "path_in_connector_config": ["credentials", "client_id"], 1176 }, 1177 "client_secret": { 1178 "type": "string,", 1179 "path_in_connector_config": ["credentials", "client_secret"], 1180 }, 1181 } 1182 ], 1183 title="OAuth server output specification", 1184 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1187class OffsetIncrement(BaseModel): 1188 type: Literal["OffsetIncrement"] 1189 page_size: Optional[Union[int, str]] = Field( 1190 None, 1191 description="The number of records to include in each pages.", 1192 examples=[100, "{{ config['page_size'] }}"], 1193 title="Limit", 1194 ) 1195 inject_on_first_request: Optional[bool] = Field( 1196 False, 1197 description="Using the `offset` with value `0` during the first request", 1198 title="Inject Offset", 1199 ) 1200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1203class PageIncrement(BaseModel): 1204 type: Literal["PageIncrement"] 1205 page_size: Optional[Union[int, str]] = Field( 1206 None, 1207 description="The number of records to include in each pages.", 1208 examples=[100, "100", "{{ config['page_size'] }}"], 1209 title="Page Size", 1210 ) 1211 start_from_page: Optional[int] = Field( 1212 0, 1213 description="Index of the first page to request.", 1214 examples=[0, 1], 1215 title="Start From Page", 1216 ) 1217 inject_on_first_request: Optional[bool] = Field( 1218 False, 1219 description="Using the `page number` with value defined by `start_from_page` during the first request", 1220 title="Inject Page Number", 1221 ) 1222 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1225class PrimaryKey(BaseModel): 1226 __root__: Union[str, List[str], List[List[str]]] = Field( 1227 ..., 1228 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1229 examples=["id", ["code", "type"]], 1230 title="Primary Key", 1231 )
class
PropertyLimitType(enum.Enum):
1234class PropertyLimitType(Enum): 1235 characters = "characters" 1236 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1239class PropertyChunking(BaseModel): 1240 type: Literal["PropertyChunking"] 1241 property_limit_type: PropertyLimitType = Field( 1242 ..., 1243 description="The type used to determine the maximum number of properties per chunk", 1244 title="Property Limit Type", 1245 ) 1246 property_limit: Optional[int] = Field( 1247 None, 1248 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1249 title="Property Limit", 1250 ) 1251 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1252 None, 1253 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1254 title="Record Merge Strategy", 1255 ) 1256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1259class RecordFilter(BaseModel): 1260 type: Literal["RecordFilter"] 1261 condition: Optional[str] = Field( 1262 "", 1263 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1264 examples=[ 1265 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1266 "{{ record.status in ['active', 'expired'] }}", 1267 ], 1268 title="Condition", 1269 ) 1270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1278class RemoveFields(BaseModel): 1279 type: Literal["RemoveFields"] 1280 condition: Optional[str] = Field( 1281 "", 1282 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1283 examples=[ 1284 "{{ property|string == '' }}", 1285 "{{ property is integer }}", 1286 "{{ property|length > 5 }}", 1287 "{{ property == 'some_string_to_match' }}", 1288 ], 1289 ) 1290 field_pointers: List[List[str]] = Field( 1291 ..., 1292 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1293 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1294 title="Field Paths", 1295 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1302class InjectInto(Enum): 1303 request_parameter = "request_parameter" 1304 header = "header" 1305 body_data = "body_data" 1306 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1309class RequestOption(BaseModel): 1310 type: Literal["RequestOption"] 1311 field_name: Optional[str] = Field( 1312 None, 1313 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1314 examples=["segment_id"], 1315 title="Field Name", 1316 ) 1317 field_path: Optional[List[str]] = Field( 1318 None, 1319 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1320 examples=[["data", "viewer", "id"]], 1321 title="Field Path", 1322 ) 1323 inject_into: InjectInto = Field( 1324 ..., 1325 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1326 examples=["request_parameter", "header", "body_data", "body_json"], 1327 title="Inject Into", 1328 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1338class LegacySessionTokenAuthenticator(BaseModel): 1339 type: Literal["LegacySessionTokenAuthenticator"] 1340 header: str = Field( 1341 ..., 1342 description="The name of the session token header that will be injected in the request", 1343 examples=["X-Session"], 1344 title="Session Request Header", 1345 ) 1346 login_url: str = Field( 1347 ..., 1348 description="Path of the login URL (do not include the base URL)", 1349 examples=["session"], 1350 title="Login Path", 1351 ) 1352 session_token: Optional[str] = Field( 1353 None, 1354 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1355 example=["{{ config['session_token'] }}"], 1356 title="Session Token", 1357 ) 1358 session_token_response_key: str = Field( 1359 ..., 1360 description="Name of the key of the session token to be extracted from the response", 1361 examples=["id"], 1362 title="Response Token Response Key", 1363 ) 1364 username: Optional[str] = Field( 1365 None, 1366 description="Username used to authenticate and obtain a session token", 1367 examples=[" {{ config['username'] }}"], 1368 title="Username", 1369 ) 1370 password: Optional[str] = Field( 1371 "", 1372 description="Password used to authenticate and obtain a session token", 1373 examples=["{{ config['password'] }}", ""], 1374 title="Password", 1375 ) 1376 validate_session_url: str = Field( 1377 ..., 1378 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1379 examples=["user/current"], 1380 title="Validate Session Path", 1381 ) 1382 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1399class ValueType(Enum): 1400 string = "string" 1401 number = "number" 1402 integer = "integer" 1403 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1406class WaitTimeFromHeader(BaseModel): 1407 type: Literal["WaitTimeFromHeader"] 1408 header: str = Field( 1409 ..., 1410 description="The name of the response header defining how long to wait before retrying.", 1411 examples=["Retry-After"], 1412 title="Response Header Name", 1413 ) 1414 regex: Optional[str] = Field( 1415 None, 1416 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1417 examples=["([-+]?\\d+)"], 1418 title="Extraction Regex", 1419 ) 1420 max_waiting_time_in_seconds: Optional[float] = Field( 1421 None, 1422 description="Given the value extracted from the header is greater than this value, stop the stream.", 1423 examples=[3600], 1424 title="Max Waiting Time in Seconds", 1425 ) 1426 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1429class WaitUntilTimeFromHeader(BaseModel): 1430 type: Literal["WaitUntilTimeFromHeader"] 1431 header: str = Field( 1432 ..., 1433 description="The name of the response header defining how long to wait before retrying.", 1434 examples=["wait_time"], 1435 title="Response Header", 1436 ) 1437 min_wait: Optional[Union[float, str]] = Field( 1438 None, 1439 description="Minimum time to wait before retrying.", 1440 examples=[10, "60"], 1441 title="Minimum Wait Time", 1442 ) 1443 regex: Optional[str] = Field( 1444 None, 1445 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1446 examples=["([-+]?\\d+)"], 1447 title="Extraction Regex", 1448 ) 1449 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1452class ComponentMappingDefinition(BaseModel): 1453 type: Literal["ComponentMappingDefinition"] 1454 field_path: List[str] = Field( 1455 ..., 1456 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1457 examples=[ 1458 ["data"], 1459 ["data", "records"], 1460 ["data", 1, "name"], 1461 ["data", "{{ components_values.name }}"], 1462 ["data", "*", "record"], 1463 ["*", "**", "name"], 1464 ], 1465 title="Field Path", 1466 ) 1467 value: str = Field( 1468 ..., 1469 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1470 examples=[ 1471 "{{ components_values['updates'] }}", 1472 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1473 "{{ config['segment_id'] }}", 1474 "{{ stream_slice['parent_id'] }}", 1475 "{{ stream_slice['extra_fields']['name'] }}", 1476 ], 1477 title="Value", 1478 ) 1479 value_type: Optional[ValueType] = Field( 1480 None, 1481 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1482 title="Value Type", 1483 ) 1484 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1487class StreamConfig(BaseModel): 1488 type: Literal["StreamConfig"] 1489 configs_pointer: List[str] = Field( 1490 ..., 1491 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1492 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1493 title="Configs Pointer", 1494 ) 1495 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1498class ConfigComponentsResolver(BaseModel): 1499 type: Literal["ConfigComponentsResolver"] 1500 stream_config: StreamConfig 1501 components_mapping: List[ComponentMappingDefinition] 1502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: StreamConfig
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1527class AddedFieldDefinition(BaseModel): 1528 type: Literal["AddedFieldDefinition"] 1529 path: List[str] = Field( 1530 ..., 1531 description="List of strings defining the path where to add the value on the record.", 1532 examples=[["segment_id"], ["metadata", "segment_id"]], 1533 title="Path", 1534 ) 1535 value: str = Field( 1536 ..., 1537 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1538 examples=[ 1539 "{{ record['updates'] }}", 1540 "{{ record['MetaData']['LastUpdatedTime'] }}", 1541 "{{ stream_partition['segment_id'] }}", 1542 ], 1543 title="Value", 1544 ) 1545 value_type: Optional[ValueType] = Field( 1546 None, 1547 description="Type of the value. If not specified, the type will be inferred from the value.", 1548 title="Value Type", 1549 ) 1550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1553class AddFields(BaseModel): 1554 type: Literal["AddFields"] 1555 fields: List[AddedFieldDefinition] = Field( 1556 ..., 1557 description="List of transformations (path and corresponding value) that will be added to the record.", 1558 title="Fields", 1559 ) 1560 condition: Optional[str] = Field( 1561 "", 1562 description="Fields will be added if expression is evaluated to True.", 1563 examples=[ 1564 "{{ property|string == '' }}", 1565 "{{ property is integer }}", 1566 "{{ property|length > 5 }}", 1567 "{{ property == 'some_string_to_match' }}", 1568 ], 1569 ) 1570 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1573class ApiKeyAuthenticator(BaseModel): 1574 type: Literal["ApiKeyAuthenticator"] 1575 api_token: Optional[str] = Field( 1576 None, 1577 description="The API key to inject in the request. Fill it in the user inputs.", 1578 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1579 title="API Key", 1580 ) 1581 header: Optional[str] = Field( 1582 None, 1583 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1584 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1585 title="Header Name", 1586 ) 1587 inject_into: Optional[RequestOption] = Field( 1588 None, 1589 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1590 examples=[ 1591 {"inject_into": "header", "field_name": "Authorization"}, 1592 {"inject_into": "request_parameter", "field_name": "authKey"}, 1593 ], 1594 title="Inject API Key Into Outgoing HTTP Request", 1595 ) 1596 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1599class AuthFlow(BaseModel): 1600 auth_flow_type: Optional[AuthFlowType] = Field( 1601 None, description="The type of auth to use", title="Auth flow type" 1602 ) 1603 predicate_key: Optional[List[str]] = Field( 1604 None, 1605 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1606 examples=[["credentials", "auth_type"]], 1607 title="Predicate key", 1608 ) 1609 predicate_value: Optional[str] = Field( 1610 None, 1611 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1612 examples=["Oauth"], 1613 title="Predicate value", 1614 ) 1615 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1618class CheckStream(BaseModel): 1619 type: Literal["CheckStream"] 1620 stream_names: Optional[List[str]] = Field( 1621 None, 1622 description="Names of the streams to try reading from when running a check operation.", 1623 examples=[["users"], ["users", "contacts"]], 1624 title="Stream Names", 1625 ) 1626 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1629class IncrementingCountCursor(BaseModel): 1630 type: Literal["IncrementingCountCursor"] 1631 cursor_field: str = Field( 1632 ..., 1633 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1634 examples=["created_at", "{{ config['record_cursor'] }}"], 1635 title="Cursor Field", 1636 ) 1637 start_value: Optional[Union[str, int]] = Field( 1638 None, 1639 description="The value that determines the earliest record that should be synced.", 1640 examples=[0, "{{ config['start_value'] }}"], 1641 title="Start Value", 1642 ) 1643 start_value_option: Optional[RequestOption] = Field( 1644 None, 1645 description="Optionally configures how the start value will be sent in requests to the source API.", 1646 title="Inject Start Value Into Outgoing HTTP Request", 1647 ) 1648 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1651class DatetimeBasedCursor(BaseModel): 1652 type: Literal["DatetimeBasedCursor"] 1653 clamping: Optional[Clamping] = Field( 1654 None, 1655 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1656 title="Date Range Clamping", 1657 ) 1658 cursor_field: str = Field( 1659 ..., 1660 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1661 examples=["created_at", "{{ config['record_cursor'] }}"], 1662 title="Cursor Field", 1663 ) 1664 datetime_format: str = Field( 1665 ..., 1666 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1667 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1668 title="Outgoing Datetime Format", 1669 ) 1670 start_datetime: Union[str, MinMaxDatetime] = Field( 1671 ..., 1672 description="The datetime that determines the earliest record that should be synced.", 1673 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1674 title="Start Datetime", 1675 ) 1676 cursor_datetime_formats: Optional[List[str]] = Field( 1677 None, 1678 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", 1679 title="Cursor Datetime Formats", 1680 ) 1681 cursor_granularity: Optional[str] = Field( 1682 None, 1683 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", 1684 examples=["PT1S"], 1685 title="Cursor Granularity", 1686 ) 1687 end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( 1688 None, 1689 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1690 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1691 title="End Datetime", 1692 ) 1693 end_time_option: Optional[RequestOption] = Field( 1694 None, 1695 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1696 title="Inject End Time Into Outgoing HTTP Request", 1697 ) 1698 is_data_feed: Optional[bool] = Field( 1699 None, 1700 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1701 title="Whether the target API is formatted as a data feed", 1702 ) 1703 is_client_side_incremental: Optional[bool] = Field( 1704 None, 1705 description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1706 title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", 1707 ) 1708 is_compare_strictly: Optional[bool] = Field( 1709 False, 1710 description="Set to True if the target API does not accept queries where the start time equal the end time.", 1711 title="Whether to skip requests if the start time equals the end time", 1712 ) 1713 global_substream_cursor: Optional[bool] = Field( 1714 False, 1715 description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1716 title="Whether to store cursor as one value instead of per partition", 1717 ) 1718 lookback_window: Optional[str] = Field( 1719 None, 1720 description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", 1721 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1722 title="Lookback Window", 1723 ) 1724 partition_field_end: Optional[str] = Field( 1725 None, 1726 description="Name of the partition start time field.", 1727 examples=["ending_time"], 1728 title="Partition Field End", 1729 ) 1730 partition_field_start: Optional[str] = Field( 1731 None, 1732 description="Name of the partition end time field.", 1733 examples=["starting_time"], 1734 title="Partition Field Start", 1735 ) 1736 start_time_option: Optional[RequestOption] = Field( 1737 None, 1738 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1739 title="Inject Start Time Into Outgoing HTTP Request", 1740 ) 1741 step: Optional[str] = Field( 1742 None, 1743 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", 1744 examples=["P1W", "{{ config['step_increment'] }}"], 1745 title="Step", 1746 ) 1747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[str, MinMaxDatetime]
end_datetime: Union[str, MinMaxDatetime, NoneType]
end_time_option: Optional[RequestOption]
start_time_option: Optional[RequestOption]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1750class FixedWindowCallRatePolicy(BaseModel): 1751 class Config: 1752 extra = Extra.allow 1753 1754 type: Literal["FixedWindowCallRatePolicy"] 1755 period: str = Field( 1756 ..., description="The time interval for the rate limit window.", title="Period" 1757 ) 1758 call_limit: int = Field( 1759 ..., 1760 description="The maximum number of calls allowed within the period.", 1761 title="Call Limit", 1762 ) 1763 matchers: List[HttpRequestRegexMatcher] = Field( 1764 ..., 1765 description="List of matchers that define which requests this policy applies to.", 1766 title="Matchers", 1767 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1770class MovingWindowCallRatePolicy(BaseModel): 1771 class Config: 1772 extra = Extra.allow 1773 1774 type: Literal["MovingWindowCallRatePolicy"] 1775 rates: List[Rate] = Field( 1776 ..., 1777 description="List of rates that define the call limits for different time intervals.", 1778 title="Rates", 1779 ) 1780 matchers: List[HttpRequestRegexMatcher] = Field( 1781 ..., 1782 description="List of matchers that define which requests this policy applies to.", 1783 title="Matchers", 1784 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1787class UnlimitedCallRatePolicy(BaseModel): 1788 class Config: 1789 extra = Extra.allow 1790 1791 type: Literal["UnlimitedCallRatePolicy"] 1792 matchers: List[HttpRequestRegexMatcher] = Field( 1793 ..., 1794 description="List of matchers that define which requests this policy applies to.", 1795 title="Matchers", 1796 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1799class DefaultErrorHandler(BaseModel): 1800 type: Literal["DefaultErrorHandler"] 1801 backoff_strategies: Optional[ 1802 List[ 1803 Union[ 1804 ConstantBackoffStrategy, 1805 CustomBackoffStrategy, 1806 ExponentialBackoffStrategy, 1807 WaitTimeFromHeader, 1808 WaitUntilTimeFromHeader, 1809 ] 1810 ] 1811 ] = Field( 1812 None, 1813 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1814 title="Backoff Strategies", 1815 ) 1816 max_retries: Optional[int] = Field( 1817 5, 1818 description="The maximum number of time to retry a retryable request before giving up and failing.", 1819 examples=[5, 0, 10], 1820 title="Max Retry Count", 1821 ) 1822 response_filters: Optional[List[HttpResponseFilter]] = Field( 1823 None, 1824 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1825 title="Response Filters", 1826 ) 1827 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, CustomBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1830class DefaultPaginator(BaseModel): 1831 type: Literal["DefaultPaginator"] 1832 pagination_strategy: Union[ 1833 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1834 ] = Field( 1835 ..., 1836 description="Strategy defining how records are paginated.", 1837 title="Pagination Strategy", 1838 ) 1839 page_size_option: Optional[RequestOption] = Field( 1840 None, title="Inject Page Size Into Outgoing HTTP Request" 1841 ) 1842 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1843 None, title="Inject Page Token Into Outgoing HTTP Request" 1844 ) 1845 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1848class SessionTokenRequestApiKeyAuthenticator(BaseModel): 1849 type: Literal["ApiKey"] 1850 inject_into: RequestOption = Field( 1851 ..., 1852 description="Configure how the API Key will be sent in requests to the source API.", 1853 examples=[ 1854 {"inject_into": "header", "field_name": "Authorization"}, 1855 {"inject_into": "request_parameter", "field_name": "authKey"}, 1856 ], 1857 title="Inject API Key Into Outgoing HTTP Request", 1858 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
1861class ListPartitionRouter(BaseModel): 1862 type: Literal["ListPartitionRouter"] 1863 cursor_field: str = Field( 1864 ..., 1865 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 1866 examples=["section", "{{ config['section_key'] }}"], 1867 title="Current Partition Value Identifier", 1868 ) 1869 values: Union[str, List[str]] = Field( 1870 ..., 1871 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 1872 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 1873 title="Partition Values", 1874 ) 1875 request_option: Optional[RequestOption] = Field( 1876 None, 1877 description="A request option describing where the list value should be injected into and under what field name if applicable.", 1878 title="Inject Partition Value Into Outgoing HTTP Request", 1879 ) 1880 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
1883class RecordSelector(BaseModel): 1884 type: Literal["RecordSelector"] 1885 extractor: Union[DpathExtractor, CustomRecordExtractor] 1886 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 1887 None, 1888 description="Responsible for filtering records to be emitted by the Source.", 1889 title="Record Filter", 1890 ) 1891 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 1892 None, 1893 description="Responsible for normalization according to the schema.", 1894 title="Schema Normalization", 1895 ) 1896 transform_before_filtering: Optional[bool] = Field( 1897 None, 1898 description="If true, transformation will be applied before record filtering.", 1899 title="Transform Before Filtering", 1900 ) 1901 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
1904class GzipDecoder(BaseModel): 1905 type: Literal["GzipDecoder"] 1906 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
Spec(pydantic.v1.main.BaseModel):
1909class Spec(BaseModel): 1910 type: Literal["Spec"] 1911 connection_specification: Dict[str, Any] = Field( 1912 ..., 1913 description="A connection specification describing how a the connector can be configured.", 1914 title="Connection Specification", 1915 ) 1916 documentation_url: Optional[str] = Field( 1917 None, 1918 description="URL of the connector's documentation page.", 1919 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 1920 title="Documentation URL", 1921 ) 1922 advanced_auth: Optional[AuthFlow] = Field( 1923 None, 1924 description="Advanced specification for configuring the authentication flow.", 1925 title="Advanced Auth", 1926 )
advanced_auth: Optional[AuthFlow]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
1929class RequestBodyGraphQL(BaseModel): 1930 type: Literal["RequestBodyGraphQL"] 1931 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
1934class CompositeErrorHandler(BaseModel): 1935 type: Literal["CompositeErrorHandler"] 1936 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 1937 ..., 1938 description="List of error handlers to iterate on to determine how to handle a failed response.", 1939 title="Error Handlers", 1940 ) 1941 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
1944class HTTPAPIBudget(BaseModel): 1945 class Config: 1946 extra = Extra.allow 1947 1948 type: Literal["HTTPAPIBudget"] 1949 policies: List[ 1950 Union[ 1951 FixedWindowCallRatePolicy, 1952 MovingWindowCallRatePolicy, 1953 UnlimitedCallRatePolicy, 1954 ] 1955 ] = Field( 1956 ..., 1957 description="List of call rate policies that define how many calls are allowed.", 1958 title="Policies", 1959 ) 1960 ratelimit_reset_header: Optional[str] = Field( 1961 "ratelimit-reset", 1962 description="The HTTP response header name that indicates when the rate limit resets.", 1963 title="Rate Limit Reset Header", 1964 ) 1965 ratelimit_remaining_header: Optional[str] = Field( 1966 "ratelimit-remaining", 1967 description="The HTTP response header name that indicates the number of remaining allowed calls.", 1968 title="Rate Limit Remaining Header", 1969 ) 1970 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 1971 [429], 1972 description="List of HTTP status codes that indicate a rate limit has been hit.", 1973 title="Status Codes for Rate Limit Hit", 1974 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
1977class ZipfileDecoder(BaseModel): 1978 class Config: 1979 extra = Extra.allow 1980 1981 type: Literal["ZipfileDecoder"] 1982 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 1983 ..., 1984 description="Parser to parse the decompressed data from the zipfile(s).", 1985 title="Parser", 1986 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
1989class DeclarativeSource1(BaseModel): 1990 class Config: 1991 extra = Extra.forbid 1992 1993 type: Literal["DeclarativeSource"] 1994 check: Union[CheckStream, CheckDynamicStream] 1995 streams: List[Union[DeclarativeStream, StateDelegatingStream]] 1996 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 1997 version: str = Field( 1998 ..., 1999 description="The version of the Airbyte CDK used to build and test the source.", 2000 ) 2001 schemas: Optional[Schemas] = None 2002 definitions: Optional[Dict[str, Any]] = None 2003 spec: Optional[Spec] = None 2004 concurrency_level: Optional[ConcurrencyLevel] = None 2005 api_budget: Optional[HTTPAPIBudget] = None 2006 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2007 None, 2008 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2009 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2010 title="Maximum Concurrent Asynchronous Jobs", 2011 ) 2012 metadata: Optional[Dict[str, Any]] = Field( 2013 None, 2014 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2015 ) 2016 description: Optional[str] = Field( 2017 None, 2018 description="A description of the connector. It will be presented on the Source documentation page.", 2019 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2022class DeclarativeSource2(BaseModel): 2023 class Config: 2024 extra = Extra.forbid 2025 2026 type: Literal["DeclarativeSource"] 2027 check: Union[CheckStream, CheckDynamicStream] 2028 streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None 2029 dynamic_streams: List[DynamicDeclarativeStream] 2030 version: str = Field( 2031 ..., 2032 description="The version of the Airbyte CDK used to build and test the source.", 2033 ) 2034 schemas: Optional[Schemas] = None 2035 definitions: Optional[Dict[str, Any]] = None 2036 spec: Optional[Spec] = None 2037 concurrency_level: Optional[ConcurrencyLevel] = None 2038 api_budget: Optional[HTTPAPIBudget] = None 2039 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2040 None, 2041 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2042 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2043 title="Maximum Concurrent Asynchronous Jobs", 2044 ) 2045 metadata: Optional[Dict[str, Any]] = Field( 2046 None, 2047 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2048 ) 2049 description: Optional[str] = Field( 2050 None, 2051 description="A description of the connector. It will be presented on the Source documentation page.", 2052 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2055class DeclarativeSource(BaseModel): 2056 class Config: 2057 extra = Extra.forbid 2058 2059 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2060 ..., 2061 description="An API source that extracts data according to its declarative components.", 2062 title="DeclarativeSource", 2063 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2066class SelectiveAuthenticator(BaseModel): 2067 class Config: 2068 extra = Extra.allow 2069 2070 type: Literal["SelectiveAuthenticator"] 2071 authenticator_selection_path: List[str] = Field( 2072 ..., 2073 description="Path of the field in config with selected authenticator name", 2074 examples=[["auth"], ["auth", "type"]], 2075 title="Authenticator Selection Path", 2076 ) 2077 authenticators: Dict[ 2078 str, 2079 Union[ 2080 ApiKeyAuthenticator, 2081 BasicHttpAuthenticator, 2082 BearerAuthenticator, 2083 CustomAuthenticator, 2084 OAuthAuthenticator, 2085 JwtAuthenticator, 2086 SessionTokenAuthenticator, 2087 NoAuth, 2088 LegacySessionTokenAuthenticator, 2089 ], 2090 ] = Field( 2091 ..., 2092 description="Authenticators to select from.", 2093 examples=[ 2094 { 2095 "authenticators": { 2096 "token": "#/definitions/ApiKeyAuthenticator", 2097 "oauth": "#/definitions/OAuthAuthenticator", 2098 "jwt": "#/definitions/JwtAuthenticator", 2099 } 2100 } 2101 ], 2102 title="Authenticators", 2103 ) 2104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, NoAuth, LegacySessionTokenAuthenticator]]
class
SelectiveAuthenticator.Config:
class
FileUploader(pydantic.v1.main.BaseModel):
2107class FileUploader(BaseModel): 2108 type: Literal["FileUploader"] 2109 requester: Union[HttpRequester, CustomRequester] = Field( 2110 ..., 2111 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2112 ) 2113 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2114 ..., 2115 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2116 ) 2117 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2118 None, 2119 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2120 ) 2121 filename_extractor: Optional[str] = Field( 2122 None, 2123 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2124 examples=[ 2125 "{{ record.id }}/{{ record.file_name }}/", 2126 "{{ record.id }}_{{ record.file_name }}/", 2127 ], 2128 ) 2129 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2132class DeclarativeStream(BaseModel): 2133 class Config: 2134 extra = Extra.allow 2135 2136 type: Literal["DeclarativeStream"] 2137 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2138 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2139 ..., 2140 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2141 title="Retriever", 2142 ) 2143 incremental_sync: Optional[ 2144 Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync] 2145 ] = Field( 2146 None, 2147 description="Component used to fetch data incrementally based on a time field in the data.", 2148 title="Incremental Sync", 2149 ) 2150 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2151 schema_loader: Optional[ 2152 Union[ 2153 InlineSchemaLoader, 2154 DynamicSchemaLoader, 2155 JsonFileSchemaLoader, 2156 CustomSchemaLoader, 2157 List[ 2158 Union[ 2159 InlineSchemaLoader, 2160 DynamicSchemaLoader, 2161 JsonFileSchemaLoader, 2162 CustomSchemaLoader, 2163 ] 2164 ], 2165 ] 2166 ] = Field( 2167 None, 2168 description="Component used to retrieve the schema for the current stream.", 2169 title="Schema Loader", 2170 ) 2171 transformations: Optional[ 2172 List[ 2173 Union[ 2174 AddFields, 2175 CustomTransformation, 2176 RemoveFields, 2177 KeysToLower, 2178 KeysToSnakeCase, 2179 FlattenFields, 2180 DpathFlattenFields, 2181 KeysReplace, 2182 ] 2183 ] 2184 ] = Field( 2185 None, 2186 description="A list of transformations to be applied to each output record.", 2187 title="Transformations", 2188 ) 2189 state_migrations: Optional[ 2190 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2191 ] = Field( 2192 [], 2193 description="Array of state migrations to be applied on the input state", 2194 title="State Migrations", 2195 ) 2196 file_uploader: Optional[FileUploader] = Field( 2197 None, 2198 description="(experimental) Describes how to fetch a file", 2199 title="File Uploader", 2200 ) 2201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], NoneType]
transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2204class SessionTokenAuthenticator(BaseModel): 2205 type: Literal["SessionTokenAuthenticator"] 2206 login_requester: HttpRequester = Field( 2207 ..., 2208 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2209 examples=[ 2210 { 2211 "type": "HttpRequester", 2212 "url_base": "https://my_api.com", 2213 "path": "/login", 2214 "authenticator": { 2215 "type": "BasicHttpAuthenticator", 2216 "username": "{{ config.username }}", 2217 "password": "{{ config.password }}", 2218 }, 2219 } 2220 ], 2221 title="Login Requester", 2222 ) 2223 session_token_path: List[str] = Field( 2224 ..., 2225 description="The path in the response body returned from the login requester to the session token.", 2226 examples=[["access_token"], ["result", "token"]], 2227 title="Session Token Path", 2228 ) 2229 expiration_duration: Optional[str] = Field( 2230 None, 2231 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", 2232 examples=["PT1H", "P1D"], 2233 title="Expiration Duration", 2234 ) 2235 request_authentication: Union[ 2236 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2237 ] = Field( 2238 ..., 2239 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2240 title="Data Request Authentication", 2241 ) 2242 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2243 None, description="Component used to decode the response.", title="Decoder" 2244 ) 2245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2248class HttpRequester(BaseModelWithDeprecations): 2249 type: Literal["HttpRequester"] 2250 url_base: Optional[str] = Field( 2251 None, 2252 deprecated=True, 2253 deprecation_message="Use `url` field instead.", 2254 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2255 examples=[ 2256 "https://connect.squareup.com/v2", 2257 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2258 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2259 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2260 ], 2261 title="API Base URL", 2262 ) 2263 url: Optional[str] = Field( 2264 None, 2265 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2266 examples=[ 2267 "https://connect.squareup.com/v2", 2268 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2269 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2270 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2271 ], 2272 title="API Endpoint URL", 2273 ) 2274 path: Optional[str] = Field( 2275 None, 2276 deprecated=True, 2277 deprecation_message="Use `url` field instead.", 2278 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2279 examples=[ 2280 "/products", 2281 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2282 "/trades/{{ config['symbol_id'] }}/history", 2283 ], 2284 title="URL Path", 2285 ) 2286 http_method: Optional[HttpMethod] = Field( 2287 HttpMethod.GET, 2288 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2289 examples=["GET", "POST"], 2290 title="HTTP Method", 2291 ) 2292 authenticator: Optional[ 2293 Union[ 2294 ApiKeyAuthenticator, 2295 BasicHttpAuthenticator, 2296 BearerAuthenticator, 2297 OAuthAuthenticator, 2298 JwtAuthenticator, 2299 SessionTokenAuthenticator, 2300 SelectiveAuthenticator, 2301 CustomAuthenticator, 2302 NoAuth, 2303 LegacySessionTokenAuthenticator, 2304 ] 2305 ] = Field( 2306 None, 2307 description="Authentication method to use for requests sent to the API.", 2308 title="Authenticator", 2309 ) 2310 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2311 None, 2312 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2313 title="Fetch Properties from Endpoint", 2314 ) 2315 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2316 None, 2317 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2318 examples=[ 2319 {"unit": "day"}, 2320 { 2321 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2322 }, 2323 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2324 {"sort_by[asc]": "updated_at"}, 2325 ], 2326 title="Query Parameters", 2327 ) 2328 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2329 None, 2330 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2331 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2332 title="Request Headers", 2333 ) 2334 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2335 None, 2336 deprecated=True, 2337 deprecation_message="Use `request_body` field instead.", 2338 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2339 examples=[ 2340 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2341 ], 2342 title="Request Body Payload (Non-JSON)", 2343 ) 2344 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2345 None, 2346 deprecated=True, 2347 deprecation_message="Use `request_body` field instead.", 2348 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2349 examples=[ 2350 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2351 {"key": "{{ config['value'] }}"}, 2352 {"sort": {"field": "updated_at", "order": "ascending"}}, 2353 ], 2354 title="Request Body JSON Payload", 2355 ) 2356 request_body: Optional[ 2357 Union[ 2358 RequestBodyPlainText, 2359 RequestBodyUrlEncodedForm, 2360 RequestBodyJsonObject, 2361 RequestBodyGraphQL, 2362 ] 2363 ] = Field( 2364 None, 2365 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2366 examples=[ 2367 { 2368 "type": "RequestBodyJsonObject", 2369 "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2370 }, 2371 { 2372 "type": "RequestBodyJsonObject", 2373 "value": {"key": "{{ config['value'] }}"}, 2374 }, 2375 { 2376 "type": "RequestBodyJsonObject", 2377 "value": {"sort": {"field": "updated_at", "order": "ascending"}}, 2378 }, 2379 {"type": "RequestBodyPlainText", "value": "plain_text_body"}, 2380 { 2381 "type": "RequestBodyUrlEncodedForm", 2382 "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, 2383 }, 2384 { 2385 "type": "RequestBodyGraphQL", 2386 "value": { 2387 "query": { 2388 "param1": "value1", 2389 "param2": "{{ config['param2_value'] }}", 2390 } 2391 }, 2392 }, 2393 ], 2394 title="Request Body", 2395 ) 2396 error_handler: Optional[ 2397 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2398 ] = Field( 2399 None, 2400 description="Error handler component that defines how to handle errors.", 2401 title="Error Handler", 2402 ) 2403 use_cache: Optional[bool] = Field( 2404 False, 2405 description="Enables stream requests caching. This field is automatically set by the CDK.", 2406 title="Use Cache", 2407 ) 2408 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2411class DynamicSchemaLoader(BaseModel): 2412 type: Literal["DynamicSchemaLoader"] 2413 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2414 ..., 2415 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2416 title="Retriever", 2417 ) 2418 schema_transformations: Optional[ 2419 List[ 2420 Union[ 2421 AddFields, 2422 CustomTransformation, 2423 RemoveFields, 2424 KeysToLower, 2425 KeysToSnakeCase, 2426 FlattenFields, 2427 DpathFlattenFields, 2428 KeysReplace, 2429 ] 2430 ] 2431 ] = Field( 2432 None, 2433 description="A list of transformations to be applied to the schema.", 2434 title="Schema Transformations", 2435 ) 2436 schema_type_identifier: SchemaTypeIdentifier 2437 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
schema_transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2440class ParentStreamConfig(BaseModel): 2441 type: Literal["ParentStreamConfig"] 2442 lazy_read_pointer: Optional[List[str]] = Field( 2443 [], 2444 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2445 title="Lazy Read Pointer", 2446 ) 2447 parent_key: str = Field( 2448 ..., 2449 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2450 examples=["id", "{{ config['parent_record_id'] }}"], 2451 title="Parent Key", 2452 ) 2453 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2454 ..., description="Reference to the parent stream.", title="Parent Stream" 2455 ) 2456 partition_field: str = Field( 2457 ..., 2458 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2459 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2460 title="Current Parent Key Value Identifier", 2461 ) 2462 request_option: Optional[RequestOption] = Field( 2463 None, 2464 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2465 title="Request Option", 2466 ) 2467 incremental_dependency: Optional[bool] = Field( 2468 False, 2469 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2470 title="Incremental Dependency", 2471 ) 2472 extra_fields: Optional[List[List[str]]] = Field( 2473 None, 2474 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2475 title="Extra Fields", 2476 ) 2477 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2480class PropertiesFromEndpoint(BaseModel): 2481 type: Literal["PropertiesFromEndpoint"] 2482 property_field_path: List[str] = Field( 2483 ..., 2484 description="Describes the path to the field that should be extracted", 2485 examples=[["name"]], 2486 ) 2487 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2488 ..., 2489 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2490 ) 2491 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2494class QueryProperties(BaseModel): 2495 type: Literal["QueryProperties"] 2496 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2497 ..., 2498 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2499 title="Property List", 2500 ) 2501 always_include_properties: Optional[List[str]] = Field( 2502 None, 2503 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2504 title="Always Include Properties", 2505 ) 2506 property_chunking: Optional[PropertyChunking] = Field( 2507 None, 2508 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2509 title="Property Chunking", 2510 ) 2511 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2514class StateDelegatingStream(BaseModel): 2515 type: Literal["StateDelegatingStream"] 2516 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2517 full_refresh_stream: DeclarativeStream = Field( 2518 ..., 2519 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2520 title="Full Refresh Stream", 2521 ) 2522 incremental_stream: DeclarativeStream = Field( 2523 ..., 2524 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2525 title="Incremental Stream", 2526 ) 2527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2530class SimpleRetriever(BaseModel): 2531 type: Literal["SimpleRetriever"] 2532 requester: Union[HttpRequester, CustomRequester] = Field( 2533 ..., 2534 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2535 ) 2536 decoder: Optional[ 2537 Union[ 2538 JsonDecoder, 2539 XmlDecoder, 2540 CsvDecoder, 2541 JsonlDecoder, 2542 GzipDecoder, 2543 IterableDecoder, 2544 ZipfileDecoder, 2545 CustomDecoder, 2546 ] 2547 ] = Field( 2548 None, 2549 description="Component decoding the response so records can be extracted.", 2550 title="HTTP Response Format", 2551 ) 2552 record_selector: RecordSelector = Field( 2553 ..., 2554 description="Component that describes how to extract records from a HTTP response.", 2555 ) 2556 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2557 None, 2558 description="Paginator component that describes how to navigate through the API's pages.", 2559 ) 2560 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2561 False, 2562 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2563 ) 2564 partition_router: Optional[ 2565 Union[ 2566 ListPartitionRouter, 2567 SubstreamPartitionRouter, 2568 GroupingPartitionRouter, 2569 CustomPartitionRouter, 2570 List[ 2571 Union[ 2572 ListPartitionRouter, 2573 SubstreamPartitionRouter, 2574 GroupingPartitionRouter, 2575 CustomPartitionRouter, 2576 ] 2577 ], 2578 ] 2579 ] = Field( 2580 [], 2581 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2582 title="Partition Router", 2583 ) 2584 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2587class AsyncRetriever(BaseModel): 2588 type: Literal["AsyncRetriever"] 2589 record_selector: RecordSelector = Field( 2590 ..., 2591 description="Component that describes how to extract records from a HTTP response.", 2592 ) 2593 status_mapping: AsyncJobStatusMap = Field( 2594 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2595 ) 2596 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2597 ..., description="Responsible for fetching the actual status of the async job." 2598 ) 2599 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2600 ..., 2601 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2602 ) 2603 download_extractor: Optional[ 2604 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2605 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2606 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2607 ..., 2608 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2609 ) 2610 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2611 ..., 2612 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2613 ) 2614 polling_job_timeout: Optional[Union[int, str]] = Field( 2615 None, 2616 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2617 ) 2618 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2619 None, 2620 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2621 ) 2622 download_requester: Union[HttpRequester, CustomRequester] = Field( 2623 ..., 2624 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2625 ) 2626 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2627 None, 2628 description="Paginator component that describes how to navigate through the API's pages during download.", 2629 ) 2630 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2631 None, 2632 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2633 ) 2634 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2635 None, 2636 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2637 ) 2638 partition_router: Optional[ 2639 Union[ 2640 ListPartitionRouter, 2641 SubstreamPartitionRouter, 2642 GroupingPartitionRouter, 2643 CustomPartitionRouter, 2644 List[ 2645 Union[ 2646 ListPartitionRouter, 2647 SubstreamPartitionRouter, 2648 GroupingPartitionRouter, 2649 CustomPartitionRouter, 2650 ] 2651 ], 2652 ] 2653 ] = Field( 2654 [], 2655 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2656 title="Partition Router", 2657 ) 2658 decoder: Optional[ 2659 Union[ 2660 CsvDecoder, 2661 GzipDecoder, 2662 JsonDecoder, 2663 JsonlDecoder, 2664 IterableDecoder, 2665 XmlDecoder, 2666 ZipfileDecoder, 2667 CustomDecoder, 2668 ] 2669 ] = Field( 2670 None, 2671 description="Component decoding the response so records can be extracted.", 2672 title="Decoder", 2673 ) 2674 download_decoder: Optional[ 2675 Union[ 2676 CsvDecoder, 2677 GzipDecoder, 2678 JsonDecoder, 2679 JsonlDecoder, 2680 IterableDecoder, 2681 XmlDecoder, 2682 ZipfileDecoder, 2683 CustomDecoder, 2684 ] 2685 ] = Field( 2686 None, 2687 description="Component decoding the download response so records can be extracted.", 2688 title="Download Decoder", 2689 ) 2690 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[ListPartitionRouter, SubstreamPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2693class SubstreamPartitionRouter(BaseModel): 2694 type: Literal["SubstreamPartitionRouter"] 2695 parent_stream_configs: List[ParentStreamConfig] = Field( 2696 ..., 2697 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2698 title="Parent Stream Configs", 2699 ) 2700 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2703class GroupingPartitionRouter(BaseModel): 2704 type: Literal["GroupingPartitionRouter"] 2705 group_size: int = Field( 2706 ..., 2707 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2708 examples=[10, 50], 2709 title="Group Size", 2710 ) 2711 underlying_partition_router: Union[ 2712 CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter 2713 ] = Field( 2714 ..., 2715 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2716 title="Underlying Partition Router", 2717 ) 2718 deduplicate: Optional[bool] = Field( 2719 True, 2720 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2721 title="Deduplicate Partitions", 2722 ) 2723 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2726class HttpComponentsResolver(BaseModel): 2727 type: Literal["HttpComponentsResolver"] 2728 retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field( 2729 ..., 2730 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2731 title="Retriever", 2732 ) 2733 components_mapping: List[ComponentMappingDefinition] 2734 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
2737class DynamicDeclarativeStream(BaseModel): 2738 type: Literal["DynamicDeclarativeStream"] 2739 name: Optional[str] = Field( 2740 "", description="The dynamic stream name.", example=["Tables"], title="Name" 2741 ) 2742 stream_template: DeclarativeStream = Field( 2743 ..., description="Reference to the stream template.", title="Stream Template" 2744 ) 2745 components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field( 2746 ..., 2747 description="Component resolve and populates stream templates with components values.", 2748 title="Components Resolver", 2749 )
stream_template: DeclarativeStream
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver]