airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 0, 62 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 63 title="Stream Count", 64 ) 65 66 67class CheckDynamicStream(BaseModel): 68 type: Literal["CheckDynamicStream"] 69 stream_count: int = Field( 70 ..., 71 description="Numbers of the streams to try reading from when running a check operation.", 72 title="Stream Count", 73 ) 74 use_check_availability: Optional[bool] = Field( 75 True, 76 description="Enables stream check availability. This field is automatically set by the CDK.", 77 title="Use Check Availability", 78 ) 79 80 81class ConcurrencyLevel(BaseModel): 82 type: Optional[Literal["ConcurrencyLevel"]] = None 83 default_concurrency: Union[int, str] = Field( 84 ..., 85 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 86 examples=[10, "{{ config['num_workers'] or 10 }}"], 87 title="Default Concurrency", 88 ) 89 max_concurrency: Optional[int] = Field( 90 None, 91 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 92 examples=[20, 100], 93 title="Max Concurrency", 94 ) 95 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 96 97 98class ConstantBackoffStrategy(BaseModel): 99 type: Literal["ConstantBackoffStrategy"] 100 backoff_time_in_seconds: Union[float, str] = Field( 101 ..., 102 description="Backoff time in seconds.", 103 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 104 title="Backoff Time", 105 ) 106 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 107 108 109class CursorPagination(BaseModel): 110 type: Literal["CursorPagination"] 111 cursor_value: str = Field( 112 ..., 113 description="Value of the cursor defining the next page to fetch.", 114 examples=[ 115 "{{ headers.link.next.cursor }}", 116 "{{ last_record['key'] }}", 117 "{{ response['nextPage'] }}", 118 ], 119 title="Cursor Value", 120 ) 121 page_size: Optional[Union[int, str]] = Field( 122 None, 123 description="The number of records to include in each pages.", 124 examples=[100, "{{ config['page_size'] }}"], 125 title="Page Size", 126 ) 127 stop_condition: Optional[str] = Field( 128 None, 129 description="Template string evaluating when to stop paginating.", 130 examples=[ 131 "{{ response.data.has_more is false }}", 132 "{{ 'next' not in headers['link'] }}", 133 ], 134 title="Stop Condition", 135 ) 136 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 137 138 139class CustomAuthenticator(BaseModel): 140 class Config: 141 extra = Extra.allow 142 143 type: Literal["CustomAuthenticator"] 144 class_name: str = Field( 145 ..., 146 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 147 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 148 title="Class Name", 149 ) 150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 151 152 153class CustomBackoffStrategy(BaseModel): 154 class Config: 155 extra = Extra.allow 156 157 type: Literal["CustomBackoffStrategy"] 158 class_name: str = Field( 159 ..., 160 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 161 examples=["source_railz.components.MyCustomBackoffStrategy"], 162 title="Class Name", 163 ) 164 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 165 166 167class CustomErrorHandler(BaseModel): 168 class Config: 169 extra = Extra.allow 170 171 type: Literal["CustomErrorHandler"] 172 class_name: str = Field( 173 ..., 174 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 175 examples=["source_railz.components.MyCustomErrorHandler"], 176 title="Class Name", 177 ) 178 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 179 180 181class CustomPaginationStrategy(BaseModel): 182 class Config: 183 extra = Extra.allow 184 185 type: Literal["CustomPaginationStrategy"] 186 class_name: str = Field( 187 ..., 188 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 189 examples=["source_railz.components.MyCustomPaginationStrategy"], 190 title="Class Name", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomRecordExtractor(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomRecordExtractor"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomRecordExtractor"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordFilter(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordFilter"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomCustomRecordFilter"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRequester(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRequester"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomRecordExtractor"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRetriever(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRetriever"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRetriever"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomPartitionRouter(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomPartitionRouter"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomPartitionRouter"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomSchemaLoader(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomSchemaLoader"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomSchemaLoader"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaNormalization(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaNormalization"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 287 examples=[ 288 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 289 ], 290 title="Class Name", 291 ) 292 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 293 294 295class CustomStateMigration(BaseModel): 296 class Config: 297 extra = Extra.allow 298 299 type: Literal["CustomStateMigration"] 300 class_name: str = Field( 301 ..., 302 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 303 examples=["source_railz.components.MyCustomStateMigration"], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomTransformation(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomTransformation"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomTransformation"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class LegacyToPerPartitionStateMigration(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 328 329 330class Clamping(BaseModel): 331 target: str = Field( 332 ..., 333 description="The period of time that datetime windows will be clamped by", 334 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 335 title="Target", 336 ) 337 target_details: Optional[Dict[str, Any]] = None 338 339 340class Algorithm(Enum): 341 HS256 = "HS256" 342 HS384 = "HS384" 343 HS512 = "HS512" 344 ES256 = "ES256" 345 ES256K = "ES256K" 346 ES384 = "ES384" 347 ES512 = "ES512" 348 RS256 = "RS256" 349 RS384 = "RS384" 350 RS512 = "RS512" 351 PS256 = "PS256" 352 PS384 = "PS384" 353 PS512 = "PS512" 354 EdDSA = "EdDSA" 355 356 357class JwtHeaders(BaseModel): 358 class Config: 359 extra = Extra.forbid 360 361 kid: Optional[str] = Field( 362 None, 363 description="Private key ID for user account.", 364 examples=["{{ config['kid'] }}"], 365 title="Key Identifier", 366 ) 367 typ: Optional[str] = Field( 368 "JWT", 369 description="The media type of the complete JWT.", 370 examples=["JWT"], 371 title="Type", 372 ) 373 cty: Optional[str] = Field( 374 None, 375 description="Content type of JWT header.", 376 examples=["JWT"], 377 title="Content Type", 378 ) 379 380 381class JwtPayload(BaseModel): 382 class Config: 383 extra = Extra.forbid 384 385 iss: Optional[str] = Field( 386 None, 387 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 388 examples=["{{ config['iss'] }}"], 389 title="Issuer", 390 ) 391 sub: Optional[str] = Field( 392 None, 393 description="The subject of the JWT. Commonly defined by the API.", 394 title="Subject", 395 ) 396 aud: Optional[str] = Field( 397 None, 398 description="The recipient that the JWT is intended for. Commonly defined by the API.", 399 examples=["appstoreconnect-v1"], 400 title="Audience", 401 ) 402 403 404class RefreshTokenUpdater(BaseModel): 405 refresh_token_name: Optional[str] = Field( 406 "refresh_token", 407 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 408 examples=["refresh_token"], 409 title="Refresh Token Property Name", 410 ) 411 access_token_config_path: Optional[List[str]] = Field( 412 ["credentials", "access_token"], 413 description="Config path to the access token. Make sure the field actually exists in the config.", 414 examples=[["credentials", "access_token"], ["access_token"]], 415 title="Config Path To Access Token", 416 ) 417 refresh_token_config_path: Optional[List[str]] = Field( 418 ["credentials", "refresh_token"], 419 description="Config path to the access token. Make sure the field actually exists in the config.", 420 examples=[["credentials", "refresh_token"], ["refresh_token"]], 421 title="Config Path To Refresh Token", 422 ) 423 token_expiry_date_config_path: Optional[List[str]] = Field( 424 ["credentials", "token_expiry_date"], 425 description="Config path to the expiry date. Make sure actually exists in the config.", 426 examples=[["credentials", "token_expiry_date"]], 427 title="Config Path To Expiry Date", 428 ) 429 refresh_token_error_status_codes: Optional[List[int]] = Field( 430 [], 431 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 432 examples=[[400, 500]], 433 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 434 ) 435 refresh_token_error_key: Optional[str] = Field( 436 "", 437 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 438 examples=["error"], 439 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 440 ) 441 refresh_token_error_values: Optional[List[str]] = Field( 442 [], 443 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 444 examples=[["invalid_grant", "invalid_permissions"]], 445 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 446 ) 447 448 449class Rate(BaseModel): 450 class Config: 451 extra = Extra.allow 452 453 limit: Union[int, str] = Field( 454 ..., 455 description="The maximum number of calls allowed within the interval.", 456 title="Limit", 457 ) 458 interval: str = Field( 459 ..., 460 description="The time interval for the rate limit.", 461 examples=["PT1H", "P1D"], 462 title="Interval", 463 ) 464 465 466class HttpRequestRegexMatcher(BaseModel): 467 class Config: 468 extra = Extra.allow 469 470 method: Optional[str] = Field( 471 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 472 ) 473 url_base: Optional[str] = Field( 474 None, 475 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 476 title="URL Base", 477 ) 478 url_path_pattern: Optional[str] = Field( 479 None, 480 description="A regular expression pattern to match the URL path.", 481 title="URL Path Pattern", 482 ) 483 params: Optional[Dict[str, Any]] = Field( 484 None, description="The query parameters to match.", title="Parameters" 485 ) 486 headers: Optional[Dict[str, Any]] = Field( 487 None, description="The headers to match.", title="Headers" 488 ) 489 weight: Optional[Union[int, str]] = Field( 490 None, 491 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 492 title="Weight", 493 ) 494 495 496class DpathExtractor(BaseModel): 497 type: Literal["DpathExtractor"] 498 field_path: List[str] = Field( 499 ..., 500 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 501 examples=[ 502 ["data"], 503 ["data", "records"], 504 ["data", "{{ parameters.name }}"], 505 ["data", "*", "record"], 506 ], 507 title="Field Path", 508 ) 509 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 510 511 512class ResponseToFileExtractor(BaseModel): 513 type: Literal["ResponseToFileExtractor"] 514 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 515 516 517class ExponentialBackoffStrategy(BaseModel): 518 type: Literal["ExponentialBackoffStrategy"] 519 factor: Optional[Union[float, str]] = Field( 520 5, 521 description="Multiplicative constant applied on each retry.", 522 examples=[5, 5.5, "10"], 523 title="Factor", 524 ) 525 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 526 527 528class GroupByKeyMergeStrategy(BaseModel): 529 type: Literal["GroupByKeyMergeStrategy"] 530 key: Union[str, List[str]] = Field( 531 ..., 532 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 533 examples=["id", ["parent_id", "end_date"]], 534 title="Key", 535 ) 536 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 537 538 539class SessionTokenRequestBearerAuthenticator(BaseModel): 540 type: Literal["Bearer"] 541 542 543class HttpMethod(Enum): 544 GET = "GET" 545 POST = "POST" 546 547 548class Action(Enum): 549 SUCCESS = "SUCCESS" 550 FAIL = "FAIL" 551 RETRY = "RETRY" 552 IGNORE = "IGNORE" 553 RESET_PAGINATION = "RESET_PAGINATION" 554 RATE_LIMITED = "RATE_LIMITED" 555 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 556 557 558class FailureType(Enum): 559 system_error = "system_error" 560 config_error = "config_error" 561 transient_error = "transient_error" 562 563 564class HttpResponseFilter(BaseModel): 565 type: Literal["HttpResponseFilter"] 566 action: Optional[Action] = Field( 567 None, 568 description="Action to execute if a response matches the filter.", 569 examples=[ 570 "SUCCESS", 571 "FAIL", 572 "RETRY", 573 "IGNORE", 574 "RESET_PAGINATION", 575 "RATE_LIMITED", 576 "REFRESH_TOKEN_THEN_RETRY", 577 ], 578 title="Action", 579 ) 580 failure_type: Optional[FailureType] = Field( 581 None, 582 description="Failure type of traced exception if a response matches the filter.", 583 examples=["system_error", "config_error", "transient_error"], 584 title="Failure Type", 585 ) 586 error_message: Optional[str] = Field( 587 None, 588 description="Error Message to display if the response matches the filter.", 589 title="Error Message", 590 ) 591 error_message_contains: Optional[str] = Field( 592 None, 593 description="Match the response if its error message contains the substring.", 594 example=["This API operation is not enabled for this site"], 595 title="Error Message Substring", 596 ) 597 http_codes: Optional[List[int]] = Field( 598 None, 599 description="Match the response if its HTTP code is included in this list.", 600 examples=[[420, 429], [500]], 601 title="HTTP Codes", 602 unique_items=True, 603 ) 604 predicate: Optional[str] = Field( 605 None, 606 description="Match the response if the predicate evaluates to true.", 607 examples=[ 608 "{{ 'Too much requests' in response }}", 609 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 610 ], 611 title="Predicate", 612 ) 613 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 614 615 616class ComplexFieldType(BaseModel): 617 field_type: str 618 items: Optional[Union[str, ComplexFieldType]] = None 619 620 621class TypesMap(BaseModel): 622 target_type: Union[str, List[str], ComplexFieldType] 623 current_type: Union[str, List[str]] 624 condition: Optional[str] = None 625 626 627class SchemaTypeIdentifier(BaseModel): 628 type: Optional[Literal["SchemaTypeIdentifier"]] = None 629 schema_pointer: Optional[List[str]] = Field( 630 [], 631 description="List of nested fields defining the schema field path to extract. Defaults to [].", 632 title="Schema Path", 633 ) 634 key_pointer: List[str] = Field( 635 ..., 636 description="List of potentially nested fields describing the full path of the field key to extract.", 637 title="Key Path", 638 ) 639 type_pointer: Optional[List[str]] = Field( 640 None, 641 description="List of potentially nested fields describing the full path of the field type to extract.", 642 title="Type Path", 643 ) 644 types_mapping: Optional[List[TypesMap]] = None 645 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 646 647 648class InlineSchemaLoader(BaseModel): 649 type: Literal["InlineSchemaLoader"] 650 schema_: Optional[Dict[str, Any]] = Field( 651 None, 652 alias="schema", 653 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 654 title="Schema", 655 ) 656 657 658class JsonFileSchemaLoader(BaseModel): 659 type: Literal["JsonFileSchemaLoader"] 660 file_path: Optional[str] = Field( 661 None, 662 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 663 example=["./schemas/users.json"], 664 title="File Path", 665 ) 666 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 667 668 669class JsonDecoder(BaseModel): 670 type: Literal["JsonDecoder"] 671 672 673class JsonlDecoder(BaseModel): 674 type: Literal["JsonlDecoder"] 675 676 677class KeysToLower(BaseModel): 678 type: Literal["KeysToLower"] 679 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 680 681 682class KeysToSnakeCase(BaseModel): 683 type: Literal["KeysToSnakeCase"] 684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 685 686 687class FlattenFields(BaseModel): 688 type: Literal["FlattenFields"] 689 flatten_lists: Optional[bool] = Field( 690 True, 691 description="Whether to flatten lists or leave it as is. Default is True.", 692 title="Flatten Lists", 693 ) 694 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 695 696 697class KeyTransformation(BaseModel): 698 type: Literal["KeyTransformation"] 699 prefix: Optional[str] = Field( 700 None, 701 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 702 examples=["flattened_"], 703 title="Key Prefix", 704 ) 705 suffix: Optional[str] = Field( 706 None, 707 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 708 examples=["_flattened"], 709 title="Key Suffix", 710 ) 711 712 713class DpathFlattenFields(BaseModel): 714 type: Literal["DpathFlattenFields"] 715 field_path: List[str] = Field( 716 ..., 717 description="A path to field that needs to be flattened.", 718 examples=[["data"], ["data", "*", "field"]], 719 title="Field Path", 720 ) 721 delete_origin_value: Optional[bool] = Field( 722 None, 723 description="Whether to delete the origin value or keep it. Default is False.", 724 title="Delete Origin Value", 725 ) 726 replace_record: Optional[bool] = Field( 727 None, 728 description="Whether to replace the origin record or not. Default is False.", 729 title="Replace Origin Record", 730 ) 731 key_transformation: Optional[KeyTransformation] = Field( 732 None, 733 description="Transformation for object keys. If not provided, original key will be used.", 734 title="Key transformation", 735 ) 736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 737 738 739class KeysReplace(BaseModel): 740 type: Literal["KeysReplace"] 741 old: str = Field( 742 ..., 743 description="Old value to replace.", 744 examples=[ 745 " ", 746 "{{ record.id }}", 747 "{{ config['id'] }}", 748 "{{ stream_slice['id'] }}", 749 ], 750 title="Old value", 751 ) 752 new: str = Field( 753 ..., 754 description="New value to set.", 755 examples=[ 756 "_", 757 "{{ record.id }}", 758 "{{ config['id'] }}", 759 "{{ stream_slice['id'] }}", 760 ], 761 title="New value", 762 ) 763 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 764 765 766class IterableDecoder(BaseModel): 767 type: Literal["IterableDecoder"] 768 769 770class XmlDecoder(BaseModel): 771 type: Literal["XmlDecoder"] 772 773 774class CustomDecoder(BaseModel): 775 class Config: 776 extra = Extra.allow 777 778 type: Literal["CustomDecoder"] 779 class_name: str = Field( 780 ..., 781 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 782 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 783 title="Class Name", 784 ) 785 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 786 787 788class MinMaxDatetime(BaseModel): 789 type: Literal["MinMaxDatetime"] 790 datetime: str = Field( 791 ..., 792 description="Datetime value.", 793 examples=[ 794 "2021-01-01", 795 "2021-01-01T00:00:00Z", 796 "{{ config['start_time'] }}", 797 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 798 ], 799 title="Datetime", 800 ) 801 datetime_format: Optional[str] = Field( 802 "", 803 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 804 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 805 title="Datetime Format", 806 ) 807 max_datetime: Optional[str] = Field( 808 None, 809 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 810 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 811 title="Max Datetime", 812 ) 813 min_datetime: Optional[str] = Field( 814 None, 815 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 816 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 817 title="Min Datetime", 818 ) 819 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 820 821 822class NoAuth(BaseModel): 823 type: Literal["NoAuth"] 824 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 825 826 827class NoPagination(BaseModel): 828 type: Literal["NoPagination"] 829 830 831class State(BaseModel): 832 class Config: 833 extra = Extra.allow 834 835 min: int 836 max: int 837 838 839class OAuthScope(BaseModel): 840 class Config: 841 extra = Extra.allow 842 843 scope: str = Field( 844 ..., 845 description="The OAuth scope string to request from the provider.", 846 ) 847 848 849class OauthConnectorInputSpecification(BaseModel): 850 class Config: 851 extra = Extra.allow 852 853 consent_url: str = Field( 854 ..., 855 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 856 examples=[ 857 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 858 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 859 ], 860 title="Consent URL", 861 ) 862 scope: Optional[str] = Field( 863 None, 864 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 865 examples=["user:read user:read_orders workspaces:read"], 866 title="Scopes", 867 ) 868 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 869 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 870 # The CDK schema defines the manifest contract; the platform reads these fields 871 # during the OAuth consent flow to build the authorization URL. 872 scopes: Optional[List[OAuthScope]] = Field( 873 None, 874 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 875 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 876 title="Scopes", 877 ) 878 optional_scopes: Optional[List[OAuthScope]] = Field( 879 None, 880 description="Optional OAuth scope objects that may or may not be granted.", 881 examples=[[{"scope": "admin:read"}]], 882 title="Optional Scopes", 883 ) 884 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 885 ScopesJoinStrategy.space, 886 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 887 title="Scopes Join Strategy", 888 ) 889 access_token_url: str = Field( 890 ..., 891 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 892 examples=[ 893 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 894 ], 895 title="Access Token URL", 896 ) 897 access_token_headers: Optional[Dict[str, Any]] = Field( 898 None, 899 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 900 examples=[ 901 { 902 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 903 } 904 ], 905 title="Access Token Headers", 906 ) 907 access_token_params: Optional[Dict[str, Any]] = Field( 908 None, 909 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 910 examples=[ 911 { 912 "{{ auth_code_key }}": "{{ auth_code_value }}", 913 "{{ client_id_key }}": "{{ client_id_value }}", 914 "{{ client_secret_key }}": "{{ client_secret_value }}", 915 } 916 ], 917 title="Access Token Query Params (Json Encoded)", 918 ) 919 extract_output: Optional[List[str]] = Field( 920 None, 921 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 922 examples=[["access_token", "refresh_token", "other_field"]], 923 title="Extract Output", 924 ) 925 state: Optional[State] = Field( 926 None, 927 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 928 examples=[{"min": 7, "max": 128}], 929 title="Configurable State Query Param", 930 ) 931 client_id_key: Optional[str] = Field( 932 None, 933 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 934 examples=["my_custom_client_id_key_name"], 935 title="Client ID Key Override", 936 ) 937 client_secret_key: Optional[str] = Field( 938 None, 939 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 940 examples=["my_custom_client_secret_key_name"], 941 title="Client Secret Key Override", 942 ) 943 scope_key: Optional[str] = Field( 944 None, 945 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 946 examples=["my_custom_scope_key_key_name"], 947 title="Scopes Key Override", 948 ) 949 state_key: Optional[str] = Field( 950 None, 951 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 952 examples=["my_custom_state_key_key_name"], 953 title="State Key Override", 954 ) 955 auth_code_key: Optional[str] = Field( 956 None, 957 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 958 examples=["my_custom_auth_code_key_name"], 959 title="Auth Code Key Override", 960 ) 961 redirect_uri_key: Optional[str] = Field( 962 None, 963 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 964 examples=["my_custom_redirect_uri_key_name"], 965 title="Redirect URI Key Override", 966 ) 967 968 969class OAuthConfigSpecification(BaseModel): 970 class Config: 971 extra = Extra.allow 972 973 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 974 None, 975 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 976 examples=[ 977 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 978 { 979 "app_id": { 980 "type": "string", 981 "path_in_connector_config": ["info", "app_id"], 982 } 983 }, 984 ], 985 title="OAuth user input", 986 ) 987 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 988 None, 989 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 990 title="DeclarativeOAuth Connector Specification", 991 ) 992 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 993 None, 994 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 995 examples=[ 996 { 997 "refresh_token": { 998 "type": "string,", 999 "path_in_connector_config": ["credentials", "refresh_token"], 1000 } 1001 } 1002 ], 1003 title="OAuth output specification", 1004 ) 1005 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1006 None, 1007 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1008 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1009 title="OAuth input specification", 1010 ) 1011 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1012 None, 1013 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1014 examples=[ 1015 { 1016 "client_id": { 1017 "type": "string,", 1018 "path_in_connector_config": ["credentials", "client_id"], 1019 }, 1020 "client_secret": { 1021 "type": "string,", 1022 "path_in_connector_config": ["credentials", "client_secret"], 1023 }, 1024 } 1025 ], 1026 title="OAuth server output specification", 1027 ) 1028 1029 1030class OffsetIncrement(BaseModel): 1031 type: Literal["OffsetIncrement"] 1032 page_size: Optional[Union[int, str]] = Field( 1033 None, 1034 description="The number of records to include in each pages.", 1035 examples=[100, "{{ config['page_size'] }}"], 1036 title="Limit", 1037 ) 1038 inject_on_first_request: Optional[bool] = Field( 1039 False, 1040 description="Using the `offset` with value `0` during the first request", 1041 title="Inject Offset on First Request", 1042 ) 1043 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1044 1045 1046class PageIncrement(BaseModel): 1047 type: Literal["PageIncrement"] 1048 page_size: Optional[Union[int, str]] = Field( 1049 None, 1050 description="The number of records to include in each pages.", 1051 examples=[100, "100", "{{ config['page_size'] }}"], 1052 title="Page Size", 1053 ) 1054 start_from_page: Optional[int] = Field( 1055 0, 1056 description="Index of the first page to request.", 1057 examples=[0, 1], 1058 title="Start From Page", 1059 ) 1060 inject_on_first_request: Optional[bool] = Field( 1061 False, 1062 description="Using the `page number` with value defined by `start_from_page` during the first request", 1063 title="Inject Page Number on First Request", 1064 ) 1065 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1066 1067 1068class PrimaryKey(BaseModel): 1069 __root__: Union[str, List[str], List[List[str]]] = Field( 1070 ..., 1071 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1072 examples=["id", ["code", "type"]], 1073 title="Primary Key", 1074 ) 1075 1076 1077class PropertyLimitType(Enum): 1078 characters = "characters" 1079 property_count = "property_count" 1080 1081 1082class PropertyChunking(BaseModel): 1083 type: Literal["PropertyChunking"] 1084 property_limit_type: PropertyLimitType = Field( 1085 ..., 1086 description="The type used to determine the maximum number of properties per chunk", 1087 title="Property Limit Type", 1088 ) 1089 property_limit: Optional[int] = Field( 1090 None, 1091 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1092 title="Property Limit", 1093 ) 1094 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1095 None, 1096 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1097 title="Record Merge Strategy", 1098 ) 1099 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1100 1101 1102class RecordFilter(BaseModel): 1103 type: Literal["RecordFilter"] 1104 condition: Optional[str] = Field( 1105 "", 1106 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1107 examples=[ 1108 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1109 "{{ record.status in ['active', 'expired'] }}", 1110 ], 1111 title="Condition", 1112 ) 1113 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1114 1115 1116class SchemaNormalization(Enum): 1117 Default = "Default" 1118 None_ = "None" 1119 1120 1121class RemoveFields(BaseModel): 1122 type: Literal["RemoveFields"] 1123 condition: Optional[str] = Field( 1124 "", 1125 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1126 examples=[ 1127 "{{ property|string == '' }}", 1128 "{{ property is integer }}", 1129 "{{ property|length > 5 }}", 1130 "{{ property == 'some_string_to_match' }}", 1131 ], 1132 ) 1133 field_pointers: List[List[str]] = Field( 1134 ..., 1135 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1136 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1137 title="Field Paths", 1138 ) 1139 1140 1141class RequestPath(BaseModel): 1142 type: Literal["RequestPath"] 1143 1144 1145class InjectInto(Enum): 1146 request_parameter = "request_parameter" 1147 header = "header" 1148 body_data = "body_data" 1149 body_json = "body_json" 1150 1151 1152class RequestOption(BaseModel): 1153 type: Literal["RequestOption"] 1154 inject_into: InjectInto = Field( 1155 ..., 1156 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1157 examples=["request_parameter", "header", "body_data", "body_json"], 1158 title="Inject Into", 1159 ) 1160 field_name: Optional[str] = Field( 1161 None, 1162 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1163 examples=["segment_id"], 1164 title="Field Name", 1165 ) 1166 field_path: Optional[List[str]] = Field( 1167 None, 1168 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1169 examples=[["data", "viewer", "id"]], 1170 title="Field Path", 1171 ) 1172 1173 1174class Schemas(BaseModel): 1175 pass 1176 1177 class Config: 1178 extra = Extra.allow 1179 1180 1181class LegacySessionTokenAuthenticator(BaseModel): 1182 type: Literal["LegacySessionTokenAuthenticator"] 1183 header: str = Field( 1184 ..., 1185 description="The name of the session token header that will be injected in the request", 1186 examples=["X-Session"], 1187 title="Session Request Header", 1188 ) 1189 login_url: str = Field( 1190 ..., 1191 description="Path of the login URL (do not include the base URL)", 1192 examples=["session"], 1193 title="Login Path", 1194 ) 1195 session_token: Optional[str] = Field( 1196 None, 1197 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1198 example=["{{ config['session_token'] }}"], 1199 title="Session Token", 1200 ) 1201 session_token_response_key: str = Field( 1202 ..., 1203 description="Name of the key of the session token to be extracted from the response", 1204 examples=["id"], 1205 title="Response Token Response Key", 1206 ) 1207 username: Optional[str] = Field( 1208 None, 1209 description="Username used to authenticate and obtain a session token", 1210 examples=[" {{ config['username'] }}"], 1211 title="Username", 1212 ) 1213 password: Optional[str] = Field( 1214 "", 1215 description="Password used to authenticate and obtain a session token", 1216 examples=["{{ config['password'] }}", ""], 1217 title="Password", 1218 ) 1219 validate_session_url: str = Field( 1220 ..., 1221 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1222 examples=["user/current"], 1223 title="Validate Session Path", 1224 ) 1225 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1226 1227 1228class Action1(Enum): 1229 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1230 RESET = "RESET" 1231 1232 1233class PaginationResetLimits(BaseModel): 1234 type: Literal["PaginationResetLimits"] 1235 number_of_records: Optional[int] = None 1236 1237 1238class CsvDecoder(BaseModel): 1239 type: Literal["CsvDecoder"] 1240 encoding: Optional[str] = "utf-8" 1241 delimiter: Optional[str] = "," 1242 set_values_to_none: Optional[List[str]] = None 1243 1244 1245class AsyncJobStatusMap(BaseModel): 1246 type: Optional[Literal["AsyncJobStatusMap"]] = None 1247 running: List[str] 1248 completed: List[str] 1249 failed: List[str] 1250 timeout: List[str] 1251 1252 1253class ValueType(Enum): 1254 string = "string" 1255 number = "number" 1256 integer = "integer" 1257 boolean = "boolean" 1258 1259 1260class WaitTimeFromHeader(BaseModel): 1261 type: Literal["WaitTimeFromHeader"] 1262 header: str = Field( 1263 ..., 1264 description="The name of the response header defining how long to wait before retrying.", 1265 examples=["Retry-After"], 1266 title="Response Header Name", 1267 ) 1268 regex: Optional[str] = Field( 1269 None, 1270 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1271 examples=["([-+]?\\d+)"], 1272 title="Extraction Regex", 1273 ) 1274 max_waiting_time_in_seconds: Optional[float] = Field( 1275 None, 1276 description="Given the value extracted from the header is greater than this value, stop the stream.", 1277 examples=[3600], 1278 title="Max Waiting Time in Seconds", 1279 ) 1280 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1281 1282 1283class WaitUntilTimeFromHeader(BaseModel): 1284 type: Literal["WaitUntilTimeFromHeader"] 1285 header: str = Field( 1286 ..., 1287 description="The name of the response header defining how long to wait before retrying.", 1288 examples=["wait_time"], 1289 title="Response Header", 1290 ) 1291 min_wait: Optional[Union[float, str]] = Field( 1292 None, 1293 description="Minimum time to wait before retrying.", 1294 examples=[10, "60"], 1295 title="Minimum Wait Time", 1296 ) 1297 regex: Optional[str] = Field( 1298 None, 1299 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1300 examples=["([-+]?\\d+)"], 1301 title="Extraction Regex", 1302 ) 1303 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1304 1305 1306class ComponentMappingDefinition(BaseModel): 1307 type: Literal["ComponentMappingDefinition"] 1308 field_path: List[str] = Field( 1309 ..., 1310 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1311 examples=[ 1312 ["name"], 1313 ["retriever", "requester", "url"], 1314 ["retriever", "requester", "{{ components_values.field }}"], 1315 ["*", "**", "name"], 1316 ], 1317 title="Field Path", 1318 ) 1319 value: str = Field( 1320 ..., 1321 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1322 examples=[ 1323 "{{ components_values['updates'] }}", 1324 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1325 "{{ config['segment_id'] }}", 1326 "{{ stream_slice['parent_id'] }}", 1327 "{{ stream_slice['extra_fields']['name'] }}", 1328 ], 1329 title="Value", 1330 ) 1331 value_type: Optional[ValueType] = Field( 1332 None, 1333 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1334 title="Value Type", 1335 ) 1336 create_or_update: Optional[bool] = Field( 1337 False, 1338 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1339 title="Create or Update", 1340 ) 1341 condition: Optional[str] = Field( 1342 None, 1343 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1344 examples=[ 1345 "{{ components_values.get('cursor_field', None) }}", 1346 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1347 ], 1348 title="Condition", 1349 ) 1350 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1351 1352 1353class StreamConfig(BaseModel): 1354 type: Literal["StreamConfig"] 1355 configs_pointer: List[str] = Field( 1356 ..., 1357 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1358 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1359 title="Configs Pointer", 1360 ) 1361 default_values: Optional[List[Dict[str, Any]]] = Field( 1362 None, 1363 description="A list of default values, each matching the structure expected from the parsed component value.", 1364 title="Default Values", 1365 ) 1366 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1367 1368 1369class ConfigComponentsResolver(BaseModel): 1370 type: Literal["ConfigComponentsResolver"] 1371 stream_config: Union[List[StreamConfig], StreamConfig] 1372 components_mapping: List[ComponentMappingDefinition] 1373 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1374 1375 1376class StreamParametersDefinition(BaseModel): 1377 type: Literal["StreamParametersDefinition"] 1378 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1379 ..., 1380 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1381 examples=[ 1382 [ 1383 { 1384 "name": "test stream", 1385 "$parameters": {"entity": "test entity"}, 1386 "primary_key": "test key", 1387 } 1388 ] 1389 ], 1390 title="Stream Parameters", 1391 ) 1392 1393 1394class ParametrizedComponentsResolver(BaseModel): 1395 type: Literal["ParametrizedComponentsResolver"] 1396 stream_parameters: StreamParametersDefinition 1397 components_mapping: List[ComponentMappingDefinition] 1398 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1399 1400 1401class RequestBodyPlainText(BaseModel): 1402 type: Literal["RequestBodyPlainText"] 1403 value: str 1404 1405 1406class RequestBodyUrlEncodedForm(BaseModel): 1407 type: Literal["RequestBodyUrlEncodedForm"] 1408 value: Dict[str, str] 1409 1410 1411class RequestBodyJsonObject(BaseModel): 1412 type: Literal["RequestBodyJsonObject"] 1413 value: Dict[str, Any] 1414 1415 1416class RequestBodyGraphQlQuery(BaseModel): 1417 class Config: 1418 extra = Extra.allow 1419 1420 query: str = Field(..., description="The GraphQL query to be executed") 1421 1422 1423class ValidateAdheresToSchema(BaseModel): 1424 type: Literal["ValidateAdheresToSchema"] 1425 base_schema: Union[str, Dict[str, Any]] = Field( 1426 ..., 1427 description="The base JSON schema against which the user-provided schema will be validated.", 1428 examples=[ 1429 "{{ config['report_validation_schema'] }}", 1430 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1431 { 1432 "$schema": "http://json-schema.org/draft-07/schema#", 1433 "title": "Person", 1434 "type": "object", 1435 "properties": { 1436 "name": {"type": "string", "description": "The person's name"}, 1437 "age": { 1438 "type": "integer", 1439 "minimum": 0, 1440 "description": "The person's age", 1441 }, 1442 }, 1443 "required": ["name", "age"], 1444 }, 1445 ], 1446 title="Base JSON Schema", 1447 ) 1448 1449 1450class CustomValidationStrategy(BaseModel): 1451 class Config: 1452 extra = Extra.allow 1453 1454 type: Literal["CustomValidationStrategy"] 1455 class_name: str = Field( 1456 ..., 1457 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1458 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1459 title="Class Name", 1460 ) 1461 1462 1463class ConfigRemapField(BaseModel): 1464 type: Literal["ConfigRemapField"] 1465 map: Union[Dict[str, Any], str] = Field( 1466 ..., 1467 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1468 examples=[ 1469 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1470 "{{ config['status_mapping'] }}", 1471 ], 1472 title="Value Mapping", 1473 ) 1474 field_path: List[str] = Field( 1475 ..., 1476 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1477 examples=[ 1478 ["status"], 1479 ["data", "status"], 1480 ["data", "{{ config.name }}", "status"], 1481 ["data", "*", "status"], 1482 ], 1483 title="Field Path", 1484 ) 1485 1486 1487class ConfigRemoveFields(BaseModel): 1488 type: Literal["ConfigRemoveFields"] 1489 field_pointers: List[List[str]] = Field( 1490 ..., 1491 description="A list of field pointers to be removed from the config.", 1492 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1493 title="Field Pointers", 1494 ) 1495 condition: Optional[str] = Field( 1496 "", 1497 description="Fields will be removed if expression is evaluated to True.", 1498 examples=[ 1499 "{{ config['environemnt'] == 'sandbox' }}", 1500 "{{ property is integer }}", 1501 "{{ property|length > 5 }}", 1502 "{{ property == 'some_string_to_match' }}", 1503 ], 1504 ) 1505 1506 1507class CustomConfigTransformation(BaseModel): 1508 type: Literal["CustomConfigTransformation"] 1509 class_name: str = Field( 1510 ..., 1511 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1512 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1513 ) 1514 parameters: Optional[Dict[str, Any]] = Field( 1515 None, 1516 alias="$parameters", 1517 description="Additional parameters to be passed to the custom config transformation.", 1518 ) 1519 1520 1521class AddedFieldDefinition(BaseModel): 1522 type: Literal["AddedFieldDefinition"] 1523 path: List[str] = Field( 1524 ..., 1525 description="List of strings defining the path where to add the value on the record.", 1526 examples=[["segment_id"], ["metadata", "segment_id"]], 1527 title="Path", 1528 ) 1529 value: str = Field( 1530 ..., 1531 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1532 examples=[ 1533 "{{ record['updates'] }}", 1534 "{{ record['MetaData']['LastUpdatedTime'] }}", 1535 "{{ stream_partition['segment_id'] }}", 1536 ], 1537 title="Value", 1538 ) 1539 value_type: Optional[ValueType] = Field( 1540 None, 1541 description="Type of the value. If not specified, the type will be inferred from the value.", 1542 title="Value Type", 1543 ) 1544 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1545 1546 1547class AddFields(BaseModel): 1548 type: Literal["AddFields"] 1549 fields: List[AddedFieldDefinition] = Field( 1550 ..., 1551 description="List of transformations (path and corresponding value) that will be added to the record.", 1552 title="Fields", 1553 ) 1554 condition: Optional[str] = Field( 1555 "", 1556 description="Fields will be added if expression is evaluated to True.", 1557 examples=[ 1558 "{{ property|string == '' }}", 1559 "{{ property is integer }}", 1560 "{{ property|length > 5 }}", 1561 "{{ property == 'some_string_to_match' }}", 1562 ], 1563 ) 1564 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1565 1566 1567class ApiKeyAuthenticator(BaseModel): 1568 type: Literal["ApiKeyAuthenticator"] 1569 api_token: Optional[str] = Field( 1570 None, 1571 description="The API key to inject in the request. Fill it in the user inputs.", 1572 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1573 title="API Key", 1574 ) 1575 header: Optional[str] = Field( 1576 None, 1577 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1578 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1579 title="Header Name", 1580 ) 1581 inject_into: Optional[RequestOption] = Field( 1582 None, 1583 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1584 examples=[ 1585 {"inject_into": "header", "field_name": "Authorization"}, 1586 {"inject_into": "request_parameter", "field_name": "authKey"}, 1587 ], 1588 title="Inject API Key Into Outgoing HTTP Request", 1589 ) 1590 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1591 1592 1593class AuthFlow(BaseModel): 1594 auth_flow_type: Optional[AuthFlowType] = Field( 1595 None, description="The type of auth to use", title="Auth flow type" 1596 ) 1597 predicate_key: Optional[List[str]] = Field( 1598 None, 1599 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1600 examples=[["credentials", "auth_type"]], 1601 title="Predicate key", 1602 ) 1603 predicate_value: Optional[str] = Field( 1604 None, 1605 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1606 examples=["Oauth"], 1607 title="Predicate value", 1608 ) 1609 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1610 1611 1612class CheckStream(BaseModel): 1613 type: Literal["CheckStream"] 1614 stream_names: Optional[List[str]] = Field( 1615 None, 1616 description="Names of the streams to try reading from when running a check operation.", 1617 examples=[["users"], ["users", "contacts"]], 1618 title="Stream Names", 1619 ) 1620 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1621 1622 1623class IncrementingCountCursor(BaseModel): 1624 type: Literal["IncrementingCountCursor"] 1625 cursor_field: str = Field( 1626 ..., 1627 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1628 examples=["created_at", "{{ config['record_cursor'] }}"], 1629 title="Cursor Field", 1630 ) 1631 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1632 None, 1633 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1634 title="Allow Catalog Defined Cursor Field", 1635 ) 1636 start_value: Optional[Union[str, int]] = Field( 1637 None, 1638 description="The value that determines the earliest record that should be synced.", 1639 examples=[0, "{{ config['start_value'] }}"], 1640 title="Start Value", 1641 ) 1642 start_value_option: Optional[RequestOption] = Field( 1643 None, 1644 description="Optionally configures how the start value will be sent in requests to the source API.", 1645 title="Inject Start Value Into Outgoing HTTP Request", 1646 ) 1647 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1648 1649 1650class DatetimeBasedCursor(BaseModel): 1651 type: Literal["DatetimeBasedCursor"] 1652 clamping: Optional[Clamping] = Field( 1653 None, 1654 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1655 title="Date Range Clamping", 1656 ) 1657 cursor_field: str = Field( 1658 ..., 1659 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1660 examples=["created_at", "{{ config['record_cursor'] }}"], 1661 title="Cursor Field", 1662 ) 1663 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1664 None, 1665 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1666 title="Allow Catalog Defined Cursor Field", 1667 ) 1668 cursor_datetime_formats: Optional[List[str]] = Field( 1669 None, 1670 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1671 examples=[ 1672 "%Y-%m-%d", 1673 "%Y-%m-%d %H:%M:%S", 1674 "%Y-%m-%dT%H:%M:%S", 1675 "%Y-%m-%dT%H:%M:%SZ", 1676 "%Y-%m-%dT%H:%M:%S%z", 1677 "%Y-%m-%dT%H:%M:%S.%fZ", 1678 "%Y-%m-%dT%H:%M:%S.%f%z", 1679 "%Y-%m-%d %H:%M:%S.%f+00:00", 1680 "%s", 1681 "%ms", 1682 ], 1683 title="Cursor Datetime Formats", 1684 ) 1685 start_datetime: Union[MinMaxDatetime, str] = Field( 1686 ..., 1687 description="The datetime that determines the earliest record that should be synced.", 1688 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1689 title="Start Datetime", 1690 ) 1691 start_time_option: Optional[RequestOption] = Field( 1692 None, 1693 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1694 title="Inject Start Time Into Outgoing HTTP Request", 1695 ) 1696 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1697 None, 1698 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1699 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1700 title="End Datetime", 1701 ) 1702 end_time_option: Optional[RequestOption] = Field( 1703 None, 1704 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1705 title="Inject End Time Into Outgoing HTTP Request", 1706 ) 1707 datetime_format: str = Field( 1708 ..., 1709 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1710 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1711 title="Outgoing Datetime Format", 1712 ) 1713 cursor_granularity: Optional[str] = Field( 1714 None, 1715 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1716 examples=["PT1S"], 1717 title="Cursor Granularity", 1718 ) 1719 is_data_feed: Optional[bool] = Field( 1720 None, 1721 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1722 title="Data Feed API", 1723 ) 1724 is_client_side_incremental: Optional[bool] = Field( 1725 None, 1726 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1727 title="Client-side Incremental Filtering", 1728 ) 1729 is_compare_strictly: Optional[bool] = Field( 1730 False, 1731 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1732 title="Strict Start-End Time Comparison", 1733 ) 1734 global_substream_cursor: Optional[bool] = Field( 1735 False, 1736 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1737 title="Global Substream Cursor", 1738 ) 1739 lookback_window: Optional[str] = Field( 1740 None, 1741 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1742 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1743 title="Lookback Window", 1744 ) 1745 partition_field_end: Optional[str] = Field( 1746 None, 1747 description="Name of the partition start time field.", 1748 examples=["ending_time"], 1749 title="Partition Field End", 1750 ) 1751 partition_field_start: Optional[str] = Field( 1752 None, 1753 description="Name of the partition end time field.", 1754 examples=["starting_time"], 1755 title="Partition Field Start", 1756 ) 1757 step: Optional[str] = Field( 1758 None, 1759 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1760 examples=["P1W", "{{ config['step_increment'] }}"], 1761 title="Step", 1762 ) 1763 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1764 1765 1766class JwtAuthenticator(BaseModel): 1767 type: Literal["JwtAuthenticator"] 1768 secret_key: str = Field( 1769 ..., 1770 description="Secret used to sign the JSON web token.", 1771 examples=["{{ config['secret_key'] }}"], 1772 title="Secret Key", 1773 ) 1774 base64_encode_secret_key: Optional[bool] = Field( 1775 False, 1776 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1777 title="Base64-encode Secret Key", 1778 ) 1779 algorithm: Algorithm = Field( 1780 ..., 1781 description="Algorithm used to sign the JSON web token.", 1782 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1783 title="Algorithm", 1784 ) 1785 token_duration: Optional[int] = Field( 1786 1200, 1787 description="The amount of time in seconds a JWT token can be valid after being issued.", 1788 examples=[1200, 3600], 1789 title="Token Duration", 1790 ) 1791 header_prefix: Optional[str] = Field( 1792 None, 1793 description="The prefix to be used within the Authentication header.", 1794 examples=["Bearer", "Basic"], 1795 title="Header Prefix", 1796 ) 1797 jwt_headers: Optional[JwtHeaders] = Field( 1798 None, 1799 description="JWT headers used when signing JSON web token.", 1800 title="JWT Headers", 1801 ) 1802 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1803 None, 1804 description="Additional headers to be included with the JWT headers object.", 1805 title="Additional JWT Headers", 1806 ) 1807 jwt_payload: Optional[JwtPayload] = Field( 1808 None, 1809 description="JWT Payload used when signing JSON web token.", 1810 title="JWT Payload", 1811 ) 1812 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1813 None, 1814 description="Additional properties to be added to the JWT payload.", 1815 title="Additional JWT Payload Properties", 1816 ) 1817 passphrase: Optional[str] = Field( 1818 None, 1819 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1820 examples=["{{ config['passphrase'] }}"], 1821 title="Passphrase", 1822 ) 1823 request_option: Optional[RequestOption] = Field( 1824 None, 1825 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1826 title="Request Option", 1827 ) 1828 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1829 1830 1831class OAuthAuthenticator(BaseModel): 1832 type: Literal["OAuthAuthenticator"] 1833 client_id_name: Optional[str] = Field( 1834 "client_id", 1835 description="The name of the property to use to refresh the `access_token`.", 1836 examples=["custom_app_id"], 1837 title="Client ID Property Name", 1838 ) 1839 client_id: Optional[str] = Field( 1840 None, 1841 description="The OAuth client ID. Fill it in the user inputs.", 1842 examples=[ 1843 "{{ config['client_id'] }}", 1844 "{{ config['credentials']['client_id }}", 1845 ], 1846 title="Client ID", 1847 ) 1848 client_secret_name: Optional[str] = Field( 1849 "client_secret", 1850 description="The name of the property to use to refresh the `access_token`.", 1851 examples=["custom_app_secret"], 1852 title="Client Secret Property Name", 1853 ) 1854 client_secret: Optional[str] = Field( 1855 None, 1856 description="The OAuth client secret. Fill it in the user inputs.", 1857 examples=[ 1858 "{{ config['client_secret'] }}", 1859 "{{ config['credentials']['client_secret }}", 1860 ], 1861 title="Client Secret", 1862 ) 1863 refresh_token_name: Optional[str] = Field( 1864 "refresh_token", 1865 description="The name of the property to use to refresh the `access_token`.", 1866 examples=["custom_app_refresh_value"], 1867 title="Refresh Token Property Name", 1868 ) 1869 refresh_token: Optional[str] = Field( 1870 None, 1871 description="Credential artifact used to get a new access token.", 1872 examples=[ 1873 "{{ config['refresh_token'] }}", 1874 "{{ config['credentials]['refresh_token'] }}", 1875 ], 1876 title="Refresh Token", 1877 ) 1878 token_refresh_endpoint: Optional[str] = Field( 1879 None, 1880 description="The full URL to call to obtain a new access token.", 1881 examples=["https://connect.squareup.com/oauth2/token"], 1882 title="Token Refresh Endpoint", 1883 ) 1884 access_token_name: Optional[str] = Field( 1885 "access_token", 1886 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1887 examples=["access_token"], 1888 title="Access Token Property Name", 1889 ) 1890 access_token_value: Optional[str] = Field( 1891 None, 1892 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1893 examples=["secret_access_token_value"], 1894 title="Access Token Value", 1895 ) 1896 expires_in_name: Optional[str] = Field( 1897 "expires_in", 1898 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1899 examples=["expires_in"], 1900 title="Token Expiry Property Name", 1901 ) 1902 grant_type_name: Optional[str] = Field( 1903 "grant_type", 1904 description="The name of the property to use to refresh the `access_token`.", 1905 examples=["custom_grant_type"], 1906 title="Grant Type Property Name", 1907 ) 1908 grant_type: Optional[str] = Field( 1909 "refresh_token", 1910 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1911 examples=["refresh_token", "client_credentials"], 1912 title="Grant Type", 1913 ) 1914 refresh_request_body: Optional[Dict[str, Any]] = Field( 1915 None, 1916 description="Body of the request sent to get a new access token.", 1917 examples=[ 1918 { 1919 "applicationId": "{{ config['application_id'] }}", 1920 "applicationSecret": "{{ config['application_secret'] }}", 1921 "token": "{{ config['token'] }}", 1922 } 1923 ], 1924 title="Refresh Request Body", 1925 ) 1926 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1927 None, 1928 description="Headers of the request sent to get a new access token.", 1929 examples=[ 1930 { 1931 "Authorization": "<AUTH_TOKEN>", 1932 "Content-Type": "application/x-www-form-urlencoded", 1933 } 1934 ], 1935 title="Refresh Request Headers", 1936 ) 1937 scopes: Optional[List[str]] = Field( 1938 None, 1939 description="List of scopes that should be granted to the access token.", 1940 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1941 title="Scopes", 1942 ) 1943 token_expiry_date: Optional[str] = Field( 1944 None, 1945 description="The access token expiry date.", 1946 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1947 title="Token Expiry Date", 1948 ) 1949 token_expiry_date_format: Optional[str] = Field( 1950 None, 1951 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1952 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1953 title="Token Expiry Date Format", 1954 ) 1955 refresh_token_error_status_codes: Optional[List[int]] = Field( 1956 None, 1957 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1958 examples=[[400, 500]], 1959 title="Refresh Token Error Status Codes", 1960 ) 1961 refresh_token_error_key: Optional[str] = Field( 1962 None, 1963 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1964 examples=["error"], 1965 title="Refresh Token Error Key", 1966 ) 1967 refresh_token_error_values: Optional[List[str]] = Field( 1968 None, 1969 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1970 examples=[["invalid_grant", "invalid_permissions"]], 1971 title="Refresh Token Error Values", 1972 ) 1973 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1974 None, 1975 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1976 title="Refresh Token Updater", 1977 ) 1978 profile_assertion: Optional[JwtAuthenticator] = Field( 1979 None, 1980 description="The authenticator being used to authenticate the client authenticator.", 1981 title="Profile Assertion", 1982 ) 1983 use_profile_assertion: Optional[bool] = Field( 1984 False, 1985 description="Enable using profile assertion as a flow for OAuth authorization.", 1986 title="Use Profile Assertion", 1987 ) 1988 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1989 1990 1991class FixedWindowCallRatePolicy(BaseModel): 1992 class Config: 1993 extra = Extra.allow 1994 1995 type: Literal["FixedWindowCallRatePolicy"] 1996 period: str = Field( 1997 ..., description="The time interval for the rate limit window.", title="Period" 1998 ) 1999 call_limit: int = Field( 2000 ..., 2001 description="The maximum number of calls allowed within the period.", 2002 title="Call Limit", 2003 ) 2004 matchers: List[HttpRequestRegexMatcher] = Field( 2005 ..., 2006 description="List of matchers that define which requests this policy applies to.", 2007 title="Matchers", 2008 ) 2009 2010 2011class MovingWindowCallRatePolicy(BaseModel): 2012 class Config: 2013 extra = Extra.allow 2014 2015 type: Literal["MovingWindowCallRatePolicy"] 2016 rates: List[Rate] = Field( 2017 ..., 2018 description="List of rates that define the call limits for different time intervals.", 2019 title="Rates", 2020 ) 2021 matchers: List[HttpRequestRegexMatcher] = Field( 2022 ..., 2023 description="List of matchers that define which requests this policy applies to.", 2024 title="Matchers", 2025 ) 2026 2027 2028class UnlimitedCallRatePolicy(BaseModel): 2029 class Config: 2030 extra = Extra.allow 2031 2032 type: Literal["UnlimitedCallRatePolicy"] 2033 matchers: List[HttpRequestRegexMatcher] = Field( 2034 ..., 2035 description="List of matchers that define which requests this policy applies to.", 2036 title="Matchers", 2037 ) 2038 2039 2040class DefaultErrorHandler(BaseModel): 2041 type: Literal["DefaultErrorHandler"] 2042 backoff_strategies: Optional[ 2043 List[ 2044 Union[ 2045 ConstantBackoffStrategy, 2046 ExponentialBackoffStrategy, 2047 WaitTimeFromHeader, 2048 WaitUntilTimeFromHeader, 2049 CustomBackoffStrategy, 2050 ] 2051 ] 2052 ] = Field( 2053 None, 2054 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2055 title="Backoff Strategies", 2056 ) 2057 max_retries: Optional[int] = Field( 2058 5, 2059 description="The maximum number of time to retry a retryable request before giving up and failing.", 2060 examples=[5, 0, 10], 2061 title="Max Retry Count", 2062 ) 2063 response_filters: Optional[List[HttpResponseFilter]] = Field( 2064 None, 2065 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2066 title="Response Filters", 2067 ) 2068 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2069 2070 2071class DefaultPaginator(BaseModel): 2072 type: Literal["DefaultPaginator"] 2073 pagination_strategy: Union[ 2074 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2075 ] = Field( 2076 ..., 2077 description="Strategy defining how records are paginated.", 2078 title="Pagination Strategy", 2079 ) 2080 page_size_option: Optional[RequestOption] = Field( 2081 None, title="Inject Page Size Into Outgoing HTTP Request" 2082 ) 2083 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2084 None, 2085 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2086 title="Inject Page Token Into Outgoing HTTP Request", 2087 ) 2088 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2089 2090 2091class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2092 type: Literal["ApiKey"] 2093 inject_into: RequestOption = Field( 2094 ..., 2095 description="Configure how the API Key will be sent in requests to the source API.", 2096 examples=[ 2097 {"inject_into": "header", "field_name": "Authorization"}, 2098 {"inject_into": "request_parameter", "field_name": "authKey"}, 2099 ], 2100 title="Inject API Key Into Outgoing HTTP Request", 2101 ) 2102 api_token: Optional[str] = Field( 2103 "{{ session_token }}", 2104 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2105 examples=[ 2106 "{{ session_token }}", 2107 "Token {{ session_token }}", 2108 "Bearer {{ session_token }}", 2109 ], 2110 title="API Token Template", 2111 ) 2112 2113 2114class JsonSchemaPropertySelector(BaseModel): 2115 type: Literal["JsonSchemaPropertySelector"] 2116 transformations: Optional[ 2117 List[ 2118 Union[ 2119 AddFields, 2120 RemoveFields, 2121 KeysToLower, 2122 KeysToSnakeCase, 2123 FlattenFields, 2124 DpathFlattenFields, 2125 KeysReplace, 2126 CustomTransformation, 2127 ] 2128 ] 2129 ] = Field( 2130 None, 2131 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2132 title="Transformations", 2133 ) 2134 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2135 2136 2137class ListPartitionRouter(BaseModel): 2138 type: Literal["ListPartitionRouter"] 2139 cursor_field: str = Field( 2140 ..., 2141 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2142 examples=["section", "{{ config['section_key'] }}"], 2143 title="Current Partition Value Identifier", 2144 ) 2145 values: Union[str, List[str]] = Field( 2146 ..., 2147 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2148 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2149 title="Partition Values", 2150 ) 2151 request_option: Optional[RequestOption] = Field( 2152 None, 2153 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2154 title="Inject Partition Value Into Outgoing HTTP Request", 2155 ) 2156 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2157 2158 2159class RecordSelector(BaseModel): 2160 type: Literal["RecordSelector"] 2161 extractor: Union[DpathExtractor, CustomRecordExtractor] 2162 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2163 None, 2164 description="Responsible for filtering records to be emitted by the Source.", 2165 title="Record Filter", 2166 ) 2167 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2168 None, 2169 description="Responsible for normalization according to the schema.", 2170 title="Schema Normalization", 2171 ) 2172 transform_before_filtering: Optional[bool] = Field( 2173 None, 2174 description="If true, transformation will be applied before record filtering.", 2175 title="Transform Before Filtering", 2176 ) 2177 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2178 2179 2180class PaginationReset(BaseModel): 2181 type: Literal["PaginationReset"] 2182 action: Action1 2183 limits: Optional[PaginationResetLimits] = None 2184 2185 2186class GzipDecoder(BaseModel): 2187 type: Literal["GzipDecoder"] 2188 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2189 2190 2191class RequestBodyGraphQL(BaseModel): 2192 type: Literal["RequestBodyGraphQL"] 2193 value: RequestBodyGraphQlQuery 2194 2195 2196class DpathValidator(BaseModel): 2197 type: Literal["DpathValidator"] 2198 field_path: List[str] = Field( 2199 ..., 2200 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2201 examples=[ 2202 ["data"], 2203 ["data", "records"], 2204 ["data", "{{ parameters.name }}"], 2205 ["data", "*", "record"], 2206 ], 2207 title="Field Path", 2208 ) 2209 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2210 ..., 2211 description="The condition that the specified config value will be evaluated against", 2212 title="Validation Strategy", 2213 ) 2214 2215 2216class PredicateValidator(BaseModel): 2217 type: Literal["PredicateValidator"] 2218 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2219 ..., 2220 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2221 examples=[ 2222 "test-value", 2223 "{{ config['api_version'] }}", 2224 "{{ config['tenant_id'] }}", 2225 123, 2226 ], 2227 title="Value", 2228 ) 2229 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2230 ..., 2231 description="The validation strategy to apply to the value.", 2232 title="Validation Strategy", 2233 ) 2234 2235 2236class ConfigAddFields(BaseModel): 2237 type: Literal["ConfigAddFields"] 2238 fields: List[AddedFieldDefinition] = Field( 2239 ..., 2240 description="A list of transformations (path and corresponding value) that will be added to the config.", 2241 title="Fields", 2242 ) 2243 condition: Optional[str] = Field( 2244 "", 2245 description="Fields will be added if expression is evaluated to True.", 2246 examples=[ 2247 "{{ config['environemnt'] == 'sandbox' }}", 2248 "{{ property is integer }}", 2249 "{{ property|length > 5 }}", 2250 "{{ property == 'some_string_to_match' }}", 2251 ], 2252 ) 2253 2254 2255class CompositeErrorHandler(BaseModel): 2256 type: Literal["CompositeErrorHandler"] 2257 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2258 Field( 2259 ..., 2260 description="List of error handlers to iterate on to determine how to handle a failed response.", 2261 title="Error Handlers", 2262 ) 2263 ) 2264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2265 2266 2267class HTTPAPIBudget(BaseModel): 2268 class Config: 2269 extra = Extra.allow 2270 2271 type: Literal["HTTPAPIBudget"] 2272 policies: List[ 2273 Union[ 2274 FixedWindowCallRatePolicy, 2275 MovingWindowCallRatePolicy, 2276 UnlimitedCallRatePolicy, 2277 ] 2278 ] = Field( 2279 ..., 2280 description="List of call rate policies that define how many calls are allowed.", 2281 title="Policies", 2282 ) 2283 ratelimit_reset_header: Optional[str] = Field( 2284 "ratelimit-reset", 2285 description="The HTTP response header name that indicates when the rate limit resets.", 2286 title="Rate Limit Reset Header", 2287 ) 2288 ratelimit_remaining_header: Optional[str] = Field( 2289 "ratelimit-remaining", 2290 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2291 title="Rate Limit Remaining Header", 2292 ) 2293 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2294 [429], 2295 description="List of HTTP status codes that indicate a rate limit has been hit.", 2296 title="Status Codes for Rate Limit Hit", 2297 ) 2298 2299 2300class ZipfileDecoder(BaseModel): 2301 class Config: 2302 extra = Extra.allow 2303 2304 type: Literal["ZipfileDecoder"] 2305 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2306 ..., 2307 description="Parser to parse the decompressed data from the zipfile(s).", 2308 title="Parser", 2309 ) 2310 2311 2312class ConfigMigration(BaseModel): 2313 type: Literal["ConfigMigration"] 2314 description: Optional[str] = Field( 2315 None, description="The description/purpose of the config migration." 2316 ) 2317 transformations: List[ 2318 Union[ 2319 ConfigRemapField, 2320 ConfigAddFields, 2321 ConfigRemoveFields, 2322 CustomConfigTransformation, 2323 ] 2324 ] = Field( 2325 ..., 2326 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2327 title="Transformations", 2328 ) 2329 2330 2331class ConfigNormalizationRules(BaseModel): 2332 class Config: 2333 extra = Extra.forbid 2334 2335 type: Literal["ConfigNormalizationRules"] 2336 config_migrations: Optional[List[ConfigMigration]] = Field( 2337 [], 2338 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2339 title="Config Migrations", 2340 ) 2341 transformations: Optional[ 2342 List[ 2343 Union[ 2344 ConfigRemapField, 2345 ConfigAddFields, 2346 ConfigRemoveFields, 2347 CustomConfigTransformation, 2348 ] 2349 ] 2350 ] = Field( 2351 [], 2352 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2353 title="Transformations", 2354 ) 2355 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2356 [], 2357 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2358 title="Validations", 2359 ) 2360 2361 2362class Spec(BaseModel): 2363 type: Literal["Spec"] 2364 connection_specification: Dict[str, Any] = Field( 2365 ..., 2366 description="A connection specification describing how a the connector can be configured.", 2367 title="Connection Specification", 2368 ) 2369 documentation_url: Optional[str] = Field( 2370 None, 2371 description="URL of the connector's documentation page.", 2372 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2373 title="Documentation URL", 2374 ) 2375 advanced_auth: Optional[AuthFlow] = Field( 2376 None, 2377 description="Advanced specification for configuring the authentication flow.", 2378 title="Advanced Auth", 2379 ) 2380 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2381 None, title="Config Normalization Rules" 2382 ) 2383 2384 2385class DeclarativeSource1(BaseModel): 2386 class Config: 2387 extra = Extra.forbid 2388 2389 type: Literal["DeclarativeSource"] 2390 check: Union[CheckStream, CheckDynamicStream] 2391 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2392 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2393 version: str = Field( 2394 ..., 2395 description="The version of the Airbyte CDK used to build and test the source.", 2396 ) 2397 schemas: Optional[Schemas] = None 2398 definitions: Optional[Dict[str, Any]] = None 2399 spec: Optional[Spec] = None 2400 concurrency_level: Optional[ConcurrencyLevel] = None 2401 api_budget: Optional[HTTPAPIBudget] = None 2402 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2403 None, 2404 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2405 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2406 title="Maximum Concurrent Asynchronous Jobs", 2407 ) 2408 metadata: Optional[Dict[str, Any]] = Field( 2409 None, 2410 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2411 ) 2412 description: Optional[str] = Field( 2413 None, 2414 description="A description of the connector. It will be presented on the Source documentation page.", 2415 ) 2416 2417 2418class DeclarativeSource2(BaseModel): 2419 class Config: 2420 extra = Extra.forbid 2421 2422 type: Literal["DeclarativeSource"] 2423 check: Union[CheckStream, CheckDynamicStream] 2424 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2425 None 2426 ) 2427 dynamic_streams: List[DynamicDeclarativeStream] 2428 version: str = Field( 2429 ..., 2430 description="The version of the Airbyte CDK used to build and test the source.", 2431 ) 2432 schemas: Optional[Schemas] = None 2433 definitions: Optional[Dict[str, Any]] = None 2434 spec: Optional[Spec] = None 2435 concurrency_level: Optional[ConcurrencyLevel] = None 2436 api_budget: Optional[HTTPAPIBudget] = None 2437 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2438 None, 2439 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2440 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2441 title="Maximum Concurrent Asynchronous Jobs", 2442 ) 2443 metadata: Optional[Dict[str, Any]] = Field( 2444 None, 2445 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2446 ) 2447 description: Optional[str] = Field( 2448 None, 2449 description="A description of the connector. It will be presented on the Source documentation page.", 2450 ) 2451 2452 2453class DeclarativeSource(BaseModel): 2454 class Config: 2455 extra = Extra.forbid 2456 2457 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2458 ..., 2459 description="An API source that extracts data according to its declarative components.", 2460 title="DeclarativeSource", 2461 ) 2462 2463 2464class SelectiveAuthenticator(BaseModel): 2465 class Config: 2466 extra = Extra.allow 2467 2468 type: Literal["SelectiveAuthenticator"] 2469 authenticator_selection_path: List[str] = Field( 2470 ..., 2471 description="Path of the field in config with selected authenticator name", 2472 examples=[["auth"], ["auth", "type"]], 2473 title="Authenticator Selection Path", 2474 ) 2475 authenticators: Dict[ 2476 str, 2477 Union[ 2478 ApiKeyAuthenticator, 2479 BasicHttpAuthenticator, 2480 BearerAuthenticator, 2481 OAuthAuthenticator, 2482 JwtAuthenticator, 2483 SessionTokenAuthenticator, 2484 LegacySessionTokenAuthenticator, 2485 CustomAuthenticator, 2486 NoAuth, 2487 ], 2488 ] = Field( 2489 ..., 2490 description="Authenticators to select from.", 2491 examples=[ 2492 { 2493 "authenticators": { 2494 "token": "#/definitions/ApiKeyAuthenticator", 2495 "oauth": "#/definitions/OAuthAuthenticator", 2496 "jwt": "#/definitions/JwtAuthenticator", 2497 } 2498 } 2499 ], 2500 title="Authenticators", 2501 ) 2502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2503 2504 2505class ConditionalStreams(BaseModel): 2506 type: Literal["ConditionalStreams"] 2507 condition: str = Field( 2508 ..., 2509 description="Condition that will be evaluated to determine if a set of streams should be available.", 2510 examples=["{{ config['is_sandbox'] }}"], 2511 title="Condition", 2512 ) 2513 streams: List[DeclarativeStream] = Field( 2514 ..., 2515 description="Streams that will be used during an operation based on the condition.", 2516 title="Streams", 2517 ) 2518 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2519 2520 2521class FileUploader(BaseModel): 2522 type: Literal["FileUploader"] 2523 requester: Union[HttpRequester, CustomRequester] = Field( 2524 ..., 2525 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2526 ) 2527 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2528 ..., 2529 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2530 ) 2531 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2532 None, 2533 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2534 ) 2535 filename_extractor: Optional[str] = Field( 2536 None, 2537 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2538 examples=[ 2539 "{{ record.id }}/{{ record.file_name }}/", 2540 "{{ record.id }}_{{ record.file_name }}/", 2541 ], 2542 ) 2543 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2544 2545 2546class DeclarativeStream(BaseModel): 2547 class Config: 2548 extra = Extra.allow 2549 2550 type: Literal["DeclarativeStream"] 2551 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2552 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2553 ..., 2554 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2555 title="Retriever", 2556 ) 2557 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2558 None, 2559 description="Component used to fetch data incrementally based on a time field in the data.", 2560 title="Incremental Sync", 2561 ) 2562 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2563 schema_loader: Optional[ 2564 Union[ 2565 InlineSchemaLoader, 2566 DynamicSchemaLoader, 2567 JsonFileSchemaLoader, 2568 List[ 2569 Union[ 2570 InlineSchemaLoader, 2571 DynamicSchemaLoader, 2572 JsonFileSchemaLoader, 2573 CustomSchemaLoader, 2574 ] 2575 ], 2576 CustomSchemaLoader, 2577 ] 2578 ] = Field( 2579 None, 2580 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2581 title="Schema Loader", 2582 ) 2583 transformations: Optional[ 2584 List[ 2585 Union[ 2586 AddFields, 2587 RemoveFields, 2588 KeysToLower, 2589 KeysToSnakeCase, 2590 FlattenFields, 2591 DpathFlattenFields, 2592 KeysReplace, 2593 CustomTransformation, 2594 ] 2595 ] 2596 ] = Field( 2597 None, 2598 description="A list of transformations to be applied to each output record.", 2599 title="Transformations", 2600 ) 2601 state_migrations: Optional[ 2602 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2603 ] = Field( 2604 [], 2605 description="Array of state migrations to be applied on the input state", 2606 title="State Migrations", 2607 ) 2608 file_uploader: Optional[FileUploader] = Field( 2609 None, 2610 description="(experimental) Describes how to fetch a file", 2611 title="File Uploader", 2612 ) 2613 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2614 2615 2616class SessionTokenAuthenticator(BaseModel): 2617 type: Literal["SessionTokenAuthenticator"] 2618 login_requester: HttpRequester = Field( 2619 ..., 2620 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2621 examples=[ 2622 { 2623 "type": "HttpRequester", 2624 "url_base": "https://my_api.com", 2625 "path": "/login", 2626 "authenticator": { 2627 "type": "BasicHttpAuthenticator", 2628 "username": "{{ config.username }}", 2629 "password": "{{ config.password }}", 2630 }, 2631 } 2632 ], 2633 title="Login Requester", 2634 ) 2635 session_token_path: List[str] = Field( 2636 ..., 2637 description="The path in the response body returned from the login requester to the session token.", 2638 examples=[["access_token"], ["result", "token"]], 2639 title="Session Token Path", 2640 ) 2641 expiration_duration: Optional[str] = Field( 2642 None, 2643 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2644 examples=["PT1H", "P1D"], 2645 title="Expiration Duration", 2646 ) 2647 request_authentication: Union[ 2648 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2649 ] = Field( 2650 ..., 2651 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2652 title="Data Request Authentication", 2653 ) 2654 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2655 None, description="Component used to decode the response.", title="Decoder" 2656 ) 2657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2658 2659 2660class HttpRequester(BaseModelWithDeprecations): 2661 type: Literal["HttpRequester"] 2662 url_base: Optional[str] = Field( 2663 None, 2664 deprecated=True, 2665 deprecation_message="Use `url` field instead.", 2666 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2667 examples=[ 2668 "https://connect.squareup.com/v2", 2669 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2670 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2671 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2672 ], 2673 title="API Base URL", 2674 ) 2675 url: Optional[str] = Field( 2676 None, 2677 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2678 examples=[ 2679 "https://connect.squareup.com/v2", 2680 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2681 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2682 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2683 ], 2684 title="API Endpoint URL", 2685 ) 2686 path: Optional[str] = Field( 2687 None, 2688 deprecated=True, 2689 deprecation_message="Use `url` field instead.", 2690 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2691 examples=[ 2692 "/products", 2693 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2694 "/trades/{{ config['symbol_id'] }}/history", 2695 ], 2696 title="URL Path", 2697 ) 2698 http_method: Optional[HttpMethod] = Field( 2699 HttpMethod.GET, 2700 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2701 examples=["GET", "POST"], 2702 title="HTTP Method", 2703 ) 2704 authenticator: Optional[ 2705 Union[ 2706 ApiKeyAuthenticator, 2707 BasicHttpAuthenticator, 2708 BearerAuthenticator, 2709 OAuthAuthenticator, 2710 JwtAuthenticator, 2711 SessionTokenAuthenticator, 2712 SelectiveAuthenticator, 2713 CustomAuthenticator, 2714 NoAuth, 2715 LegacySessionTokenAuthenticator, 2716 ] 2717 ] = Field( 2718 None, 2719 description="Authentication method to use for requests sent to the API.", 2720 title="Authenticator", 2721 ) 2722 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2723 None, 2724 deprecated=True, 2725 deprecation_message="Use `query_properties` field instead.", 2726 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2727 title="Fetch Properties from Endpoint", 2728 ) 2729 query_properties: Optional[QueryProperties] = Field( 2730 None, 2731 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2732 title="Query Properties", 2733 ) 2734 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2735 None, 2736 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2737 examples=[ 2738 {"unit": "day"}, 2739 { 2740 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2741 }, 2742 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2743 {"sort_by[asc]": "updated_at"}, 2744 ], 2745 title="Query Parameters", 2746 ) 2747 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2748 None, 2749 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2750 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2751 title="Request Headers", 2752 ) 2753 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2754 None, 2755 deprecated=True, 2756 deprecation_message="Use `request_body` field instead.", 2757 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2758 examples=[ 2759 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2760 ], 2761 title="Request Body Payload (Non-JSON)", 2762 ) 2763 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2764 None, 2765 deprecated=True, 2766 deprecation_message="Use `request_body` field instead.", 2767 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2768 examples=[ 2769 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2770 {"key": "{{ config['value'] }}"}, 2771 {"sort": {"field": "updated_at", "order": "ascending"}}, 2772 ], 2773 title="Request Body JSON Payload", 2774 ) 2775 request_body: Optional[ 2776 Union[ 2777 RequestBodyPlainText, 2778 RequestBodyUrlEncodedForm, 2779 RequestBodyJsonObject, 2780 RequestBodyGraphQL, 2781 ] 2782 ] = Field( 2783 None, 2784 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2785 title="Request Body", 2786 ) 2787 error_handler: Optional[ 2788 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2789 ] = Field( 2790 None, 2791 description="Error handler component that defines how to handle errors.", 2792 title="Error Handler", 2793 ) 2794 use_cache: Optional[bool] = Field( 2795 False, 2796 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2797 title="Use Cache", 2798 ) 2799 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2800 2801 2802class DynamicSchemaLoader(BaseModel): 2803 type: Literal["DynamicSchemaLoader"] 2804 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2805 ..., 2806 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2807 title="Retriever", 2808 ) 2809 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2810 None, 2811 description="Responsible for filtering fields to be added to json schema.", 2812 title="Schema Filter", 2813 ) 2814 schema_transformations: Optional[ 2815 List[ 2816 Union[ 2817 AddFields, 2818 RemoveFields, 2819 KeysToLower, 2820 KeysToSnakeCase, 2821 FlattenFields, 2822 DpathFlattenFields, 2823 KeysReplace, 2824 CustomTransformation, 2825 ] 2826 ] 2827 ] = Field( 2828 None, 2829 description="A list of transformations to be applied to the schema.", 2830 title="Schema Transformations", 2831 ) 2832 schema_type_identifier: SchemaTypeIdentifier 2833 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2834 2835 2836class ParentStreamConfig(BaseModel): 2837 type: Literal["ParentStreamConfig"] 2838 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2839 ..., description="Reference to the parent stream.", title="Parent Stream" 2840 ) 2841 parent_key: str = Field( 2842 ..., 2843 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2844 examples=["id", "{{ config['parent_record_id'] }}"], 2845 title="Parent Key", 2846 ) 2847 partition_field: str = Field( 2848 ..., 2849 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2850 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2851 title="Current Parent Key Value Identifier", 2852 ) 2853 request_option: Optional[RequestOption] = Field( 2854 None, 2855 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2856 title="Request Option", 2857 ) 2858 incremental_dependency: Optional[bool] = Field( 2859 False, 2860 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2861 title="Incremental Dependency", 2862 ) 2863 lazy_read_pointer: Optional[List[str]] = Field( 2864 [], 2865 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2866 title="Lazy Read Pointer", 2867 ) 2868 extra_fields: Optional[List[List[str]]] = Field( 2869 None, 2870 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2871 title="Extra Fields", 2872 ) 2873 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2874 2875 2876class PropertiesFromEndpoint(BaseModel): 2877 type: Literal["PropertiesFromEndpoint"] 2878 property_field_path: List[str] = Field( 2879 ..., 2880 description="Describes the path to the field that should be extracted", 2881 examples=[["name"]], 2882 ) 2883 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2884 ..., 2885 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2886 ) 2887 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2888 2889 2890class QueryProperties(BaseModel): 2891 type: Literal["QueryProperties"] 2892 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2893 ..., 2894 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2895 title="Property List", 2896 ) 2897 always_include_properties: Optional[List[str]] = Field( 2898 None, 2899 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2900 title="Always Include Properties", 2901 ) 2902 property_chunking: Optional[PropertyChunking] = Field( 2903 None, 2904 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2905 title="Property Chunking", 2906 ) 2907 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2908 None, 2909 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2910 title="Property Selector", 2911 ) 2912 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2913 2914 2915class StateDelegatingStream(BaseModel): 2916 type: Literal["StateDelegatingStream"] 2917 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2918 full_refresh_stream: DeclarativeStream = Field( 2919 ..., 2920 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2921 title="Full Refresh Stream", 2922 ) 2923 incremental_stream: DeclarativeStream = Field( 2924 ..., 2925 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2926 title="Incremental Stream", 2927 ) 2928 api_retention_period: Optional[str] = Field( 2929 None, 2930 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2931 examples=["P30D", "P90D", "P1Y"], 2932 title="API Retention Period", 2933 ) 2934 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2935 2936 2937class SimpleRetriever(BaseModel): 2938 type: Literal["SimpleRetriever"] 2939 requester: Union[HttpRequester, CustomRequester] = Field( 2940 ..., 2941 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2942 ) 2943 decoder: Optional[ 2944 Union[ 2945 JsonDecoder, 2946 XmlDecoder, 2947 CsvDecoder, 2948 JsonlDecoder, 2949 GzipDecoder, 2950 IterableDecoder, 2951 ZipfileDecoder, 2952 CustomDecoder, 2953 ] 2954 ] = Field( 2955 None, 2956 description="Component decoding the response so records can be extracted.", 2957 title="HTTP Response Format", 2958 ) 2959 record_selector: RecordSelector = Field( 2960 ..., 2961 description="Component that describes how to extract records from a HTTP response.", 2962 ) 2963 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2964 None, 2965 description="Paginator component that describes how to navigate through the API's pages.", 2966 ) 2967 pagination_reset: Optional[PaginationReset] = Field( 2968 None, 2969 description="Describes what triggers pagination reset and how to handle it.", 2970 ) 2971 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2972 False, 2973 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2974 ) 2975 partition_router: Optional[ 2976 Union[ 2977 SubstreamPartitionRouter, 2978 ListPartitionRouter, 2979 GroupingPartitionRouter, 2980 CustomPartitionRouter, 2981 List[ 2982 Union[ 2983 SubstreamPartitionRouter, 2984 ListPartitionRouter, 2985 GroupingPartitionRouter, 2986 CustomPartitionRouter, 2987 ] 2988 ], 2989 ] 2990 ] = Field( 2991 None, 2992 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2993 title="Partition Router", 2994 ) 2995 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2996 2997 2998class AsyncRetriever(BaseModel): 2999 type: Literal["AsyncRetriever"] 3000 record_selector: RecordSelector = Field( 3001 ..., 3002 description="Component that describes how to extract records from a HTTP response.", 3003 ) 3004 status_mapping: AsyncJobStatusMap = Field( 3005 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3006 ) 3007 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3008 ..., description="Responsible for fetching the actual status of the async job." 3009 ) 3010 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3011 None, 3012 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3013 ) 3014 download_extractor: Optional[ 3015 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3016 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3017 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3018 ..., 3019 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3020 ) 3021 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3022 ..., 3023 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3024 ) 3025 polling_job_timeout: Optional[Union[int, str]] = Field( 3026 None, 3027 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3028 ) 3029 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3030 None, 3031 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3032 ) 3033 download_requester: Union[HttpRequester, CustomRequester] = Field( 3034 ..., 3035 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3036 ) 3037 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3038 None, 3039 description="Paginator component that describes how to navigate through the API's pages during download.", 3040 ) 3041 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3042 None, 3043 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3044 ) 3045 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3046 None, 3047 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3048 ) 3049 partition_router: Optional[ 3050 Union[ 3051 ListPartitionRouter, 3052 SubstreamPartitionRouter, 3053 GroupingPartitionRouter, 3054 CustomPartitionRouter, 3055 List[ 3056 Union[ 3057 ListPartitionRouter, 3058 SubstreamPartitionRouter, 3059 GroupingPartitionRouter, 3060 CustomPartitionRouter, 3061 ] 3062 ], 3063 ] 3064 ] = Field( 3065 [], 3066 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3067 title="Partition Router", 3068 ) 3069 decoder: Optional[ 3070 Union[ 3071 CsvDecoder, 3072 GzipDecoder, 3073 JsonDecoder, 3074 JsonlDecoder, 3075 IterableDecoder, 3076 XmlDecoder, 3077 ZipfileDecoder, 3078 CustomDecoder, 3079 ] 3080 ] = Field( 3081 None, 3082 description="Component decoding the response so records can be extracted.", 3083 title="HTTP Response Format", 3084 ) 3085 download_decoder: Optional[ 3086 Union[ 3087 CsvDecoder, 3088 GzipDecoder, 3089 JsonDecoder, 3090 JsonlDecoder, 3091 IterableDecoder, 3092 XmlDecoder, 3093 ZipfileDecoder, 3094 CustomDecoder, 3095 ] 3096 ] = Field( 3097 None, 3098 description="Component decoding the download response so records can be extracted.", 3099 title="Download HTTP Response Format", 3100 ) 3101 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3102 3103 3104class SubstreamPartitionRouter(BaseModel): 3105 type: Literal["SubstreamPartitionRouter"] 3106 parent_stream_configs: List[ParentStreamConfig] = Field( 3107 ..., 3108 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3109 title="Parent Stream Configs", 3110 ) 3111 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3112 3113 3114class GroupingPartitionRouter(BaseModel): 3115 type: Literal["GroupingPartitionRouter"] 3116 group_size: int = Field( 3117 ..., 3118 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3119 examples=[10, 50], 3120 title="Group Size", 3121 ) 3122 underlying_partition_router: Union[ 3123 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3124 ] = Field( 3125 ..., 3126 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3127 title="Underlying Partition Router", 3128 ) 3129 deduplicate: Optional[bool] = Field( 3130 True, 3131 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3132 title="Deduplicate Partitions", 3133 ) 3134 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3135 3136 3137class HttpComponentsResolver(BaseModel): 3138 type: Literal["HttpComponentsResolver"] 3139 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3140 ..., 3141 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3142 title="Retriever", 3143 ) 3144 components_mapping: List[ComponentMappingDefinition] 3145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3146 3147 3148class DynamicDeclarativeStream(BaseModel): 3149 type: Literal["DynamicDeclarativeStream"] 3150 name: Optional[str] = Field( 3151 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3152 ) 3153 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3154 ..., description="Reference to the stream template.", title="Stream Template" 3155 ) 3156 components_resolver: Union[ 3157 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3158 ] = Field( 3159 ..., 3160 description="Component resolve and populates stream templates with components values.", 3161 title="Components Resolver", 3162 ) 3163 use_parent_parameters: Optional[bool] = Field( 3164 True, 3165 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3166 title="Use Parent Parameters", 3167 ) 3168 3169 3170ComplexFieldType.update_forward_refs() 3171GzipDecoder.update_forward_refs() 3172CompositeErrorHandler.update_forward_refs() 3173DeclarativeSource1.update_forward_refs() 3174DeclarativeSource2.update_forward_refs() 3175SelectiveAuthenticator.update_forward_refs() 3176ConditionalStreams.update_forward_refs() 3177FileUploader.update_forward_refs() 3178DeclarativeStream.update_forward_refs() 3179SessionTokenAuthenticator.update_forward_refs() 3180HttpRequester.update_forward_refs() 3181DynamicSchemaLoader.update_forward_refs() 3182ParentStreamConfig.update_forward_refs() 3183PropertiesFromEndpoint.update_forward_refs() 3184SimpleRetriever.update_forward_refs() 3185AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 0, 63 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 64 title="Stream Count", 65 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
110class CursorPagination(BaseModel): 111 type: Literal["CursorPagination"] 112 cursor_value: str = Field( 113 ..., 114 description="Value of the cursor defining the next page to fetch.", 115 examples=[ 116 "{{ headers.link.next.cursor }}", 117 "{{ last_record['key'] }}", 118 "{{ response['nextPage'] }}", 119 ], 120 title="Cursor Value", 121 ) 122 page_size: Optional[Union[int, str]] = Field( 123 None, 124 description="The number of records to include in each pages.", 125 examples=[100, "{{ config['page_size'] }}"], 126 title="Page Size", 127 ) 128 stop_condition: Optional[str] = Field( 129 None, 130 description="Template string evaluating when to stop paginating.", 131 examples=[ 132 "{{ response.data.has_more is false }}", 133 "{{ 'next' not in headers['link'] }}", 134 ], 135 title="Stop Condition", 136 ) 137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
140class CustomAuthenticator(BaseModel): 141 class Config: 142 extra = Extra.allow 143 144 type: Literal["CustomAuthenticator"] 145 class_name: str = Field( 146 ..., 147 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 148 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 149 title="Class Name", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
154class CustomBackoffStrategy(BaseModel): 155 class Config: 156 extra = Extra.allow 157 158 type: Literal["CustomBackoffStrategy"] 159 class_name: str = Field( 160 ..., 161 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 162 examples=["source_railz.components.MyCustomBackoffStrategy"], 163 title="Class Name", 164 ) 165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
168class CustomErrorHandler(BaseModel): 169 class Config: 170 extra = Extra.allow 171 172 type: Literal["CustomErrorHandler"] 173 class_name: str = Field( 174 ..., 175 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 176 examples=["source_railz.components.MyCustomErrorHandler"], 177 title="Class Name", 178 ) 179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
182class CustomPaginationStrategy(BaseModel): 183 class Config: 184 extra = Extra.allow 185 186 type: Literal["CustomPaginationStrategy"] 187 class_name: str = Field( 188 ..., 189 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 190 examples=["source_railz.components.MyCustomPaginationStrategy"], 191 title="Class Name", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
196class CustomRecordExtractor(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomRecordExtractor"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomRecordExtractor"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
210class CustomRecordFilter(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordFilter"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomCustomRecordFilter"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
224class CustomRequester(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRequester"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomRecordExtractor"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
238class CustomRetriever(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRetriever"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRetriever"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
252class CustomPartitionRouter(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomPartitionRouter"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomPartitionRouter"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
266class CustomSchemaLoader(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomSchemaLoader"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomSchemaLoader"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
280class CustomSchemaNormalization(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaNormalization"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 288 examples=[ 289 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 290 ], 291 title="Class Name", 292 ) 293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
296class CustomStateMigration(BaseModel): 297 class Config: 298 extra = Extra.allow 299 300 type: Literal["CustomStateMigration"] 301 class_name: str = Field( 302 ..., 303 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 304 examples=["source_railz.components.MyCustomStateMigration"], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
310class CustomTransformation(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomTransformation"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomTransformation"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
341class Algorithm(Enum): 342 HS256 = "HS256" 343 HS384 = "HS384" 344 HS512 = "HS512" 345 ES256 = "ES256" 346 ES256K = "ES256K" 347 ES384 = "ES384" 348 ES512 = "ES512" 349 RS256 = "RS256" 350 RS384 = "RS384" 351 RS512 = "RS512" 352 PS256 = "PS256" 353 PS384 = "PS384" 354 PS512 = "PS512" 355 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
358class JwtHeaders(BaseModel): 359 class Config: 360 extra = Extra.forbid 361 362 kid: Optional[str] = Field( 363 None, 364 description="Private key ID for user account.", 365 examples=["{{ config['kid'] }}"], 366 title="Key Identifier", 367 ) 368 typ: Optional[str] = Field( 369 "JWT", 370 description="The media type of the complete JWT.", 371 examples=["JWT"], 372 title="Type", 373 ) 374 cty: Optional[str] = Field( 375 None, 376 description="Content type of JWT header.", 377 examples=["JWT"], 378 title="Content Type", 379 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
382class JwtPayload(BaseModel): 383 class Config: 384 extra = Extra.forbid 385 386 iss: Optional[str] = Field( 387 None, 388 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 389 examples=["{{ config['iss'] }}"], 390 title="Issuer", 391 ) 392 sub: Optional[str] = Field( 393 None, 394 description="The subject of the JWT. Commonly defined by the API.", 395 title="Subject", 396 ) 397 aud: Optional[str] = Field( 398 None, 399 description="The recipient that the JWT is intended for. Commonly defined by the API.", 400 examples=["appstoreconnect-v1"], 401 title="Audience", 402 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
405class RefreshTokenUpdater(BaseModel): 406 refresh_token_name: Optional[str] = Field( 407 "refresh_token", 408 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 409 examples=["refresh_token"], 410 title="Refresh Token Property Name", 411 ) 412 access_token_config_path: Optional[List[str]] = Field( 413 ["credentials", "access_token"], 414 description="Config path to the access token. Make sure the field actually exists in the config.", 415 examples=[["credentials", "access_token"], ["access_token"]], 416 title="Config Path To Access Token", 417 ) 418 refresh_token_config_path: Optional[List[str]] = Field( 419 ["credentials", "refresh_token"], 420 description="Config path to the access token. Make sure the field actually exists in the config.", 421 examples=[["credentials", "refresh_token"], ["refresh_token"]], 422 title="Config Path To Refresh Token", 423 ) 424 token_expiry_date_config_path: Optional[List[str]] = Field( 425 ["credentials", "token_expiry_date"], 426 description="Config path to the expiry date. Make sure actually exists in the config.", 427 examples=[["credentials", "token_expiry_date"]], 428 title="Config Path To Expiry Date", 429 ) 430 refresh_token_error_status_codes: Optional[List[int]] = Field( 431 [], 432 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 433 examples=[[400, 500]], 434 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 435 ) 436 refresh_token_error_key: Optional[str] = Field( 437 "", 438 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 439 examples=["error"], 440 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 441 ) 442 refresh_token_error_values: Optional[List[str]] = Field( 443 [], 444 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 445 examples=[["invalid_grant", "invalid_permissions"]], 446 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 447 )
class
Rate(pydantic.v1.main.BaseModel):
450class Rate(BaseModel): 451 class Config: 452 extra = Extra.allow 453 454 limit: Union[int, str] = Field( 455 ..., 456 description="The maximum number of calls allowed within the interval.", 457 title="Limit", 458 ) 459 interval: str = Field( 460 ..., 461 description="The time interval for the rate limit.", 462 examples=["PT1H", "P1D"], 463 title="Interval", 464 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
467class HttpRequestRegexMatcher(BaseModel): 468 class Config: 469 extra = Extra.allow 470 471 method: Optional[str] = Field( 472 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 473 ) 474 url_base: Optional[str] = Field( 475 None, 476 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 477 title="URL Base", 478 ) 479 url_path_pattern: Optional[str] = Field( 480 None, 481 description="A regular expression pattern to match the URL path.", 482 title="URL Path Pattern", 483 ) 484 params: Optional[Dict[str, Any]] = Field( 485 None, description="The query parameters to match.", title="Parameters" 486 ) 487 headers: Optional[Dict[str, Any]] = Field( 488 None, description="The headers to match.", title="Headers" 489 ) 490 weight: Optional[Union[int, str]] = Field( 491 None, 492 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 493 title="Weight", 494 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
497class DpathExtractor(BaseModel): 498 type: Literal["DpathExtractor"] 499 field_path: List[str] = Field( 500 ..., 501 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 502 examples=[ 503 ["data"], 504 ["data", "records"], 505 ["data", "{{ parameters.name }}"], 506 ["data", "*", "record"], 507 ], 508 title="Field Path", 509 ) 510 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
518class ExponentialBackoffStrategy(BaseModel): 519 type: Literal["ExponentialBackoffStrategy"] 520 factor: Optional[Union[float, str]] = Field( 521 5, 522 description="Multiplicative constant applied on each retry.", 523 examples=[5, 5.5, "10"], 524 title="Factor", 525 ) 526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
529class GroupByKeyMergeStrategy(BaseModel): 530 type: Literal["GroupByKeyMergeStrategy"] 531 key: Union[str, List[str]] = Field( 532 ..., 533 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 534 examples=["id", ["parent_id", "end_date"]], 535 title="Key", 536 ) 537 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
549class Action(Enum): 550 SUCCESS = "SUCCESS" 551 FAIL = "FAIL" 552 RETRY = "RETRY" 553 IGNORE = "IGNORE" 554 RESET_PAGINATION = "RESET_PAGINATION" 555 RATE_LIMITED = "RATE_LIMITED" 556 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
559class FailureType(Enum): 560 system_error = "system_error" 561 config_error = "config_error" 562 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
565class HttpResponseFilter(BaseModel): 566 type: Literal["HttpResponseFilter"] 567 action: Optional[Action] = Field( 568 None, 569 description="Action to execute if a response matches the filter.", 570 examples=[ 571 "SUCCESS", 572 "FAIL", 573 "RETRY", 574 "IGNORE", 575 "RESET_PAGINATION", 576 "RATE_LIMITED", 577 "REFRESH_TOKEN_THEN_RETRY", 578 ], 579 title="Action", 580 ) 581 failure_type: Optional[FailureType] = Field( 582 None, 583 description="Failure type of traced exception if a response matches the filter.", 584 examples=["system_error", "config_error", "transient_error"], 585 title="Failure Type", 586 ) 587 error_message: Optional[str] = Field( 588 None, 589 description="Error Message to display if the response matches the filter.", 590 title="Error Message", 591 ) 592 error_message_contains: Optional[str] = Field( 593 None, 594 description="Match the response if its error message contains the substring.", 595 example=["This API operation is not enabled for this site"], 596 title="Error Message Substring", 597 ) 598 http_codes: Optional[List[int]] = Field( 599 None, 600 description="Match the response if its HTTP code is included in this list.", 601 examples=[[420, 429], [500]], 602 title="HTTP Codes", 603 unique_items=True, 604 ) 605 predicate: Optional[str] = Field( 606 None, 607 description="Match the response if the predicate evaluates to true.", 608 examples=[ 609 "{{ 'Too much requests' in response }}", 610 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 611 ], 612 title="Predicate", 613 ) 614 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
617class ComplexFieldType(BaseModel): 618 field_type: str 619 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
622class TypesMap(BaseModel): 623 target_type: Union[str, List[str], ComplexFieldType] 624 current_type: Union[str, List[str]] 625 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
628class SchemaTypeIdentifier(BaseModel): 629 type: Optional[Literal["SchemaTypeIdentifier"]] = None 630 schema_pointer: Optional[List[str]] = Field( 631 [], 632 description="List of nested fields defining the schema field path to extract. Defaults to [].", 633 title="Schema Path", 634 ) 635 key_pointer: List[str] = Field( 636 ..., 637 description="List of potentially nested fields describing the full path of the field key to extract.", 638 title="Key Path", 639 ) 640 type_pointer: Optional[List[str]] = Field( 641 None, 642 description="List of potentially nested fields describing the full path of the field type to extract.", 643 title="Type Path", 644 ) 645 types_mapping: Optional[List[TypesMap]] = None 646 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
649class InlineSchemaLoader(BaseModel): 650 type: Literal["InlineSchemaLoader"] 651 schema_: Optional[Dict[str, Any]] = Field( 652 None, 653 alias="schema", 654 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 655 title="Schema", 656 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
659class JsonFileSchemaLoader(BaseModel): 660 type: Literal["JsonFileSchemaLoader"] 661 file_path: Optional[str] = Field( 662 None, 663 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 664 example=["./schemas/users.json"], 665 title="File Path", 666 ) 667 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
688class FlattenFields(BaseModel): 689 type: Literal["FlattenFields"] 690 flatten_lists: Optional[bool] = Field( 691 True, 692 description="Whether to flatten lists or leave it as is. Default is True.", 693 title="Flatten Lists", 694 ) 695 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
698class KeyTransformation(BaseModel): 699 type: Literal["KeyTransformation"] 700 prefix: Optional[str] = Field( 701 None, 702 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 703 examples=["flattened_"], 704 title="Key Prefix", 705 ) 706 suffix: Optional[str] = Field( 707 None, 708 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 709 examples=["_flattened"], 710 title="Key Suffix", 711 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
714class DpathFlattenFields(BaseModel): 715 type: Literal["DpathFlattenFields"] 716 field_path: List[str] = Field( 717 ..., 718 description="A path to field that needs to be flattened.", 719 examples=[["data"], ["data", "*", "field"]], 720 title="Field Path", 721 ) 722 delete_origin_value: Optional[bool] = Field( 723 None, 724 description="Whether to delete the origin value or keep it. Default is False.", 725 title="Delete Origin Value", 726 ) 727 replace_record: Optional[bool] = Field( 728 None, 729 description="Whether to replace the origin record or not. Default is False.", 730 title="Replace Origin Record", 731 ) 732 key_transformation: Optional[KeyTransformation] = Field( 733 None, 734 description="Transformation for object keys. If not provided, original key will be used.", 735 title="Key transformation", 736 ) 737 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
740class KeysReplace(BaseModel): 741 type: Literal["KeysReplace"] 742 old: str = Field( 743 ..., 744 description="Old value to replace.", 745 examples=[ 746 " ", 747 "{{ record.id }}", 748 "{{ config['id'] }}", 749 "{{ stream_slice['id'] }}", 750 ], 751 title="Old value", 752 ) 753 new: str = Field( 754 ..., 755 description="New value to set.", 756 examples=[ 757 "_", 758 "{{ record.id }}", 759 "{{ config['id'] }}", 760 "{{ stream_slice['id'] }}", 761 ], 762 title="New value", 763 ) 764 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
775class CustomDecoder(BaseModel): 776 class Config: 777 extra = Extra.allow 778 779 type: Literal["CustomDecoder"] 780 class_name: str = Field( 781 ..., 782 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 783 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 784 title="Class Name", 785 ) 786 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
789class MinMaxDatetime(BaseModel): 790 type: Literal["MinMaxDatetime"] 791 datetime: str = Field( 792 ..., 793 description="Datetime value.", 794 examples=[ 795 "2021-01-01", 796 "2021-01-01T00:00:00Z", 797 "{{ config['start_time'] }}", 798 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 799 ], 800 title="Datetime", 801 ) 802 datetime_format: Optional[str] = Field( 803 "", 804 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 805 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 806 title="Datetime Format", 807 ) 808 max_datetime: Optional[str] = Field( 809 None, 810 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 811 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 812 title="Max Datetime", 813 ) 814 min_datetime: Optional[str] = Field( 815 None, 816 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 817 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 818 title="Min Datetime", 819 ) 820 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
850class OauthConnectorInputSpecification(BaseModel): 851 class Config: 852 extra = Extra.allow 853 854 consent_url: str = Field( 855 ..., 856 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 857 examples=[ 858 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 859 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 860 ], 861 title="Consent URL", 862 ) 863 scope: Optional[str] = Field( 864 None, 865 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 866 examples=["user:read user:read_orders workspaces:read"], 867 title="Scopes", 868 ) 869 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 870 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 871 # The CDK schema defines the manifest contract; the platform reads these fields 872 # during the OAuth consent flow to build the authorization URL. 873 scopes: Optional[List[OAuthScope]] = Field( 874 None, 875 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 876 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 877 title="Scopes", 878 ) 879 optional_scopes: Optional[List[OAuthScope]] = Field( 880 None, 881 description="Optional OAuth scope objects that may or may not be granted.", 882 examples=[[{"scope": "admin:read"}]], 883 title="Optional Scopes", 884 ) 885 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 886 ScopesJoinStrategy.space, 887 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 888 title="Scopes Join Strategy", 889 ) 890 access_token_url: str = Field( 891 ..., 892 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 893 examples=[ 894 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 895 ], 896 title="Access Token URL", 897 ) 898 access_token_headers: Optional[Dict[str, Any]] = Field( 899 None, 900 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 901 examples=[ 902 { 903 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 904 } 905 ], 906 title="Access Token Headers", 907 ) 908 access_token_params: Optional[Dict[str, Any]] = Field( 909 None, 910 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 911 examples=[ 912 { 913 "{{ auth_code_key }}": "{{ auth_code_value }}", 914 "{{ client_id_key }}": "{{ client_id_value }}", 915 "{{ client_secret_key }}": "{{ client_secret_value }}", 916 } 917 ], 918 title="Access Token Query Params (Json Encoded)", 919 ) 920 extract_output: Optional[List[str]] = Field( 921 None, 922 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 923 examples=[["access_token", "refresh_token", "other_field"]], 924 title="Extract Output", 925 ) 926 state: Optional[State] = Field( 927 None, 928 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 929 examples=[{"min": 7, "max": 128}], 930 title="Configurable State Query Param", 931 ) 932 client_id_key: Optional[str] = Field( 933 None, 934 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 935 examples=["my_custom_client_id_key_name"], 936 title="Client ID Key Override", 937 ) 938 client_secret_key: Optional[str] = Field( 939 None, 940 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 941 examples=["my_custom_client_secret_key_name"], 942 title="Client Secret Key Override", 943 ) 944 scope_key: Optional[str] = Field( 945 None, 946 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 947 examples=["my_custom_scope_key_key_name"], 948 title="Scopes Key Override", 949 ) 950 state_key: Optional[str] = Field( 951 None, 952 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 953 examples=["my_custom_state_key_key_name"], 954 title="State Key Override", 955 ) 956 auth_code_key: Optional[str] = Field( 957 None, 958 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 959 examples=["my_custom_auth_code_key_name"], 960 title="Auth Code Key Override", 961 ) 962 redirect_uri_key: Optional[str] = Field( 963 None, 964 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 965 examples=["my_custom_redirect_uri_key_name"], 966 title="Redirect URI Key Override", 967 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
970class OAuthConfigSpecification(BaseModel): 971 class Config: 972 extra = Extra.allow 973 974 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 975 None, 976 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 977 examples=[ 978 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 979 { 980 "app_id": { 981 "type": "string", 982 "path_in_connector_config": ["info", "app_id"], 983 } 984 }, 985 ], 986 title="OAuth user input", 987 ) 988 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 989 None, 990 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 991 title="DeclarativeOAuth Connector Specification", 992 ) 993 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 994 None, 995 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 996 examples=[ 997 { 998 "refresh_token": { 999 "type": "string,", 1000 "path_in_connector_config": ["credentials", "refresh_token"], 1001 } 1002 } 1003 ], 1004 title="OAuth output specification", 1005 ) 1006 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1007 None, 1008 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1009 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1010 title="OAuth input specification", 1011 ) 1012 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1013 None, 1014 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1015 examples=[ 1016 { 1017 "client_id": { 1018 "type": "string,", 1019 "path_in_connector_config": ["credentials", "client_id"], 1020 }, 1021 "client_secret": { 1022 "type": "string,", 1023 "path_in_connector_config": ["credentials", "client_secret"], 1024 }, 1025 } 1026 ], 1027 title="OAuth server output specification", 1028 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1031class OffsetIncrement(BaseModel): 1032 type: Literal["OffsetIncrement"] 1033 page_size: Optional[Union[int, str]] = Field( 1034 None, 1035 description="The number of records to include in each pages.", 1036 examples=[100, "{{ config['page_size'] }}"], 1037 title="Limit", 1038 ) 1039 inject_on_first_request: Optional[bool] = Field( 1040 False, 1041 description="Using the `offset` with value `0` during the first request", 1042 title="Inject Offset on First Request", 1043 ) 1044 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1047class PageIncrement(BaseModel): 1048 type: Literal["PageIncrement"] 1049 page_size: Optional[Union[int, str]] = Field( 1050 None, 1051 description="The number of records to include in each pages.", 1052 examples=[100, "100", "{{ config['page_size'] }}"], 1053 title="Page Size", 1054 ) 1055 start_from_page: Optional[int] = Field( 1056 0, 1057 description="Index of the first page to request.", 1058 examples=[0, 1], 1059 title="Start From Page", 1060 ) 1061 inject_on_first_request: Optional[bool] = Field( 1062 False, 1063 description="Using the `page number` with value defined by `start_from_page` during the first request", 1064 title="Inject Page Number on First Request", 1065 ) 1066 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1069class PrimaryKey(BaseModel): 1070 __root__: Union[str, List[str], List[List[str]]] = Field( 1071 ..., 1072 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1073 examples=["id", ["code", "type"]], 1074 title="Primary Key", 1075 )
class
PropertyLimitType(enum.Enum):
1078class PropertyLimitType(Enum): 1079 characters = "characters" 1080 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1083class PropertyChunking(BaseModel): 1084 type: Literal["PropertyChunking"] 1085 property_limit_type: PropertyLimitType = Field( 1086 ..., 1087 description="The type used to determine the maximum number of properties per chunk", 1088 title="Property Limit Type", 1089 ) 1090 property_limit: Optional[int] = Field( 1091 None, 1092 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1093 title="Property Limit", 1094 ) 1095 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1096 None, 1097 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1098 title="Record Merge Strategy", 1099 ) 1100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1103class RecordFilter(BaseModel): 1104 type: Literal["RecordFilter"] 1105 condition: Optional[str] = Field( 1106 "", 1107 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1108 examples=[ 1109 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1110 "{{ record.status in ['active', 'expired'] }}", 1111 ], 1112 title="Condition", 1113 ) 1114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1122class RemoveFields(BaseModel): 1123 type: Literal["RemoveFields"] 1124 condition: Optional[str] = Field( 1125 "", 1126 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1127 examples=[ 1128 "{{ property|string == '' }}", 1129 "{{ property is integer }}", 1130 "{{ property|length > 5 }}", 1131 "{{ property == 'some_string_to_match' }}", 1132 ], 1133 ) 1134 field_pointers: List[List[str]] = Field( 1135 ..., 1136 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1137 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1138 title="Field Paths", 1139 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1146class InjectInto(Enum): 1147 request_parameter = "request_parameter" 1148 header = "header" 1149 body_data = "body_data" 1150 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1153class RequestOption(BaseModel): 1154 type: Literal["RequestOption"] 1155 inject_into: InjectInto = Field( 1156 ..., 1157 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1158 examples=["request_parameter", "header", "body_data", "body_json"], 1159 title="Inject Into", 1160 ) 1161 field_name: Optional[str] = Field( 1162 None, 1163 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1164 examples=["segment_id"], 1165 title="Field Name", 1166 ) 1167 field_path: Optional[List[str]] = Field( 1168 None, 1169 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1170 examples=[["data", "viewer", "id"]], 1171 title="Field Path", 1172 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1182class LegacySessionTokenAuthenticator(BaseModel): 1183 type: Literal["LegacySessionTokenAuthenticator"] 1184 header: str = Field( 1185 ..., 1186 description="The name of the session token header that will be injected in the request", 1187 examples=["X-Session"], 1188 title="Session Request Header", 1189 ) 1190 login_url: str = Field( 1191 ..., 1192 description="Path of the login URL (do not include the base URL)", 1193 examples=["session"], 1194 title="Login Path", 1195 ) 1196 session_token: Optional[str] = Field( 1197 None, 1198 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1199 example=["{{ config['session_token'] }}"], 1200 title="Session Token", 1201 ) 1202 session_token_response_key: str = Field( 1203 ..., 1204 description="Name of the key of the session token to be extracted from the response", 1205 examples=["id"], 1206 title="Response Token Response Key", 1207 ) 1208 username: Optional[str] = Field( 1209 None, 1210 description="Username used to authenticate and obtain a session token", 1211 examples=[" {{ config['username'] }}"], 1212 title="Username", 1213 ) 1214 password: Optional[str] = Field( 1215 "", 1216 description="Password used to authenticate and obtain a session token", 1217 examples=["{{ config['password'] }}", ""], 1218 title="Password", 1219 ) 1220 validate_session_url: str = Field( 1221 ..., 1222 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1223 examples=["user/current"], 1224 title="Validate Session Path", 1225 ) 1226 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1254class ValueType(Enum): 1255 string = "string" 1256 number = "number" 1257 integer = "integer" 1258 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1261class WaitTimeFromHeader(BaseModel): 1262 type: Literal["WaitTimeFromHeader"] 1263 header: str = Field( 1264 ..., 1265 description="The name of the response header defining how long to wait before retrying.", 1266 examples=["Retry-After"], 1267 title="Response Header Name", 1268 ) 1269 regex: Optional[str] = Field( 1270 None, 1271 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1272 examples=["([-+]?\\d+)"], 1273 title="Extraction Regex", 1274 ) 1275 max_waiting_time_in_seconds: Optional[float] = Field( 1276 None, 1277 description="Given the value extracted from the header is greater than this value, stop the stream.", 1278 examples=[3600], 1279 title="Max Waiting Time in Seconds", 1280 ) 1281 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1284class WaitUntilTimeFromHeader(BaseModel): 1285 type: Literal["WaitUntilTimeFromHeader"] 1286 header: str = Field( 1287 ..., 1288 description="The name of the response header defining how long to wait before retrying.", 1289 examples=["wait_time"], 1290 title="Response Header", 1291 ) 1292 min_wait: Optional[Union[float, str]] = Field( 1293 None, 1294 description="Minimum time to wait before retrying.", 1295 examples=[10, "60"], 1296 title="Minimum Wait Time", 1297 ) 1298 regex: Optional[str] = Field( 1299 None, 1300 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1301 examples=["([-+]?\\d+)"], 1302 title="Extraction Regex", 1303 ) 1304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1307class ComponentMappingDefinition(BaseModel): 1308 type: Literal["ComponentMappingDefinition"] 1309 field_path: List[str] = Field( 1310 ..., 1311 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1312 examples=[ 1313 ["name"], 1314 ["retriever", "requester", "url"], 1315 ["retriever", "requester", "{{ components_values.field }}"], 1316 ["*", "**", "name"], 1317 ], 1318 title="Field Path", 1319 ) 1320 value: str = Field( 1321 ..., 1322 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1323 examples=[ 1324 "{{ components_values['updates'] }}", 1325 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1326 "{{ config['segment_id'] }}", 1327 "{{ stream_slice['parent_id'] }}", 1328 "{{ stream_slice['extra_fields']['name'] }}", 1329 ], 1330 title="Value", 1331 ) 1332 value_type: Optional[ValueType] = Field( 1333 None, 1334 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1335 title="Value Type", 1336 ) 1337 create_or_update: Optional[bool] = Field( 1338 False, 1339 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1340 title="Create or Update", 1341 ) 1342 condition: Optional[str] = Field( 1343 None, 1344 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1345 examples=[ 1346 "{{ components_values.get('cursor_field', None) }}", 1347 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1348 ], 1349 title="Condition", 1350 ) 1351 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1354class StreamConfig(BaseModel): 1355 type: Literal["StreamConfig"] 1356 configs_pointer: List[str] = Field( 1357 ..., 1358 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1359 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1360 title="Configs Pointer", 1361 ) 1362 default_values: Optional[List[Dict[str, Any]]] = Field( 1363 None, 1364 description="A list of default values, each matching the structure expected from the parsed component value.", 1365 title="Default Values", 1366 ) 1367 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1370class ConfigComponentsResolver(BaseModel): 1371 type: Literal["ConfigComponentsResolver"] 1372 stream_config: Union[List[StreamConfig], StreamConfig] 1373 components_mapping: List[ComponentMappingDefinition] 1374 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1377class StreamParametersDefinition(BaseModel): 1378 type: Literal["StreamParametersDefinition"] 1379 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1380 ..., 1381 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1382 examples=[ 1383 [ 1384 { 1385 "name": "test stream", 1386 "$parameters": {"entity": "test entity"}, 1387 "primary_key": "test key", 1388 } 1389 ] 1390 ], 1391 title="Stream Parameters", 1392 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1395class ParametrizedComponentsResolver(BaseModel): 1396 type: Literal["ParametrizedComponentsResolver"] 1397 stream_parameters: StreamParametersDefinition 1398 components_mapping: List[ComponentMappingDefinition] 1399 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1424class ValidateAdheresToSchema(BaseModel): 1425 type: Literal["ValidateAdheresToSchema"] 1426 base_schema: Union[str, Dict[str, Any]] = Field( 1427 ..., 1428 description="The base JSON schema against which the user-provided schema will be validated.", 1429 examples=[ 1430 "{{ config['report_validation_schema'] }}", 1431 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1432 { 1433 "$schema": "http://json-schema.org/draft-07/schema#", 1434 "title": "Person", 1435 "type": "object", 1436 "properties": { 1437 "name": {"type": "string", "description": "The person's name"}, 1438 "age": { 1439 "type": "integer", 1440 "minimum": 0, 1441 "description": "The person's age", 1442 }, 1443 }, 1444 "required": ["name", "age"], 1445 }, 1446 ], 1447 title="Base JSON Schema", 1448 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1451class CustomValidationStrategy(BaseModel): 1452 class Config: 1453 extra = Extra.allow 1454 1455 type: Literal["CustomValidationStrategy"] 1456 class_name: str = Field( 1457 ..., 1458 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1459 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1460 title="Class Name", 1461 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1464class ConfigRemapField(BaseModel): 1465 type: Literal["ConfigRemapField"] 1466 map: Union[Dict[str, Any], str] = Field( 1467 ..., 1468 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1469 examples=[ 1470 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1471 "{{ config['status_mapping'] }}", 1472 ], 1473 title="Value Mapping", 1474 ) 1475 field_path: List[str] = Field( 1476 ..., 1477 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1478 examples=[ 1479 ["status"], 1480 ["data", "status"], 1481 ["data", "{{ config.name }}", "status"], 1482 ["data", "*", "status"], 1483 ], 1484 title="Field Path", 1485 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1488class ConfigRemoveFields(BaseModel): 1489 type: Literal["ConfigRemoveFields"] 1490 field_pointers: List[List[str]] = Field( 1491 ..., 1492 description="A list of field pointers to be removed from the config.", 1493 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1494 title="Field Pointers", 1495 ) 1496 condition: Optional[str] = Field( 1497 "", 1498 description="Fields will be removed if expression is evaluated to True.", 1499 examples=[ 1500 "{{ config['environemnt'] == 'sandbox' }}", 1501 "{{ property is integer }}", 1502 "{{ property|length > 5 }}", 1503 "{{ property == 'some_string_to_match' }}", 1504 ], 1505 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1508class CustomConfigTransformation(BaseModel): 1509 type: Literal["CustomConfigTransformation"] 1510 class_name: str = Field( 1511 ..., 1512 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1513 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1514 ) 1515 parameters: Optional[Dict[str, Any]] = Field( 1516 None, 1517 alias="$parameters", 1518 description="Additional parameters to be passed to the custom config transformation.", 1519 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1522class AddedFieldDefinition(BaseModel): 1523 type: Literal["AddedFieldDefinition"] 1524 path: List[str] = Field( 1525 ..., 1526 description="List of strings defining the path where to add the value on the record.", 1527 examples=[["segment_id"], ["metadata", "segment_id"]], 1528 title="Path", 1529 ) 1530 value: str = Field( 1531 ..., 1532 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1533 examples=[ 1534 "{{ record['updates'] }}", 1535 "{{ record['MetaData']['LastUpdatedTime'] }}", 1536 "{{ stream_partition['segment_id'] }}", 1537 ], 1538 title="Value", 1539 ) 1540 value_type: Optional[ValueType] = Field( 1541 None, 1542 description="Type of the value. If not specified, the type will be inferred from the value.", 1543 title="Value Type", 1544 ) 1545 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1548class AddFields(BaseModel): 1549 type: Literal["AddFields"] 1550 fields: List[AddedFieldDefinition] = Field( 1551 ..., 1552 description="List of transformations (path and corresponding value) that will be added to the record.", 1553 title="Fields", 1554 ) 1555 condition: Optional[str] = Field( 1556 "", 1557 description="Fields will be added if expression is evaluated to True.", 1558 examples=[ 1559 "{{ property|string == '' }}", 1560 "{{ property is integer }}", 1561 "{{ property|length > 5 }}", 1562 "{{ property == 'some_string_to_match' }}", 1563 ], 1564 ) 1565 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1568class ApiKeyAuthenticator(BaseModel): 1569 type: Literal["ApiKeyAuthenticator"] 1570 api_token: Optional[str] = Field( 1571 None, 1572 description="The API key to inject in the request. Fill it in the user inputs.", 1573 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1574 title="API Key", 1575 ) 1576 header: Optional[str] = Field( 1577 None, 1578 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1579 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1580 title="Header Name", 1581 ) 1582 inject_into: Optional[RequestOption] = Field( 1583 None, 1584 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1585 examples=[ 1586 {"inject_into": "header", "field_name": "Authorization"}, 1587 {"inject_into": "request_parameter", "field_name": "authKey"}, 1588 ], 1589 title="Inject API Key Into Outgoing HTTP Request", 1590 ) 1591 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1594class AuthFlow(BaseModel): 1595 auth_flow_type: Optional[AuthFlowType] = Field( 1596 None, description="The type of auth to use", title="Auth flow type" 1597 ) 1598 predicate_key: Optional[List[str]] = Field( 1599 None, 1600 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1601 examples=[["credentials", "auth_type"]], 1602 title="Predicate key", 1603 ) 1604 predicate_value: Optional[str] = Field( 1605 None, 1606 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1607 examples=["Oauth"], 1608 title="Predicate value", 1609 ) 1610 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1613class CheckStream(BaseModel): 1614 type: Literal["CheckStream"] 1615 stream_names: Optional[List[str]] = Field( 1616 None, 1617 description="Names of the streams to try reading from when running a check operation.", 1618 examples=[["users"], ["users", "contacts"]], 1619 title="Stream Names", 1620 ) 1621 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1624class IncrementingCountCursor(BaseModel): 1625 type: Literal["IncrementingCountCursor"] 1626 cursor_field: str = Field( 1627 ..., 1628 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1629 examples=["created_at", "{{ config['record_cursor'] }}"], 1630 title="Cursor Field", 1631 ) 1632 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1633 None, 1634 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1635 title="Allow Catalog Defined Cursor Field", 1636 ) 1637 start_value: Optional[Union[str, int]] = Field( 1638 None, 1639 description="The value that determines the earliest record that should be synced.", 1640 examples=[0, "{{ config['start_value'] }}"], 1641 title="Start Value", 1642 ) 1643 start_value_option: Optional[RequestOption] = Field( 1644 None, 1645 description="Optionally configures how the start value will be sent in requests to the source API.", 1646 title="Inject Start Value Into Outgoing HTTP Request", 1647 ) 1648 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1651class DatetimeBasedCursor(BaseModel): 1652 type: Literal["DatetimeBasedCursor"] 1653 clamping: Optional[Clamping] = Field( 1654 None, 1655 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1656 title="Date Range Clamping", 1657 ) 1658 cursor_field: str = Field( 1659 ..., 1660 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1661 examples=["created_at", "{{ config['record_cursor'] }}"], 1662 title="Cursor Field", 1663 ) 1664 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1665 None, 1666 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1667 title="Allow Catalog Defined Cursor Field", 1668 ) 1669 cursor_datetime_formats: Optional[List[str]] = Field( 1670 None, 1671 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1672 examples=[ 1673 "%Y-%m-%d", 1674 "%Y-%m-%d %H:%M:%S", 1675 "%Y-%m-%dT%H:%M:%S", 1676 "%Y-%m-%dT%H:%M:%SZ", 1677 "%Y-%m-%dT%H:%M:%S%z", 1678 "%Y-%m-%dT%H:%M:%S.%fZ", 1679 "%Y-%m-%dT%H:%M:%S.%f%z", 1680 "%Y-%m-%d %H:%M:%S.%f+00:00", 1681 "%s", 1682 "%ms", 1683 ], 1684 title="Cursor Datetime Formats", 1685 ) 1686 start_datetime: Union[MinMaxDatetime, str] = Field( 1687 ..., 1688 description="The datetime that determines the earliest record that should be synced.", 1689 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1690 title="Start Datetime", 1691 ) 1692 start_time_option: Optional[RequestOption] = Field( 1693 None, 1694 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1695 title="Inject Start Time Into Outgoing HTTP Request", 1696 ) 1697 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1698 None, 1699 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1700 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1701 title="End Datetime", 1702 ) 1703 end_time_option: Optional[RequestOption] = Field( 1704 None, 1705 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1706 title="Inject End Time Into Outgoing HTTP Request", 1707 ) 1708 datetime_format: str = Field( 1709 ..., 1710 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1711 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1712 title="Outgoing Datetime Format", 1713 ) 1714 cursor_granularity: Optional[str] = Field( 1715 None, 1716 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1717 examples=["PT1S"], 1718 title="Cursor Granularity", 1719 ) 1720 is_data_feed: Optional[bool] = Field( 1721 None, 1722 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1723 title="Data Feed API", 1724 ) 1725 is_client_side_incremental: Optional[bool] = Field( 1726 None, 1727 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1728 title="Client-side Incremental Filtering", 1729 ) 1730 is_compare_strictly: Optional[bool] = Field( 1731 False, 1732 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1733 title="Strict Start-End Time Comparison", 1734 ) 1735 global_substream_cursor: Optional[bool] = Field( 1736 False, 1737 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1738 title="Global Substream Cursor", 1739 ) 1740 lookback_window: Optional[str] = Field( 1741 None, 1742 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1743 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1744 title="Lookback Window", 1745 ) 1746 partition_field_end: Optional[str] = Field( 1747 None, 1748 description="Name of the partition start time field.", 1749 examples=["ending_time"], 1750 title="Partition Field End", 1751 ) 1752 partition_field_start: Optional[str] = Field( 1753 None, 1754 description="Name of the partition end time field.", 1755 examples=["starting_time"], 1756 title="Partition Field Start", 1757 ) 1758 step: Optional[str] = Field( 1759 None, 1760 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1761 examples=["P1W", "{{ config['step_increment'] }}"], 1762 title="Step", 1763 ) 1764 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1767class JwtAuthenticator(BaseModel): 1768 type: Literal["JwtAuthenticator"] 1769 secret_key: str = Field( 1770 ..., 1771 description="Secret used to sign the JSON web token.", 1772 examples=["{{ config['secret_key'] }}"], 1773 title="Secret Key", 1774 ) 1775 base64_encode_secret_key: Optional[bool] = Field( 1776 False, 1777 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1778 title="Base64-encode Secret Key", 1779 ) 1780 algorithm: Algorithm = Field( 1781 ..., 1782 description="Algorithm used to sign the JSON web token.", 1783 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1784 title="Algorithm", 1785 ) 1786 token_duration: Optional[int] = Field( 1787 1200, 1788 description="The amount of time in seconds a JWT token can be valid after being issued.", 1789 examples=[1200, 3600], 1790 title="Token Duration", 1791 ) 1792 header_prefix: Optional[str] = Field( 1793 None, 1794 description="The prefix to be used within the Authentication header.", 1795 examples=["Bearer", "Basic"], 1796 title="Header Prefix", 1797 ) 1798 jwt_headers: Optional[JwtHeaders] = Field( 1799 None, 1800 description="JWT headers used when signing JSON web token.", 1801 title="JWT Headers", 1802 ) 1803 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1804 None, 1805 description="Additional headers to be included with the JWT headers object.", 1806 title="Additional JWT Headers", 1807 ) 1808 jwt_payload: Optional[JwtPayload] = Field( 1809 None, 1810 description="JWT Payload used when signing JSON web token.", 1811 title="JWT Payload", 1812 ) 1813 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1814 None, 1815 description="Additional properties to be added to the JWT payload.", 1816 title="Additional JWT Payload Properties", 1817 ) 1818 passphrase: Optional[str] = Field( 1819 None, 1820 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1821 examples=["{{ config['passphrase'] }}"], 1822 title="Passphrase", 1823 ) 1824 request_option: Optional[RequestOption] = Field( 1825 None, 1826 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1827 title="Request Option", 1828 ) 1829 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1832class OAuthAuthenticator(BaseModel): 1833 type: Literal["OAuthAuthenticator"] 1834 client_id_name: Optional[str] = Field( 1835 "client_id", 1836 description="The name of the property to use to refresh the `access_token`.", 1837 examples=["custom_app_id"], 1838 title="Client ID Property Name", 1839 ) 1840 client_id: Optional[str] = Field( 1841 None, 1842 description="The OAuth client ID. Fill it in the user inputs.", 1843 examples=[ 1844 "{{ config['client_id'] }}", 1845 "{{ config['credentials']['client_id }}", 1846 ], 1847 title="Client ID", 1848 ) 1849 client_secret_name: Optional[str] = Field( 1850 "client_secret", 1851 description="The name of the property to use to refresh the `access_token`.", 1852 examples=["custom_app_secret"], 1853 title="Client Secret Property Name", 1854 ) 1855 client_secret: Optional[str] = Field( 1856 None, 1857 description="The OAuth client secret. Fill it in the user inputs.", 1858 examples=[ 1859 "{{ config['client_secret'] }}", 1860 "{{ config['credentials']['client_secret }}", 1861 ], 1862 title="Client Secret", 1863 ) 1864 refresh_token_name: Optional[str] = Field( 1865 "refresh_token", 1866 description="The name of the property to use to refresh the `access_token`.", 1867 examples=["custom_app_refresh_value"], 1868 title="Refresh Token Property Name", 1869 ) 1870 refresh_token: Optional[str] = Field( 1871 None, 1872 description="Credential artifact used to get a new access token.", 1873 examples=[ 1874 "{{ config['refresh_token'] }}", 1875 "{{ config['credentials]['refresh_token'] }}", 1876 ], 1877 title="Refresh Token", 1878 ) 1879 token_refresh_endpoint: Optional[str] = Field( 1880 None, 1881 description="The full URL to call to obtain a new access token.", 1882 examples=["https://connect.squareup.com/oauth2/token"], 1883 title="Token Refresh Endpoint", 1884 ) 1885 access_token_name: Optional[str] = Field( 1886 "access_token", 1887 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1888 examples=["access_token"], 1889 title="Access Token Property Name", 1890 ) 1891 access_token_value: Optional[str] = Field( 1892 None, 1893 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1894 examples=["secret_access_token_value"], 1895 title="Access Token Value", 1896 ) 1897 expires_in_name: Optional[str] = Field( 1898 "expires_in", 1899 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1900 examples=["expires_in"], 1901 title="Token Expiry Property Name", 1902 ) 1903 grant_type_name: Optional[str] = Field( 1904 "grant_type", 1905 description="The name of the property to use to refresh the `access_token`.", 1906 examples=["custom_grant_type"], 1907 title="Grant Type Property Name", 1908 ) 1909 grant_type: Optional[str] = Field( 1910 "refresh_token", 1911 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1912 examples=["refresh_token", "client_credentials"], 1913 title="Grant Type", 1914 ) 1915 refresh_request_body: Optional[Dict[str, Any]] = Field( 1916 None, 1917 description="Body of the request sent to get a new access token.", 1918 examples=[ 1919 { 1920 "applicationId": "{{ config['application_id'] }}", 1921 "applicationSecret": "{{ config['application_secret'] }}", 1922 "token": "{{ config['token'] }}", 1923 } 1924 ], 1925 title="Refresh Request Body", 1926 ) 1927 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1928 None, 1929 description="Headers of the request sent to get a new access token.", 1930 examples=[ 1931 { 1932 "Authorization": "<AUTH_TOKEN>", 1933 "Content-Type": "application/x-www-form-urlencoded", 1934 } 1935 ], 1936 title="Refresh Request Headers", 1937 ) 1938 scopes: Optional[List[str]] = Field( 1939 None, 1940 description="List of scopes that should be granted to the access token.", 1941 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1942 title="Scopes", 1943 ) 1944 token_expiry_date: Optional[str] = Field( 1945 None, 1946 description="The access token expiry date.", 1947 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1948 title="Token Expiry Date", 1949 ) 1950 token_expiry_date_format: Optional[str] = Field( 1951 None, 1952 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1953 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1954 title="Token Expiry Date Format", 1955 ) 1956 refresh_token_error_status_codes: Optional[List[int]] = Field( 1957 None, 1958 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1959 examples=[[400, 500]], 1960 title="Refresh Token Error Status Codes", 1961 ) 1962 refresh_token_error_key: Optional[str] = Field( 1963 None, 1964 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1965 examples=["error"], 1966 title="Refresh Token Error Key", 1967 ) 1968 refresh_token_error_values: Optional[List[str]] = Field( 1969 None, 1970 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1971 examples=[["invalid_grant", "invalid_permissions"]], 1972 title="Refresh Token Error Values", 1973 ) 1974 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1975 None, 1976 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1977 title="Refresh Token Updater", 1978 ) 1979 profile_assertion: Optional[JwtAuthenticator] = Field( 1980 None, 1981 description="The authenticator being used to authenticate the client authenticator.", 1982 title="Profile Assertion", 1983 ) 1984 use_profile_assertion: Optional[bool] = Field( 1985 False, 1986 description="Enable using profile assertion as a flow for OAuth authorization.", 1987 title="Use Profile Assertion", 1988 ) 1989 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1992class FixedWindowCallRatePolicy(BaseModel): 1993 class Config: 1994 extra = Extra.allow 1995 1996 type: Literal["FixedWindowCallRatePolicy"] 1997 period: str = Field( 1998 ..., description="The time interval for the rate limit window.", title="Period" 1999 ) 2000 call_limit: int = Field( 2001 ..., 2002 description="The maximum number of calls allowed within the period.", 2003 title="Call Limit", 2004 ) 2005 matchers: List[HttpRequestRegexMatcher] = Field( 2006 ..., 2007 description="List of matchers that define which requests this policy applies to.", 2008 title="Matchers", 2009 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2012class MovingWindowCallRatePolicy(BaseModel): 2013 class Config: 2014 extra = Extra.allow 2015 2016 type: Literal["MovingWindowCallRatePolicy"] 2017 rates: List[Rate] = Field( 2018 ..., 2019 description="List of rates that define the call limits for different time intervals.", 2020 title="Rates", 2021 ) 2022 matchers: List[HttpRequestRegexMatcher] = Field( 2023 ..., 2024 description="List of matchers that define which requests this policy applies to.", 2025 title="Matchers", 2026 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2029class UnlimitedCallRatePolicy(BaseModel): 2030 class Config: 2031 extra = Extra.allow 2032 2033 type: Literal["UnlimitedCallRatePolicy"] 2034 matchers: List[HttpRequestRegexMatcher] = Field( 2035 ..., 2036 description="List of matchers that define which requests this policy applies to.", 2037 title="Matchers", 2038 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2041class DefaultErrorHandler(BaseModel): 2042 type: Literal["DefaultErrorHandler"] 2043 backoff_strategies: Optional[ 2044 List[ 2045 Union[ 2046 ConstantBackoffStrategy, 2047 ExponentialBackoffStrategy, 2048 WaitTimeFromHeader, 2049 WaitUntilTimeFromHeader, 2050 CustomBackoffStrategy, 2051 ] 2052 ] 2053 ] = Field( 2054 None, 2055 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2056 title="Backoff Strategies", 2057 ) 2058 max_retries: Optional[int] = Field( 2059 5, 2060 description="The maximum number of time to retry a retryable request before giving up and failing.", 2061 examples=[5, 0, 10], 2062 title="Max Retry Count", 2063 ) 2064 response_filters: Optional[List[HttpResponseFilter]] = Field( 2065 None, 2066 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2067 title="Response Filters", 2068 ) 2069 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2072class DefaultPaginator(BaseModel): 2073 type: Literal["DefaultPaginator"] 2074 pagination_strategy: Union[ 2075 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2076 ] = Field( 2077 ..., 2078 description="Strategy defining how records are paginated.", 2079 title="Pagination Strategy", 2080 ) 2081 page_size_option: Optional[RequestOption] = Field( 2082 None, title="Inject Page Size Into Outgoing HTTP Request" 2083 ) 2084 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2085 None, 2086 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2087 title="Inject Page Token Into Outgoing HTTP Request", 2088 ) 2089 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2092class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2093 type: Literal["ApiKey"] 2094 inject_into: RequestOption = Field( 2095 ..., 2096 description="Configure how the API Key will be sent in requests to the source API.", 2097 examples=[ 2098 {"inject_into": "header", "field_name": "Authorization"}, 2099 {"inject_into": "request_parameter", "field_name": "authKey"}, 2100 ], 2101 title="Inject API Key Into Outgoing HTTP Request", 2102 ) 2103 api_token: Optional[str] = Field( 2104 "{{ session_token }}", 2105 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2106 examples=[ 2107 "{{ session_token }}", 2108 "Token {{ session_token }}", 2109 "Bearer {{ session_token }}", 2110 ], 2111 title="API Token Template", 2112 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2115class JsonSchemaPropertySelector(BaseModel): 2116 type: Literal["JsonSchemaPropertySelector"] 2117 transformations: Optional[ 2118 List[ 2119 Union[ 2120 AddFields, 2121 RemoveFields, 2122 KeysToLower, 2123 KeysToSnakeCase, 2124 FlattenFields, 2125 DpathFlattenFields, 2126 KeysReplace, 2127 CustomTransformation, 2128 ] 2129 ] 2130 ] = Field( 2131 None, 2132 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2133 title="Transformations", 2134 ) 2135 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2138class ListPartitionRouter(BaseModel): 2139 type: Literal["ListPartitionRouter"] 2140 cursor_field: str = Field( 2141 ..., 2142 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2143 examples=["section", "{{ config['section_key'] }}"], 2144 title="Current Partition Value Identifier", 2145 ) 2146 values: Union[str, List[str]] = Field( 2147 ..., 2148 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2149 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2150 title="Partition Values", 2151 ) 2152 request_option: Optional[RequestOption] = Field( 2153 None, 2154 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2155 title="Inject Partition Value Into Outgoing HTTP Request", 2156 ) 2157 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2160class RecordSelector(BaseModel): 2161 type: Literal["RecordSelector"] 2162 extractor: Union[DpathExtractor, CustomRecordExtractor] 2163 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2164 None, 2165 description="Responsible for filtering records to be emitted by the Source.", 2166 title="Record Filter", 2167 ) 2168 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2169 None, 2170 description="Responsible for normalization according to the schema.", 2171 title="Schema Normalization", 2172 ) 2173 transform_before_filtering: Optional[bool] = Field( 2174 None, 2175 description="If true, transformation will be applied before record filtering.", 2176 title="Transform Before Filtering", 2177 ) 2178 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
PaginationReset(pydantic.v1.main.BaseModel):
2181class PaginationReset(BaseModel): 2182 type: Literal["PaginationReset"] 2183 action: Action1 2184 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2187class GzipDecoder(BaseModel): 2188 type: Literal["GzipDecoder"] 2189 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2192class RequestBodyGraphQL(BaseModel): 2193 type: Literal["RequestBodyGraphQL"] 2194 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2197class DpathValidator(BaseModel): 2198 type: Literal["DpathValidator"] 2199 field_path: List[str] = Field( 2200 ..., 2201 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2202 examples=[ 2203 ["data"], 2204 ["data", "records"], 2205 ["data", "{{ parameters.name }}"], 2206 ["data", "*", "record"], 2207 ], 2208 title="Field Path", 2209 ) 2210 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2211 ..., 2212 description="The condition that the specified config value will be evaluated against", 2213 title="Validation Strategy", 2214 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2217class PredicateValidator(BaseModel): 2218 type: Literal["PredicateValidator"] 2219 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2220 ..., 2221 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2222 examples=[ 2223 "test-value", 2224 "{{ config['api_version'] }}", 2225 "{{ config['tenant_id'] }}", 2226 123, 2227 ], 2228 title="Value", 2229 ) 2230 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2231 ..., 2232 description="The validation strategy to apply to the value.", 2233 title="Validation Strategy", 2234 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2237class ConfigAddFields(BaseModel): 2238 type: Literal["ConfigAddFields"] 2239 fields: List[AddedFieldDefinition] = Field( 2240 ..., 2241 description="A list of transformations (path and corresponding value) that will be added to the config.", 2242 title="Fields", 2243 ) 2244 condition: Optional[str] = Field( 2245 "", 2246 description="Fields will be added if expression is evaluated to True.", 2247 examples=[ 2248 "{{ config['environemnt'] == 'sandbox' }}", 2249 "{{ property is integer }}", 2250 "{{ property|length > 5 }}", 2251 "{{ property == 'some_string_to_match' }}", 2252 ], 2253 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2256class CompositeErrorHandler(BaseModel): 2257 type: Literal["CompositeErrorHandler"] 2258 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2259 Field( 2260 ..., 2261 description="List of error handlers to iterate on to determine how to handle a failed response.", 2262 title="Error Handlers", 2263 ) 2264 ) 2265 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2268class HTTPAPIBudget(BaseModel): 2269 class Config: 2270 extra = Extra.allow 2271 2272 type: Literal["HTTPAPIBudget"] 2273 policies: List[ 2274 Union[ 2275 FixedWindowCallRatePolicy, 2276 MovingWindowCallRatePolicy, 2277 UnlimitedCallRatePolicy, 2278 ] 2279 ] = Field( 2280 ..., 2281 description="List of call rate policies that define how many calls are allowed.", 2282 title="Policies", 2283 ) 2284 ratelimit_reset_header: Optional[str] = Field( 2285 "ratelimit-reset", 2286 description="The HTTP response header name that indicates when the rate limit resets.", 2287 title="Rate Limit Reset Header", 2288 ) 2289 ratelimit_remaining_header: Optional[str] = Field( 2290 "ratelimit-remaining", 2291 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2292 title="Rate Limit Remaining Header", 2293 ) 2294 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2295 [429], 2296 description="List of HTTP status codes that indicate a rate limit has been hit.", 2297 title="Status Codes for Rate Limit Hit", 2298 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2301class ZipfileDecoder(BaseModel): 2302 class Config: 2303 extra = Extra.allow 2304 2305 type: Literal["ZipfileDecoder"] 2306 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2307 ..., 2308 description="Parser to parse the decompressed data from the zipfile(s).", 2309 title="Parser", 2310 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2313class ConfigMigration(BaseModel): 2314 type: Literal["ConfigMigration"] 2315 description: Optional[str] = Field( 2316 None, description="The description/purpose of the config migration." 2317 ) 2318 transformations: List[ 2319 Union[ 2320 ConfigRemapField, 2321 ConfigAddFields, 2322 ConfigRemoveFields, 2323 CustomConfigTransformation, 2324 ] 2325 ] = Field( 2326 ..., 2327 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2328 title="Transformations", 2329 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2332class ConfigNormalizationRules(BaseModel): 2333 class Config: 2334 extra = Extra.forbid 2335 2336 type: Literal["ConfigNormalizationRules"] 2337 config_migrations: Optional[List[ConfigMigration]] = Field( 2338 [], 2339 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2340 title="Config Migrations", 2341 ) 2342 transformations: Optional[ 2343 List[ 2344 Union[ 2345 ConfigRemapField, 2346 ConfigAddFields, 2347 ConfigRemoveFields, 2348 CustomConfigTransformation, 2349 ] 2350 ] 2351 ] = Field( 2352 [], 2353 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2354 title="Transformations", 2355 ) 2356 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2357 [], 2358 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2359 title="Validations", 2360 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2363class Spec(BaseModel): 2364 type: Literal["Spec"] 2365 connection_specification: Dict[str, Any] = Field( 2366 ..., 2367 description="A connection specification describing how a the connector can be configured.", 2368 title="Connection Specification", 2369 ) 2370 documentation_url: Optional[str] = Field( 2371 None, 2372 description="URL of the connector's documentation page.", 2373 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2374 title="Documentation URL", 2375 ) 2376 advanced_auth: Optional[AuthFlow] = Field( 2377 None, 2378 description="Advanced specification for configuring the authentication flow.", 2379 title="Advanced Auth", 2380 ) 2381 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2382 None, title="Config Normalization Rules" 2383 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2386class DeclarativeSource1(BaseModel): 2387 class Config: 2388 extra = Extra.forbid 2389 2390 type: Literal["DeclarativeSource"] 2391 check: Union[CheckStream, CheckDynamicStream] 2392 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2393 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2394 version: str = Field( 2395 ..., 2396 description="The version of the Airbyte CDK used to build and test the source.", 2397 ) 2398 schemas: Optional[Schemas] = None 2399 definitions: Optional[Dict[str, Any]] = None 2400 spec: Optional[Spec] = None 2401 concurrency_level: Optional[ConcurrencyLevel] = None 2402 api_budget: Optional[HTTPAPIBudget] = None 2403 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2404 None, 2405 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2406 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2407 title="Maximum Concurrent Asynchronous Jobs", 2408 ) 2409 metadata: Optional[Dict[str, Any]] = Field( 2410 None, 2411 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2412 ) 2413 description: Optional[str] = Field( 2414 None, 2415 description="A description of the connector. It will be presented on the Source documentation page.", 2416 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2419class DeclarativeSource2(BaseModel): 2420 class Config: 2421 extra = Extra.forbid 2422 2423 type: Literal["DeclarativeSource"] 2424 check: Union[CheckStream, CheckDynamicStream] 2425 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2426 None 2427 ) 2428 dynamic_streams: List[DynamicDeclarativeStream] 2429 version: str = Field( 2430 ..., 2431 description="The version of the Airbyte CDK used to build and test the source.", 2432 ) 2433 schemas: Optional[Schemas] = None 2434 definitions: Optional[Dict[str, Any]] = None 2435 spec: Optional[Spec] = None 2436 concurrency_level: Optional[ConcurrencyLevel] = None 2437 api_budget: Optional[HTTPAPIBudget] = None 2438 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2439 None, 2440 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2441 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2442 title="Maximum Concurrent Asynchronous Jobs", 2443 ) 2444 metadata: Optional[Dict[str, Any]] = Field( 2445 None, 2446 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2447 ) 2448 description: Optional[str] = Field( 2449 None, 2450 description="A description of the connector. It will be presented on the Source documentation page.", 2451 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2454class DeclarativeSource(BaseModel): 2455 class Config: 2456 extra = Extra.forbid 2457 2458 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2459 ..., 2460 description="An API source that extracts data according to its declarative components.", 2461 title="DeclarativeSource", 2462 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2465class SelectiveAuthenticator(BaseModel): 2466 class Config: 2467 extra = Extra.allow 2468 2469 type: Literal["SelectiveAuthenticator"] 2470 authenticator_selection_path: List[str] = Field( 2471 ..., 2472 description="Path of the field in config with selected authenticator name", 2473 examples=[["auth"], ["auth", "type"]], 2474 title="Authenticator Selection Path", 2475 ) 2476 authenticators: Dict[ 2477 str, 2478 Union[ 2479 ApiKeyAuthenticator, 2480 BasicHttpAuthenticator, 2481 BearerAuthenticator, 2482 OAuthAuthenticator, 2483 JwtAuthenticator, 2484 SessionTokenAuthenticator, 2485 LegacySessionTokenAuthenticator, 2486 CustomAuthenticator, 2487 NoAuth, 2488 ], 2489 ] = Field( 2490 ..., 2491 description="Authenticators to select from.", 2492 examples=[ 2493 { 2494 "authenticators": { 2495 "token": "#/definitions/ApiKeyAuthenticator", 2496 "oauth": "#/definitions/OAuthAuthenticator", 2497 "jwt": "#/definitions/JwtAuthenticator", 2498 } 2499 } 2500 ], 2501 title="Authenticators", 2502 ) 2503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2506class ConditionalStreams(BaseModel): 2507 type: Literal["ConditionalStreams"] 2508 condition: str = Field( 2509 ..., 2510 description="Condition that will be evaluated to determine if a set of streams should be available.", 2511 examples=["{{ config['is_sandbox'] }}"], 2512 title="Condition", 2513 ) 2514 streams: List[DeclarativeStream] = Field( 2515 ..., 2516 description="Streams that will be used during an operation based on the condition.", 2517 title="Streams", 2518 ) 2519 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2522class FileUploader(BaseModel): 2523 type: Literal["FileUploader"] 2524 requester: Union[HttpRequester, CustomRequester] = Field( 2525 ..., 2526 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2527 ) 2528 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2529 ..., 2530 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2531 ) 2532 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2533 None, 2534 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2535 ) 2536 filename_extractor: Optional[str] = Field( 2537 None, 2538 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2539 examples=[ 2540 "{{ record.id }}/{{ record.file_name }}/", 2541 "{{ record.id }}_{{ record.file_name }}/", 2542 ], 2543 ) 2544 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2547class DeclarativeStream(BaseModel): 2548 class Config: 2549 extra = Extra.allow 2550 2551 type: Literal["DeclarativeStream"] 2552 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2553 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2554 ..., 2555 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2556 title="Retriever", 2557 ) 2558 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2559 None, 2560 description="Component used to fetch data incrementally based on a time field in the data.", 2561 title="Incremental Sync", 2562 ) 2563 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2564 schema_loader: Optional[ 2565 Union[ 2566 InlineSchemaLoader, 2567 DynamicSchemaLoader, 2568 JsonFileSchemaLoader, 2569 List[ 2570 Union[ 2571 InlineSchemaLoader, 2572 DynamicSchemaLoader, 2573 JsonFileSchemaLoader, 2574 CustomSchemaLoader, 2575 ] 2576 ], 2577 CustomSchemaLoader, 2578 ] 2579 ] = Field( 2580 None, 2581 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2582 title="Schema Loader", 2583 ) 2584 transformations: Optional[ 2585 List[ 2586 Union[ 2587 AddFields, 2588 RemoveFields, 2589 KeysToLower, 2590 KeysToSnakeCase, 2591 FlattenFields, 2592 DpathFlattenFields, 2593 KeysReplace, 2594 CustomTransformation, 2595 ] 2596 ] 2597 ] = Field( 2598 None, 2599 description="A list of transformations to be applied to each output record.", 2600 title="Transformations", 2601 ) 2602 state_migrations: Optional[ 2603 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2604 ] = Field( 2605 [], 2606 description="Array of state migrations to be applied on the input state", 2607 title="State Migrations", 2608 ) 2609 file_uploader: Optional[FileUploader] = Field( 2610 None, 2611 description="(experimental) Describes how to fetch a file", 2612 title="File Uploader", 2613 ) 2614 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2617class SessionTokenAuthenticator(BaseModel): 2618 type: Literal["SessionTokenAuthenticator"] 2619 login_requester: HttpRequester = Field( 2620 ..., 2621 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2622 examples=[ 2623 { 2624 "type": "HttpRequester", 2625 "url_base": "https://my_api.com", 2626 "path": "/login", 2627 "authenticator": { 2628 "type": "BasicHttpAuthenticator", 2629 "username": "{{ config.username }}", 2630 "password": "{{ config.password }}", 2631 }, 2632 } 2633 ], 2634 title="Login Requester", 2635 ) 2636 session_token_path: List[str] = Field( 2637 ..., 2638 description="The path in the response body returned from the login requester to the session token.", 2639 examples=[["access_token"], ["result", "token"]], 2640 title="Session Token Path", 2641 ) 2642 expiration_duration: Optional[str] = Field( 2643 None, 2644 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2645 examples=["PT1H", "P1D"], 2646 title="Expiration Duration", 2647 ) 2648 request_authentication: Union[ 2649 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2650 ] = Field( 2651 ..., 2652 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2653 title="Data Request Authentication", 2654 ) 2655 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2656 None, description="Component used to decode the response.", title="Decoder" 2657 ) 2658 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2661class HttpRequester(BaseModelWithDeprecations): 2662 type: Literal["HttpRequester"] 2663 url_base: Optional[str] = Field( 2664 None, 2665 deprecated=True, 2666 deprecation_message="Use `url` field instead.", 2667 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2668 examples=[ 2669 "https://connect.squareup.com/v2", 2670 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2671 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2672 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2673 ], 2674 title="API Base URL", 2675 ) 2676 url: Optional[str] = Field( 2677 None, 2678 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2679 examples=[ 2680 "https://connect.squareup.com/v2", 2681 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2682 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2683 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2684 ], 2685 title="API Endpoint URL", 2686 ) 2687 path: Optional[str] = Field( 2688 None, 2689 deprecated=True, 2690 deprecation_message="Use `url` field instead.", 2691 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2692 examples=[ 2693 "/products", 2694 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2695 "/trades/{{ config['symbol_id'] }}/history", 2696 ], 2697 title="URL Path", 2698 ) 2699 http_method: Optional[HttpMethod] = Field( 2700 HttpMethod.GET, 2701 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2702 examples=["GET", "POST"], 2703 title="HTTP Method", 2704 ) 2705 authenticator: Optional[ 2706 Union[ 2707 ApiKeyAuthenticator, 2708 BasicHttpAuthenticator, 2709 BearerAuthenticator, 2710 OAuthAuthenticator, 2711 JwtAuthenticator, 2712 SessionTokenAuthenticator, 2713 SelectiveAuthenticator, 2714 CustomAuthenticator, 2715 NoAuth, 2716 LegacySessionTokenAuthenticator, 2717 ] 2718 ] = Field( 2719 None, 2720 description="Authentication method to use for requests sent to the API.", 2721 title="Authenticator", 2722 ) 2723 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2724 None, 2725 deprecated=True, 2726 deprecation_message="Use `query_properties` field instead.", 2727 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2728 title="Fetch Properties from Endpoint", 2729 ) 2730 query_properties: Optional[QueryProperties] = Field( 2731 None, 2732 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2733 title="Query Properties", 2734 ) 2735 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2736 None, 2737 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2738 examples=[ 2739 {"unit": "day"}, 2740 { 2741 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2742 }, 2743 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2744 {"sort_by[asc]": "updated_at"}, 2745 ], 2746 title="Query Parameters", 2747 ) 2748 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2749 None, 2750 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2751 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2752 title="Request Headers", 2753 ) 2754 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2755 None, 2756 deprecated=True, 2757 deprecation_message="Use `request_body` field instead.", 2758 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2759 examples=[ 2760 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2761 ], 2762 title="Request Body Payload (Non-JSON)", 2763 ) 2764 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2765 None, 2766 deprecated=True, 2767 deprecation_message="Use `request_body` field instead.", 2768 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2769 examples=[ 2770 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2771 {"key": "{{ config['value'] }}"}, 2772 {"sort": {"field": "updated_at", "order": "ascending"}}, 2773 ], 2774 title="Request Body JSON Payload", 2775 ) 2776 request_body: Optional[ 2777 Union[ 2778 RequestBodyPlainText, 2779 RequestBodyUrlEncodedForm, 2780 RequestBodyJsonObject, 2781 RequestBodyGraphQL, 2782 ] 2783 ] = Field( 2784 None, 2785 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2786 title="Request Body", 2787 ) 2788 error_handler: Optional[ 2789 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2790 ] = Field( 2791 None, 2792 description="Error handler component that defines how to handle errors.", 2793 title="Error Handler", 2794 ) 2795 use_cache: Optional[bool] = Field( 2796 False, 2797 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2798 title="Use Cache", 2799 ) 2800 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2803class DynamicSchemaLoader(BaseModel): 2804 type: Literal["DynamicSchemaLoader"] 2805 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2806 ..., 2807 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2808 title="Retriever", 2809 ) 2810 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2811 None, 2812 description="Responsible for filtering fields to be added to json schema.", 2813 title="Schema Filter", 2814 ) 2815 schema_transformations: Optional[ 2816 List[ 2817 Union[ 2818 AddFields, 2819 RemoveFields, 2820 KeysToLower, 2821 KeysToSnakeCase, 2822 FlattenFields, 2823 DpathFlattenFields, 2824 KeysReplace, 2825 CustomTransformation, 2826 ] 2827 ] 2828 ] = Field( 2829 None, 2830 description="A list of transformations to be applied to the schema.", 2831 title="Schema Transformations", 2832 ) 2833 schema_type_identifier: SchemaTypeIdentifier 2834 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2837class ParentStreamConfig(BaseModel): 2838 type: Literal["ParentStreamConfig"] 2839 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2840 ..., description="Reference to the parent stream.", title="Parent Stream" 2841 ) 2842 parent_key: str = Field( 2843 ..., 2844 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2845 examples=["id", "{{ config['parent_record_id'] }}"], 2846 title="Parent Key", 2847 ) 2848 partition_field: str = Field( 2849 ..., 2850 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2851 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2852 title="Current Parent Key Value Identifier", 2853 ) 2854 request_option: Optional[RequestOption] = Field( 2855 None, 2856 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2857 title="Request Option", 2858 ) 2859 incremental_dependency: Optional[bool] = Field( 2860 False, 2861 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2862 title="Incremental Dependency", 2863 ) 2864 lazy_read_pointer: Optional[List[str]] = Field( 2865 [], 2866 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2867 title="Lazy Read Pointer", 2868 ) 2869 extra_fields: Optional[List[List[str]]] = Field( 2870 None, 2871 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2872 title="Extra Fields", 2873 ) 2874 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2877class PropertiesFromEndpoint(BaseModel): 2878 type: Literal["PropertiesFromEndpoint"] 2879 property_field_path: List[str] = Field( 2880 ..., 2881 description="Describes the path to the field that should be extracted", 2882 examples=[["name"]], 2883 ) 2884 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2885 ..., 2886 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2887 ) 2888 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2891class QueryProperties(BaseModel): 2892 type: Literal["QueryProperties"] 2893 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2894 ..., 2895 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2896 title="Property List", 2897 ) 2898 always_include_properties: Optional[List[str]] = Field( 2899 None, 2900 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2901 title="Always Include Properties", 2902 ) 2903 property_chunking: Optional[PropertyChunking] = Field( 2904 None, 2905 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2906 title="Property Chunking", 2907 ) 2908 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2909 None, 2910 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2911 title="Property Selector", 2912 ) 2913 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2916class StateDelegatingStream(BaseModel): 2917 type: Literal["StateDelegatingStream"] 2918 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2919 full_refresh_stream: DeclarativeStream = Field( 2920 ..., 2921 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2922 title="Full Refresh Stream", 2923 ) 2924 incremental_stream: DeclarativeStream = Field( 2925 ..., 2926 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2927 title="Incremental Stream", 2928 ) 2929 api_retention_period: Optional[str] = Field( 2930 None, 2931 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2932 examples=["P30D", "P90D", "P1Y"], 2933 title="API Retention Period", 2934 ) 2935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2938class SimpleRetriever(BaseModel): 2939 type: Literal["SimpleRetriever"] 2940 requester: Union[HttpRequester, CustomRequester] = Field( 2941 ..., 2942 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2943 ) 2944 decoder: Optional[ 2945 Union[ 2946 JsonDecoder, 2947 XmlDecoder, 2948 CsvDecoder, 2949 JsonlDecoder, 2950 GzipDecoder, 2951 IterableDecoder, 2952 ZipfileDecoder, 2953 CustomDecoder, 2954 ] 2955 ] = Field( 2956 None, 2957 description="Component decoding the response so records can be extracted.", 2958 title="HTTP Response Format", 2959 ) 2960 record_selector: RecordSelector = Field( 2961 ..., 2962 description="Component that describes how to extract records from a HTTP response.", 2963 ) 2964 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2965 None, 2966 description="Paginator component that describes how to navigate through the API's pages.", 2967 ) 2968 pagination_reset: Optional[PaginationReset] = Field( 2969 None, 2970 description="Describes what triggers pagination reset and how to handle it.", 2971 ) 2972 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2973 False, 2974 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2975 ) 2976 partition_router: Optional[ 2977 Union[ 2978 SubstreamPartitionRouter, 2979 ListPartitionRouter, 2980 GroupingPartitionRouter, 2981 CustomPartitionRouter, 2982 List[ 2983 Union[ 2984 SubstreamPartitionRouter, 2985 ListPartitionRouter, 2986 GroupingPartitionRouter, 2987 CustomPartitionRouter, 2988 ] 2989 ], 2990 ] 2991 ] = Field( 2992 None, 2993 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2994 title="Partition Router", 2995 ) 2996 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2999class AsyncRetriever(BaseModel): 3000 type: Literal["AsyncRetriever"] 3001 record_selector: RecordSelector = Field( 3002 ..., 3003 description="Component that describes how to extract records from a HTTP response.", 3004 ) 3005 status_mapping: AsyncJobStatusMap = Field( 3006 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3007 ) 3008 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3009 ..., description="Responsible for fetching the actual status of the async job." 3010 ) 3011 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3012 None, 3013 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3014 ) 3015 download_extractor: Optional[ 3016 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3017 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3018 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3019 ..., 3020 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3021 ) 3022 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3023 ..., 3024 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3025 ) 3026 polling_job_timeout: Optional[Union[int, str]] = Field( 3027 None, 3028 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3029 ) 3030 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3031 None, 3032 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3033 ) 3034 download_requester: Union[HttpRequester, CustomRequester] = Field( 3035 ..., 3036 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3037 ) 3038 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3039 None, 3040 description="Paginator component that describes how to navigate through the API's pages during download.", 3041 ) 3042 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3043 None, 3044 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3045 ) 3046 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3047 None, 3048 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3049 ) 3050 partition_router: Optional[ 3051 Union[ 3052 ListPartitionRouter, 3053 SubstreamPartitionRouter, 3054 GroupingPartitionRouter, 3055 CustomPartitionRouter, 3056 List[ 3057 Union[ 3058 ListPartitionRouter, 3059 SubstreamPartitionRouter, 3060 GroupingPartitionRouter, 3061 CustomPartitionRouter, 3062 ] 3063 ], 3064 ] 3065 ] = Field( 3066 [], 3067 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3068 title="Partition Router", 3069 ) 3070 decoder: Optional[ 3071 Union[ 3072 CsvDecoder, 3073 GzipDecoder, 3074 JsonDecoder, 3075 JsonlDecoder, 3076 IterableDecoder, 3077 XmlDecoder, 3078 ZipfileDecoder, 3079 CustomDecoder, 3080 ] 3081 ] = Field( 3082 None, 3083 description="Component decoding the response so records can be extracted.", 3084 title="HTTP Response Format", 3085 ) 3086 download_decoder: Optional[ 3087 Union[ 3088 CsvDecoder, 3089 GzipDecoder, 3090 JsonDecoder, 3091 JsonlDecoder, 3092 IterableDecoder, 3093 XmlDecoder, 3094 ZipfileDecoder, 3095 CustomDecoder, 3096 ] 3097 ] = Field( 3098 None, 3099 description="Component decoding the download response so records can be extracted.", 3100 title="Download HTTP Response Format", 3101 ) 3102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3105class SubstreamPartitionRouter(BaseModel): 3106 type: Literal["SubstreamPartitionRouter"] 3107 parent_stream_configs: List[ParentStreamConfig] = Field( 3108 ..., 3109 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3110 title="Parent Stream Configs", 3111 ) 3112 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3115class GroupingPartitionRouter(BaseModel): 3116 type: Literal["GroupingPartitionRouter"] 3117 group_size: int = Field( 3118 ..., 3119 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3120 examples=[10, 50], 3121 title="Group Size", 3122 ) 3123 underlying_partition_router: Union[ 3124 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3125 ] = Field( 3126 ..., 3127 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3128 title="Underlying Partition Router", 3129 ) 3130 deduplicate: Optional[bool] = Field( 3131 True, 3132 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3133 title="Deduplicate Partitions", 3134 ) 3135 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3138class HttpComponentsResolver(BaseModel): 3139 type: Literal["HttpComponentsResolver"] 3140 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3141 ..., 3142 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3143 title="Retriever", 3144 ) 3145 components_mapping: List[ComponentMappingDefinition] 3146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3149class DynamicDeclarativeStream(BaseModel): 3150 type: Literal["DynamicDeclarativeStream"] 3151 name: Optional[str] = Field( 3152 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3153 ) 3154 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3155 ..., description="Reference to the stream template.", title="Stream Template" 3156 ) 3157 components_resolver: Union[ 3158 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3159 ] = Field( 3160 ..., 3161 description="Component resolve and populates stream templates with components values.", 3162 title="Components Resolver", 3163 ) 3164 use_parent_parameters: Optional[bool] = Field( 3165 True, 3166 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3167 title="Use Parent Parameters", 3168 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]