airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[Union[int, str]] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100, "{{ config['page_size'] }}"], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomPaginationStrategy(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomPaginationStrategy"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomPaginationStrategy"], 186 title="Class Name", 187 ) 188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 189 190 191class CustomRecordExtractor(BaseModel): 192 class Config: 193 extra = Extra.allow 194 195 type: Literal["CustomRecordExtractor"] 196 class_name: str = Field( 197 ..., 198 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 199 examples=["source_railz.components.MyCustomRecordExtractor"], 200 title="Class Name", 201 ) 202 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 203 204 205class CustomRecordFilter(BaseModel): 206 class Config: 207 extra = Extra.allow 208 209 type: Literal["CustomRecordFilter"] 210 class_name: str = Field( 211 ..., 212 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 213 examples=["source_railz.components.MyCustomCustomRecordFilter"], 214 title="Class Name", 215 ) 216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 217 218 219class CustomRequester(BaseModel): 220 class Config: 221 extra = Extra.allow 222 223 type: Literal["CustomRequester"] 224 class_name: str = Field( 225 ..., 226 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 227 examples=["source_railz.components.MyCustomRecordExtractor"], 228 title="Class Name", 229 ) 230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 231 232 233class CustomRetriever(BaseModel): 234 class Config: 235 extra = Extra.allow 236 237 type: Literal["CustomRetriever"] 238 class_name: str = Field( 239 ..., 240 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 241 examples=["source_railz.components.MyCustomRetriever"], 242 title="Class Name", 243 ) 244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 245 246 247class CustomPartitionRouter(BaseModel): 248 class Config: 249 extra = Extra.allow 250 251 type: Literal["CustomPartitionRouter"] 252 class_name: str = Field( 253 ..., 254 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 255 examples=["source_railz.components.MyCustomPartitionRouter"], 256 title="Class Name", 257 ) 258 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 259 260 261class CustomSchemaLoader(BaseModel): 262 class Config: 263 extra = Extra.allow 264 265 type: Literal["CustomSchemaLoader"] 266 class_name: str = Field( 267 ..., 268 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 269 examples=["source_railz.components.MyCustomSchemaLoader"], 270 title="Class Name", 271 ) 272 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 273 274 275class CustomSchemaNormalization(BaseModel): 276 class Config: 277 extra = Extra.allow 278 279 type: Literal["CustomSchemaNormalization"] 280 class_name: str = Field( 281 ..., 282 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 283 examples=[ 284 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 285 ], 286 title="Class Name", 287 ) 288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 289 290 291class CustomStateMigration(BaseModel): 292 class Config: 293 extra = Extra.allow 294 295 type: Literal["CustomStateMigration"] 296 class_name: str = Field( 297 ..., 298 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 299 examples=["source_railz.components.MyCustomStateMigration"], 300 title="Class Name", 301 ) 302 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 303 304 305class CustomTransformation(BaseModel): 306 class Config: 307 extra = Extra.allow 308 309 type: Literal["CustomTransformation"] 310 class_name: str = Field( 311 ..., 312 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 313 examples=["source_railz.components.MyCustomTransformation"], 314 title="Class Name", 315 ) 316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 317 318 319class LegacyToPerPartitionStateMigration(BaseModel): 320 class Config: 321 extra = Extra.allow 322 323 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 324 325 326class Clamping(BaseModel): 327 target: str = Field( 328 ..., 329 description="The period of time that datetime windows will be clamped by", 330 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 331 title="Target", 332 ) 333 target_details: Optional[Dict[str, Any]] = None 334 335 336class Algorithm(Enum): 337 HS256 = "HS256" 338 HS384 = "HS384" 339 HS512 = "HS512" 340 ES256 = "ES256" 341 ES256K = "ES256K" 342 ES384 = "ES384" 343 ES512 = "ES512" 344 RS256 = "RS256" 345 RS384 = "RS384" 346 RS512 = "RS512" 347 PS256 = "PS256" 348 PS384 = "PS384" 349 PS512 = "PS512" 350 EdDSA = "EdDSA" 351 352 353class JwtHeaders(BaseModel): 354 class Config: 355 extra = Extra.forbid 356 357 kid: Optional[str] = Field( 358 None, 359 description="Private key ID for user account.", 360 examples=["{{ config['kid'] }}"], 361 title="Key Identifier", 362 ) 363 typ: Optional[str] = Field( 364 "JWT", 365 description="The media type of the complete JWT.", 366 examples=["JWT"], 367 title="Type", 368 ) 369 cty: Optional[str] = Field( 370 None, 371 description="Content type of JWT header.", 372 examples=["JWT"], 373 title="Content Type", 374 ) 375 376 377class JwtPayload(BaseModel): 378 class Config: 379 extra = Extra.forbid 380 381 iss: Optional[str] = Field( 382 None, 383 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 384 examples=["{{ config['iss'] }}"], 385 title="Issuer", 386 ) 387 sub: Optional[str] = Field( 388 None, 389 description="The subject of the JWT. Commonly defined by the API.", 390 title="Subject", 391 ) 392 aud: Optional[str] = Field( 393 None, 394 description="The recipient that the JWT is intended for. Commonly defined by the API.", 395 examples=["appstoreconnect-v1"], 396 title="Audience", 397 ) 398 399 400class RefreshTokenUpdater(BaseModel): 401 refresh_token_name: Optional[str] = Field( 402 "refresh_token", 403 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 404 examples=["refresh_token"], 405 title="Refresh Token Property Name", 406 ) 407 access_token_config_path: Optional[List[str]] = Field( 408 ["credentials", "access_token"], 409 description="Config path to the access token. Make sure the field actually exists in the config.", 410 examples=[["credentials", "access_token"], ["access_token"]], 411 title="Config Path To Access Token", 412 ) 413 refresh_token_config_path: Optional[List[str]] = Field( 414 ["credentials", "refresh_token"], 415 description="Config path to the access token. Make sure the field actually exists in the config.", 416 examples=[["credentials", "refresh_token"], ["refresh_token"]], 417 title="Config Path To Refresh Token", 418 ) 419 token_expiry_date_config_path: Optional[List[str]] = Field( 420 ["credentials", "token_expiry_date"], 421 description="Config path to the expiry date. Make sure actually exists in the config.", 422 examples=[["credentials", "token_expiry_date"]], 423 title="Config Path To Expiry Date", 424 ) 425 refresh_token_error_status_codes: Optional[List[int]] = Field( 426 [], 427 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 428 examples=[[400, 500]], 429 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 430 ) 431 refresh_token_error_key: Optional[str] = Field( 432 "", 433 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 434 examples=["error"], 435 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 436 ) 437 refresh_token_error_values: Optional[List[str]] = Field( 438 [], 439 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 440 examples=[["invalid_grant", "invalid_permissions"]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 442 ) 443 444 445class Rate(BaseModel): 446 class Config: 447 extra = Extra.allow 448 449 limit: Union[int, str] = Field( 450 ..., 451 description="The maximum number of calls allowed within the interval.", 452 title="Limit", 453 ) 454 interval: str = Field( 455 ..., 456 description="The time interval for the rate limit.", 457 examples=["PT1H", "P1D"], 458 title="Interval", 459 ) 460 461 462class HttpRequestRegexMatcher(BaseModel): 463 class Config: 464 extra = Extra.allow 465 466 method: Optional[str] = Field( 467 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 468 ) 469 url_base: Optional[str] = Field( 470 None, 471 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 472 title="URL Base", 473 ) 474 url_path_pattern: Optional[str] = Field( 475 None, 476 description="A regular expression pattern to match the URL path.", 477 title="URL Path Pattern", 478 ) 479 params: Optional[Dict[str, Any]] = Field( 480 None, description="The query parameters to match.", title="Parameters" 481 ) 482 headers: Optional[Dict[str, Any]] = Field( 483 None, description="The headers to match.", title="Headers" 484 ) 485 486 487class DpathExtractor(BaseModel): 488 type: Literal["DpathExtractor"] 489 field_path: List[str] = Field( 490 ..., 491 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 492 examples=[ 493 ["data"], 494 ["data", "records"], 495 ["data", "{{ parameters.name }}"], 496 ["data", "*", "record"], 497 ], 498 title="Field Path", 499 ) 500 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 501 502 503class ResponseToFileExtractor(BaseModel): 504 type: Literal["ResponseToFileExtractor"] 505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 506 507 508class ExponentialBackoffStrategy(BaseModel): 509 type: Literal["ExponentialBackoffStrategy"] 510 factor: Optional[Union[float, str]] = Field( 511 5, 512 description="Multiplicative constant applied on each retry.", 513 examples=[5, 5.5, "10"], 514 title="Factor", 515 ) 516 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 517 518 519class GroupByKeyMergeStrategy(BaseModel): 520 type: Literal["GroupByKeyMergeStrategy"] 521 key: Union[str, List[str]] = Field( 522 ..., 523 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 524 examples=["id", ["parent_id", "end_date"]], 525 title="Key", 526 ) 527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 528 529 530class SessionTokenRequestBearerAuthenticator(BaseModel): 531 type: Literal["Bearer"] 532 533 534class HttpMethod(Enum): 535 GET = "GET" 536 POST = "POST" 537 538 539class Action(Enum): 540 SUCCESS = "SUCCESS" 541 FAIL = "FAIL" 542 RETRY = "RETRY" 543 IGNORE = "IGNORE" 544 RESET_PAGINATION = "RESET_PAGINATION" 545 RATE_LIMITED = "RATE_LIMITED" 546 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 547 548 549class FailureType(Enum): 550 system_error = "system_error" 551 config_error = "config_error" 552 transient_error = "transient_error" 553 554 555class HttpResponseFilter(BaseModel): 556 type: Literal["HttpResponseFilter"] 557 action: Optional[Action] = Field( 558 None, 559 description="Action to execute if a response matches the filter.", 560 examples=[ 561 "SUCCESS", 562 "FAIL", 563 "RETRY", 564 "IGNORE", 565 "RESET_PAGINATION", 566 "RATE_LIMITED", 567 "REFRESH_TOKEN_THEN_RETRY", 568 ], 569 title="Action", 570 ) 571 failure_type: Optional[FailureType] = Field( 572 None, 573 description="Failure type of traced exception if a response matches the filter.", 574 examples=["system_error", "config_error", "transient_error"], 575 title="Failure Type", 576 ) 577 error_message: Optional[str] = Field( 578 None, 579 description="Error Message to display if the response matches the filter.", 580 title="Error Message", 581 ) 582 error_message_contains: Optional[str] = Field( 583 None, 584 description="Match the response if its error message contains the substring.", 585 example=["This API operation is not enabled for this site"], 586 title="Error Message Substring", 587 ) 588 http_codes: Optional[List[int]] = Field( 589 None, 590 description="Match the response if its HTTP code is included in this list.", 591 examples=[[420, 429], [500]], 592 title="HTTP Codes", 593 unique_items=True, 594 ) 595 predicate: Optional[str] = Field( 596 None, 597 description="Match the response if the predicate evaluates to true.", 598 examples=[ 599 "{{ 'Too much requests' in response }}", 600 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 601 ], 602 title="Predicate", 603 ) 604 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 605 606 607class ComplexFieldType(BaseModel): 608 field_type: str 609 items: Optional[Union[str, ComplexFieldType]] = None 610 611 612class TypesMap(BaseModel): 613 target_type: Union[str, List[str], ComplexFieldType] 614 current_type: Union[str, List[str]] 615 condition: Optional[str] = None 616 617 618class SchemaTypeIdentifier(BaseModel): 619 type: Optional[Literal["SchemaTypeIdentifier"]] = None 620 schema_pointer: Optional[List[str]] = Field( 621 [], 622 description="List of nested fields defining the schema field path to extract. Defaults to [].", 623 title="Schema Path", 624 ) 625 key_pointer: List[str] = Field( 626 ..., 627 description="List of potentially nested fields describing the full path of the field key to extract.", 628 title="Key Path", 629 ) 630 type_pointer: Optional[List[str]] = Field( 631 None, 632 description="List of potentially nested fields describing the full path of the field type to extract.", 633 title="Type Path", 634 ) 635 types_mapping: Optional[List[TypesMap]] = None 636 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 637 638 639class InlineSchemaLoader(BaseModel): 640 type: Literal["InlineSchemaLoader"] 641 schema_: Optional[Dict[str, Any]] = Field( 642 None, 643 alias="schema", 644 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 645 title="Schema", 646 ) 647 648 649class JsonFileSchemaLoader(BaseModel): 650 type: Literal["JsonFileSchemaLoader"] 651 file_path: Optional[str] = Field( 652 None, 653 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 654 example=["./schemas/users.json"], 655 title="File Path", 656 ) 657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 658 659 660class JsonDecoder(BaseModel): 661 type: Literal["JsonDecoder"] 662 663 664class JsonlDecoder(BaseModel): 665 type: Literal["JsonlDecoder"] 666 667 668class KeysToLower(BaseModel): 669 type: Literal["KeysToLower"] 670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 671 672 673class KeysToSnakeCase(BaseModel): 674 type: Literal["KeysToSnakeCase"] 675 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 676 677 678class FlattenFields(BaseModel): 679 type: Literal["FlattenFields"] 680 flatten_lists: Optional[bool] = Field( 681 True, 682 description="Whether to flatten lists or leave it as is. Default is True.", 683 title="Flatten Lists", 684 ) 685 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 686 687 688class KeyTransformation(BaseModel): 689 type: Literal["KeyTransformation"] 690 prefix: Optional[str] = Field( 691 None, 692 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 693 examples=["flattened_"], 694 title="Key Prefix", 695 ) 696 suffix: Optional[str] = Field( 697 None, 698 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 699 examples=["_flattened"], 700 title="Key Suffix", 701 ) 702 703 704class DpathFlattenFields(BaseModel): 705 type: Literal["DpathFlattenFields"] 706 field_path: List[str] = Field( 707 ..., 708 description="A path to field that needs to be flattened.", 709 examples=[["data"], ["data", "*", "field"]], 710 title="Field Path", 711 ) 712 delete_origin_value: Optional[bool] = Field( 713 None, 714 description="Whether to delete the origin value or keep it. Default is False.", 715 title="Delete Origin Value", 716 ) 717 replace_record: Optional[bool] = Field( 718 None, 719 description="Whether to replace the origin record or not. Default is False.", 720 title="Replace Origin Record", 721 ) 722 key_transformation: Optional[KeyTransformation] = Field( 723 None, 724 description="Transformation for object keys. If not provided, original key will be used.", 725 title="Key transformation", 726 ) 727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 728 729 730class KeysReplace(BaseModel): 731 type: Literal["KeysReplace"] 732 old: str = Field( 733 ..., 734 description="Old value to replace.", 735 examples=[ 736 " ", 737 "{{ record.id }}", 738 "{{ config['id'] }}", 739 "{{ stream_slice['id'] }}", 740 ], 741 title="Old value", 742 ) 743 new: str = Field( 744 ..., 745 description="New value to set.", 746 examples=[ 747 "_", 748 "{{ record.id }}", 749 "{{ config['id'] }}", 750 "{{ stream_slice['id'] }}", 751 ], 752 title="New value", 753 ) 754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 755 756 757class IterableDecoder(BaseModel): 758 type: Literal["IterableDecoder"] 759 760 761class XmlDecoder(BaseModel): 762 type: Literal["XmlDecoder"] 763 764 765class CustomDecoder(BaseModel): 766 class Config: 767 extra = Extra.allow 768 769 type: Literal["CustomDecoder"] 770 class_name: str = Field( 771 ..., 772 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 773 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 774 title="Class Name", 775 ) 776 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 777 778 779class MinMaxDatetime(BaseModel): 780 type: Literal["MinMaxDatetime"] 781 datetime: str = Field( 782 ..., 783 description="Datetime value.", 784 examples=[ 785 "2021-01-01", 786 "2021-01-01T00:00:00Z", 787 "{{ config['start_time'] }}", 788 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 789 ], 790 title="Datetime", 791 ) 792 datetime_format: Optional[str] = Field( 793 "", 794 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 795 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 796 title="Datetime Format", 797 ) 798 max_datetime: Optional[str] = Field( 799 None, 800 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 801 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 802 title="Max Datetime", 803 ) 804 min_datetime: Optional[str] = Field( 805 None, 806 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 807 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 808 title="Min Datetime", 809 ) 810 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 811 812 813class NoAuth(BaseModel): 814 type: Literal["NoAuth"] 815 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 816 817 818class NoPagination(BaseModel): 819 type: Literal["NoPagination"] 820 821 822class State(BaseModel): 823 class Config: 824 extra = Extra.allow 825 826 min: int 827 max: int 828 829 830class OauthConnectorInputSpecification(BaseModel): 831 class Config: 832 extra = Extra.allow 833 834 consent_url: str = Field( 835 ..., 836 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 837 examples=[ 838 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 839 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 840 ], 841 title="Consent URL", 842 ) 843 scope: Optional[str] = Field( 844 None, 845 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 846 examples=["user:read user:read_orders workspaces:read"], 847 title="Scopes", 848 ) 849 access_token_url: str = Field( 850 ..., 851 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 852 examples=[ 853 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 854 ], 855 title="Access Token URL", 856 ) 857 access_token_headers: Optional[Dict[str, Any]] = Field( 858 None, 859 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 860 examples=[ 861 { 862 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 863 } 864 ], 865 title="Access Token Headers", 866 ) 867 access_token_params: Optional[Dict[str, Any]] = Field( 868 None, 869 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 870 examples=[ 871 { 872 "{{ auth_code_key }}": "{{ auth_code_value }}", 873 "{{ client_id_key }}": "{{ client_id_value }}", 874 "{{ client_secret_key }}": "{{ client_secret_value }}", 875 } 876 ], 877 title="Access Token Query Params (Json Encoded)", 878 ) 879 extract_output: Optional[List[str]] = Field( 880 None, 881 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 882 examples=[["access_token", "refresh_token", "other_field"]], 883 title="Extract Output", 884 ) 885 state: Optional[State] = Field( 886 None, 887 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 888 examples=[{"min": 7, "max": 128}], 889 title="Configurable State Query Param", 890 ) 891 client_id_key: Optional[str] = Field( 892 None, 893 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 894 examples=["my_custom_client_id_key_name"], 895 title="Client ID Key Override", 896 ) 897 client_secret_key: Optional[str] = Field( 898 None, 899 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 900 examples=["my_custom_client_secret_key_name"], 901 title="Client Secret Key Override", 902 ) 903 scope_key: Optional[str] = Field( 904 None, 905 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 906 examples=["my_custom_scope_key_key_name"], 907 title="Scopes Key Override", 908 ) 909 state_key: Optional[str] = Field( 910 None, 911 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 912 examples=["my_custom_state_key_key_name"], 913 title="State Key Override", 914 ) 915 auth_code_key: Optional[str] = Field( 916 None, 917 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 918 examples=["my_custom_auth_code_key_name"], 919 title="Auth Code Key Override", 920 ) 921 redirect_uri_key: Optional[str] = Field( 922 None, 923 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 924 examples=["my_custom_redirect_uri_key_name"], 925 title="Redirect URI Key Override", 926 ) 927 928 929class OAuthConfigSpecification(BaseModel): 930 class Config: 931 extra = Extra.allow 932 933 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 934 None, 935 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 936 examples=[ 937 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 938 { 939 "app_id": { 940 "type": "string", 941 "path_in_connector_config": ["info", "app_id"], 942 } 943 }, 944 ], 945 title="OAuth user input", 946 ) 947 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 948 None, 949 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 950 title="DeclarativeOAuth Connector Specification", 951 ) 952 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 953 None, 954 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 955 examples=[ 956 { 957 "refresh_token": { 958 "type": "string,", 959 "path_in_connector_config": ["credentials", "refresh_token"], 960 } 961 } 962 ], 963 title="OAuth output specification", 964 ) 965 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 966 None, 967 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 968 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 969 title="OAuth input specification", 970 ) 971 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 972 None, 973 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 974 examples=[ 975 { 976 "client_id": { 977 "type": "string,", 978 "path_in_connector_config": ["credentials", "client_id"], 979 }, 980 "client_secret": { 981 "type": "string,", 982 "path_in_connector_config": ["credentials", "client_secret"], 983 }, 984 } 985 ], 986 title="OAuth server output specification", 987 ) 988 989 990class OffsetIncrement(BaseModel): 991 type: Literal["OffsetIncrement"] 992 page_size: Optional[Union[int, str]] = Field( 993 None, 994 description="The number of records to include in each pages.", 995 examples=[100, "{{ config['page_size'] }}"], 996 title="Limit", 997 ) 998 inject_on_first_request: Optional[bool] = Field( 999 False, 1000 description="Using the `offset` with value `0` during the first request", 1001 title="Inject Offset on First Request", 1002 ) 1003 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1004 1005 1006class PageIncrement(BaseModel): 1007 type: Literal["PageIncrement"] 1008 page_size: Optional[Union[int, str]] = Field( 1009 None, 1010 description="The number of records to include in each pages.", 1011 examples=[100, "100", "{{ config['page_size'] }}"], 1012 title="Page Size", 1013 ) 1014 start_from_page: Optional[int] = Field( 1015 0, 1016 description="Index of the first page to request.", 1017 examples=[0, 1], 1018 title="Start From Page", 1019 ) 1020 inject_on_first_request: Optional[bool] = Field( 1021 False, 1022 description="Using the `page number` with value defined by `start_from_page` during the first request", 1023 title="Inject Page Number on First Request", 1024 ) 1025 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1026 1027 1028class PrimaryKey(BaseModel): 1029 __root__: Union[str, List[str], List[List[str]]] = Field( 1030 ..., 1031 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1032 examples=["id", ["code", "type"]], 1033 title="Primary Key", 1034 ) 1035 1036 1037class PropertyLimitType(Enum): 1038 characters = "characters" 1039 property_count = "property_count" 1040 1041 1042class PropertyChunking(BaseModel): 1043 type: Literal["PropertyChunking"] 1044 property_limit_type: PropertyLimitType = Field( 1045 ..., 1046 description="The type used to determine the maximum number of properties per chunk", 1047 title="Property Limit Type", 1048 ) 1049 property_limit: Optional[int] = Field( 1050 None, 1051 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1052 title="Property Limit", 1053 ) 1054 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1055 None, 1056 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1057 title="Record Merge Strategy", 1058 ) 1059 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1060 1061 1062class RecordFilter(BaseModel): 1063 type: Literal["RecordFilter"] 1064 condition: Optional[str] = Field( 1065 "", 1066 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1067 examples=[ 1068 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1069 "{{ record.status in ['active', 'expired'] }}", 1070 ], 1071 title="Condition", 1072 ) 1073 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1074 1075 1076class SchemaNormalization(Enum): 1077 Default = "Default" 1078 None_ = "None" 1079 1080 1081class RemoveFields(BaseModel): 1082 type: Literal["RemoveFields"] 1083 condition: Optional[str] = Field( 1084 "", 1085 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1086 examples=[ 1087 "{{ property|string == '' }}", 1088 "{{ property is integer }}", 1089 "{{ property|length > 5 }}", 1090 "{{ property == 'some_string_to_match' }}", 1091 ], 1092 ) 1093 field_pointers: List[List[str]] = Field( 1094 ..., 1095 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1096 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1097 title="Field Paths", 1098 ) 1099 1100 1101class RequestPath(BaseModel): 1102 type: Literal["RequestPath"] 1103 1104 1105class InjectInto(Enum): 1106 request_parameter = "request_parameter" 1107 header = "header" 1108 body_data = "body_data" 1109 body_json = "body_json" 1110 1111 1112class RequestOption(BaseModel): 1113 type: Literal["RequestOption"] 1114 inject_into: InjectInto = Field( 1115 ..., 1116 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1117 examples=["request_parameter", "header", "body_data", "body_json"], 1118 title="Inject Into", 1119 ) 1120 field_name: Optional[str] = Field( 1121 None, 1122 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1123 examples=["segment_id"], 1124 title="Field Name", 1125 ) 1126 field_path: Optional[List[str]] = Field( 1127 None, 1128 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1129 examples=[["data", "viewer", "id"]], 1130 title="Field Path", 1131 ) 1132 1133 1134class Schemas(BaseModel): 1135 pass 1136 1137 class Config: 1138 extra = Extra.allow 1139 1140 1141class LegacySessionTokenAuthenticator(BaseModel): 1142 type: Literal["LegacySessionTokenAuthenticator"] 1143 header: str = Field( 1144 ..., 1145 description="The name of the session token header that will be injected in the request", 1146 examples=["X-Session"], 1147 title="Session Request Header", 1148 ) 1149 login_url: str = Field( 1150 ..., 1151 description="Path of the login URL (do not include the base URL)", 1152 examples=["session"], 1153 title="Login Path", 1154 ) 1155 session_token: Optional[str] = Field( 1156 None, 1157 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1158 example=["{{ config['session_token'] }}"], 1159 title="Session Token", 1160 ) 1161 session_token_response_key: str = Field( 1162 ..., 1163 description="Name of the key of the session token to be extracted from the response", 1164 examples=["id"], 1165 title="Response Token Response Key", 1166 ) 1167 username: Optional[str] = Field( 1168 None, 1169 description="Username used to authenticate and obtain a session token", 1170 examples=[" {{ config['username'] }}"], 1171 title="Username", 1172 ) 1173 password: Optional[str] = Field( 1174 "", 1175 description="Password used to authenticate and obtain a session token", 1176 examples=["{{ config['password'] }}", ""], 1177 title="Password", 1178 ) 1179 validate_session_url: str = Field( 1180 ..., 1181 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1182 examples=["user/current"], 1183 title="Validate Session Path", 1184 ) 1185 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1186 1187 1188class Action1(Enum): 1189 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1190 RESET = "RESET" 1191 1192 1193class PaginationResetLimits(BaseModel): 1194 type: Literal["PaginationResetLimits"] 1195 number_of_records: Optional[int] = None 1196 1197 1198class CsvDecoder(BaseModel): 1199 type: Literal["CsvDecoder"] 1200 encoding: Optional[str] = "utf-8" 1201 delimiter: Optional[str] = "," 1202 set_values_to_none: Optional[List[str]] = None 1203 1204 1205class AsyncJobStatusMap(BaseModel): 1206 type: Optional[Literal["AsyncJobStatusMap"]] = None 1207 running: List[str] 1208 completed: List[str] 1209 failed: List[str] 1210 timeout: List[str] 1211 1212 1213class ValueType(Enum): 1214 string = "string" 1215 number = "number" 1216 integer = "integer" 1217 boolean = "boolean" 1218 1219 1220class WaitTimeFromHeader(BaseModel): 1221 type: Literal["WaitTimeFromHeader"] 1222 header: str = Field( 1223 ..., 1224 description="The name of the response header defining how long to wait before retrying.", 1225 examples=["Retry-After"], 1226 title="Response Header Name", 1227 ) 1228 regex: Optional[str] = Field( 1229 None, 1230 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1231 examples=["([-+]?\\d+)"], 1232 title="Extraction Regex", 1233 ) 1234 max_waiting_time_in_seconds: Optional[float] = Field( 1235 None, 1236 description="Given the value extracted from the header is greater than this value, stop the stream.", 1237 examples=[3600], 1238 title="Max Waiting Time in Seconds", 1239 ) 1240 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1241 1242 1243class WaitUntilTimeFromHeader(BaseModel): 1244 type: Literal["WaitUntilTimeFromHeader"] 1245 header: str = Field( 1246 ..., 1247 description="The name of the response header defining how long to wait before retrying.", 1248 examples=["wait_time"], 1249 title="Response Header", 1250 ) 1251 min_wait: Optional[Union[float, str]] = Field( 1252 None, 1253 description="Minimum time to wait before retrying.", 1254 examples=[10, "60"], 1255 title="Minimum Wait Time", 1256 ) 1257 regex: Optional[str] = Field( 1258 None, 1259 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1260 examples=["([-+]?\\d+)"], 1261 title="Extraction Regex", 1262 ) 1263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1264 1265 1266class ComponentMappingDefinition(BaseModel): 1267 type: Literal["ComponentMappingDefinition"] 1268 field_path: List[str] = Field( 1269 ..., 1270 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1271 examples=[ 1272 ["name"], 1273 ["retriever", "requester", "url"], 1274 ["retriever", "requester", "{{ components_values.field }}"], 1275 ["*", "**", "name"], 1276 ], 1277 title="Field Path", 1278 ) 1279 value: str = Field( 1280 ..., 1281 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1282 examples=[ 1283 "{{ components_values['updates'] }}", 1284 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1285 "{{ config['segment_id'] }}", 1286 "{{ stream_slice['parent_id'] }}", 1287 "{{ stream_slice['extra_fields']['name'] }}", 1288 ], 1289 title="Value", 1290 ) 1291 value_type: Optional[ValueType] = Field( 1292 None, 1293 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1294 title="Value Type", 1295 ) 1296 create_or_update: Optional[bool] = Field( 1297 False, 1298 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1299 title="Create or Update", 1300 ) 1301 condition: Optional[str] = Field( 1302 None, 1303 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1304 examples=[ 1305 "{{ components_values.get('cursor_field', None) }}", 1306 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1307 ], 1308 title="Condition", 1309 ) 1310 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1311 1312 1313class StreamConfig(BaseModel): 1314 type: Literal["StreamConfig"] 1315 configs_pointer: List[str] = Field( 1316 ..., 1317 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1318 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1319 title="Configs Pointer", 1320 ) 1321 default_values: Optional[List[Dict[str, Any]]] = Field( 1322 None, 1323 description="A list of default values, each matching the structure expected from the parsed component value.", 1324 title="Default Values", 1325 ) 1326 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1327 1328 1329class ConfigComponentsResolver(BaseModel): 1330 type: Literal["ConfigComponentsResolver"] 1331 stream_config: Union[List[StreamConfig], StreamConfig] 1332 components_mapping: List[ComponentMappingDefinition] 1333 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1334 1335 1336class StreamParametersDefinition(BaseModel): 1337 type: Literal["StreamParametersDefinition"] 1338 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1339 ..., 1340 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1341 examples=[ 1342 [ 1343 { 1344 "name": "test stream", 1345 "$parameters": {"entity": "test entity"}, 1346 "primary_key": "test key", 1347 } 1348 ] 1349 ], 1350 title="Stream Parameters", 1351 ) 1352 1353 1354class ParametrizedComponentsResolver(BaseModel): 1355 type: Literal["ParametrizedComponentsResolver"] 1356 stream_parameters: StreamParametersDefinition 1357 components_mapping: List[ComponentMappingDefinition] 1358 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1359 1360 1361class RequestBodyPlainText(BaseModel): 1362 type: Literal["RequestBodyPlainText"] 1363 value: str 1364 1365 1366class RequestBodyUrlEncodedForm(BaseModel): 1367 type: Literal["RequestBodyUrlEncodedForm"] 1368 value: Dict[str, str] 1369 1370 1371class RequestBodyJsonObject(BaseModel): 1372 type: Literal["RequestBodyJsonObject"] 1373 value: Dict[str, Any] 1374 1375 1376class RequestBodyGraphQlQuery(BaseModel): 1377 class Config: 1378 extra = Extra.allow 1379 1380 query: str = Field(..., description="The GraphQL query to be executed") 1381 1382 1383class ValidateAdheresToSchema(BaseModel): 1384 type: Literal["ValidateAdheresToSchema"] 1385 base_schema: Union[str, Dict[str, Any]] = Field( 1386 ..., 1387 description="The base JSON schema against which the user-provided schema will be validated.", 1388 examples=[ 1389 "{{ config['report_validation_schema'] }}", 1390 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1391 { 1392 "$schema": "http://json-schema.org/draft-07/schema#", 1393 "title": "Person", 1394 "type": "object", 1395 "properties": { 1396 "name": {"type": "string", "description": "The person's name"}, 1397 "age": { 1398 "type": "integer", 1399 "minimum": 0, 1400 "description": "The person's age", 1401 }, 1402 }, 1403 "required": ["name", "age"], 1404 }, 1405 ], 1406 title="Base JSON Schema", 1407 ) 1408 1409 1410class CustomValidationStrategy(BaseModel): 1411 class Config: 1412 extra = Extra.allow 1413 1414 type: Literal["CustomValidationStrategy"] 1415 class_name: str = Field( 1416 ..., 1417 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1418 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1419 title="Class Name", 1420 ) 1421 1422 1423class ConfigRemapField(BaseModel): 1424 type: Literal["ConfigRemapField"] 1425 map: Union[Dict[str, Any], str] = Field( 1426 ..., 1427 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1428 examples=[ 1429 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1430 "{{ config['status_mapping'] }}", 1431 ], 1432 title="Value Mapping", 1433 ) 1434 field_path: List[str] = Field( 1435 ..., 1436 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1437 examples=[ 1438 ["status"], 1439 ["data", "status"], 1440 ["data", "{{ config.name }}", "status"], 1441 ["data", "*", "status"], 1442 ], 1443 title="Field Path", 1444 ) 1445 1446 1447class ConfigRemoveFields(BaseModel): 1448 type: Literal["ConfigRemoveFields"] 1449 field_pointers: List[List[str]] = Field( 1450 ..., 1451 description="A list of field pointers to be removed from the config.", 1452 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1453 title="Field Pointers", 1454 ) 1455 condition: Optional[str] = Field( 1456 "", 1457 description="Fields will be removed if expression is evaluated to True.", 1458 examples=[ 1459 "{{ config['environemnt'] == 'sandbox' }}", 1460 "{{ property is integer }}", 1461 "{{ property|length > 5 }}", 1462 "{{ property == 'some_string_to_match' }}", 1463 ], 1464 ) 1465 1466 1467class CustomConfigTransformation(BaseModel): 1468 type: Literal["CustomConfigTransformation"] 1469 class_name: str = Field( 1470 ..., 1471 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1472 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1473 ) 1474 parameters: Optional[Dict[str, Any]] = Field( 1475 None, 1476 alias="$parameters", 1477 description="Additional parameters to be passed to the custom config transformation.", 1478 ) 1479 1480 1481class AddedFieldDefinition(BaseModel): 1482 type: Literal["AddedFieldDefinition"] 1483 path: List[str] = Field( 1484 ..., 1485 description="List of strings defining the path where to add the value on the record.", 1486 examples=[["segment_id"], ["metadata", "segment_id"]], 1487 title="Path", 1488 ) 1489 value: str = Field( 1490 ..., 1491 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1492 examples=[ 1493 "{{ record['updates'] }}", 1494 "{{ record['MetaData']['LastUpdatedTime'] }}", 1495 "{{ stream_partition['segment_id'] }}", 1496 ], 1497 title="Value", 1498 ) 1499 value_type: Optional[ValueType] = Field( 1500 None, 1501 description="Type of the value. If not specified, the type will be inferred from the value.", 1502 title="Value Type", 1503 ) 1504 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1505 1506 1507class AddFields(BaseModel): 1508 type: Literal["AddFields"] 1509 fields: List[AddedFieldDefinition] = Field( 1510 ..., 1511 description="List of transformations (path and corresponding value) that will be added to the record.", 1512 title="Fields", 1513 ) 1514 condition: Optional[str] = Field( 1515 "", 1516 description="Fields will be added if expression is evaluated to True.", 1517 examples=[ 1518 "{{ property|string == '' }}", 1519 "{{ property is integer }}", 1520 "{{ property|length > 5 }}", 1521 "{{ property == 'some_string_to_match' }}", 1522 ], 1523 ) 1524 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1525 1526 1527class ApiKeyAuthenticator(BaseModel): 1528 type: Literal["ApiKeyAuthenticator"] 1529 api_token: Optional[str] = Field( 1530 None, 1531 description="The API key to inject in the request. Fill it in the user inputs.", 1532 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1533 title="API Key", 1534 ) 1535 header: Optional[str] = Field( 1536 None, 1537 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1538 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1539 title="Header Name", 1540 ) 1541 inject_into: Optional[RequestOption] = Field( 1542 None, 1543 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1544 examples=[ 1545 {"inject_into": "header", "field_name": "Authorization"}, 1546 {"inject_into": "request_parameter", "field_name": "authKey"}, 1547 ], 1548 title="Inject API Key Into Outgoing HTTP Request", 1549 ) 1550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1551 1552 1553class AuthFlow(BaseModel): 1554 auth_flow_type: Optional[AuthFlowType] = Field( 1555 None, description="The type of auth to use", title="Auth flow type" 1556 ) 1557 predicate_key: Optional[List[str]] = Field( 1558 None, 1559 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1560 examples=[["credentials", "auth_type"]], 1561 title="Predicate key", 1562 ) 1563 predicate_value: Optional[str] = Field( 1564 None, 1565 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1566 examples=["Oauth"], 1567 title="Predicate value", 1568 ) 1569 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1570 1571 1572class CheckStream(BaseModel): 1573 type: Literal["CheckStream"] 1574 stream_names: Optional[List[str]] = Field( 1575 None, 1576 description="Names of the streams to try reading from when running a check operation.", 1577 examples=[["users"], ["users", "contacts"]], 1578 title="Stream Names", 1579 ) 1580 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1581 1582 1583class IncrementingCountCursor(BaseModel): 1584 type: Literal["IncrementingCountCursor"] 1585 cursor_field: str = Field( 1586 ..., 1587 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1588 examples=["created_at", "{{ config['record_cursor'] }}"], 1589 title="Cursor Field", 1590 ) 1591 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1592 None, 1593 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1594 title="Allow Catalog Defined Cursor Field", 1595 ) 1596 start_value: Optional[Union[str, int]] = Field( 1597 None, 1598 description="The value that determines the earliest record that should be synced.", 1599 examples=[0, "{{ config['start_value'] }}"], 1600 title="Start Value", 1601 ) 1602 start_value_option: Optional[RequestOption] = Field( 1603 None, 1604 description="Optionally configures how the start value will be sent in requests to the source API.", 1605 title="Inject Start Value Into Outgoing HTTP Request", 1606 ) 1607 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1608 1609 1610class DatetimeBasedCursor(BaseModel): 1611 type: Literal["DatetimeBasedCursor"] 1612 clamping: Optional[Clamping] = Field( 1613 None, 1614 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1615 title="Date Range Clamping", 1616 ) 1617 cursor_field: str = Field( 1618 ..., 1619 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1620 examples=["created_at", "{{ config['record_cursor'] }}"], 1621 title="Cursor Field", 1622 ) 1623 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1624 None, 1625 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1626 title="Allow Catalog Defined Cursor Field", 1627 ) 1628 cursor_datetime_formats: Optional[List[str]] = Field( 1629 None, 1630 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1631 examples=[ 1632 "%Y-%m-%d", 1633 "%Y-%m-%d %H:%M:%S", 1634 "%Y-%m-%dT%H:%M:%S", 1635 "%Y-%m-%dT%H:%M:%SZ", 1636 "%Y-%m-%dT%H:%M:%S%z", 1637 "%Y-%m-%dT%H:%M:%S.%fZ", 1638 "%Y-%m-%dT%H:%M:%S.%f%z", 1639 "%Y-%m-%d %H:%M:%S.%f+00:00", 1640 "%s", 1641 "%ms", 1642 ], 1643 title="Cursor Datetime Formats", 1644 ) 1645 start_datetime: Union[MinMaxDatetime, str] = Field( 1646 ..., 1647 description="The datetime that determines the earliest record that should be synced.", 1648 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1649 title="Start Datetime", 1650 ) 1651 start_time_option: Optional[RequestOption] = Field( 1652 None, 1653 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1654 title="Inject Start Time Into Outgoing HTTP Request", 1655 ) 1656 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1657 None, 1658 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1659 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1660 title="End Datetime", 1661 ) 1662 end_time_option: Optional[RequestOption] = Field( 1663 None, 1664 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1665 title="Inject End Time Into Outgoing HTTP Request", 1666 ) 1667 datetime_format: str = Field( 1668 ..., 1669 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1670 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1671 title="Outgoing Datetime Format", 1672 ) 1673 cursor_granularity: Optional[str] = Field( 1674 None, 1675 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1676 examples=["PT1S"], 1677 title="Cursor Granularity", 1678 ) 1679 is_data_feed: Optional[bool] = Field( 1680 None, 1681 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1682 title="Data Feed API", 1683 ) 1684 is_client_side_incremental: Optional[bool] = Field( 1685 None, 1686 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1687 title="Client-side Incremental Filtering", 1688 ) 1689 is_compare_strictly: Optional[bool] = Field( 1690 False, 1691 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1692 title="Strict Start-End Time Comparison", 1693 ) 1694 global_substream_cursor: Optional[bool] = Field( 1695 False, 1696 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1697 title="Global Substream Cursor", 1698 ) 1699 lookback_window: Optional[str] = Field( 1700 None, 1701 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1702 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1703 title="Lookback Window", 1704 ) 1705 partition_field_end: Optional[str] = Field( 1706 None, 1707 description="Name of the partition start time field.", 1708 examples=["ending_time"], 1709 title="Partition Field End", 1710 ) 1711 partition_field_start: Optional[str] = Field( 1712 None, 1713 description="Name of the partition end time field.", 1714 examples=["starting_time"], 1715 title="Partition Field Start", 1716 ) 1717 step: Optional[str] = Field( 1718 None, 1719 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1720 examples=["P1W", "{{ config['step_increment'] }}"], 1721 title="Step", 1722 ) 1723 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1724 1725 1726class JwtAuthenticator(BaseModel): 1727 type: Literal["JwtAuthenticator"] 1728 secret_key: str = Field( 1729 ..., 1730 description="Secret used to sign the JSON web token.", 1731 examples=["{{ config['secret_key'] }}"], 1732 title="Secret Key", 1733 ) 1734 base64_encode_secret_key: Optional[bool] = Field( 1735 False, 1736 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1737 title="Base64-encode Secret Key", 1738 ) 1739 algorithm: Algorithm = Field( 1740 ..., 1741 description="Algorithm used to sign the JSON web token.", 1742 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1743 title="Algorithm", 1744 ) 1745 token_duration: Optional[int] = Field( 1746 1200, 1747 description="The amount of time in seconds a JWT token can be valid after being issued.", 1748 examples=[1200, 3600], 1749 title="Token Duration", 1750 ) 1751 header_prefix: Optional[str] = Field( 1752 None, 1753 description="The prefix to be used within the Authentication header.", 1754 examples=["Bearer", "Basic"], 1755 title="Header Prefix", 1756 ) 1757 jwt_headers: Optional[JwtHeaders] = Field( 1758 None, 1759 description="JWT headers used when signing JSON web token.", 1760 title="JWT Headers", 1761 ) 1762 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1763 None, 1764 description="Additional headers to be included with the JWT headers object.", 1765 title="Additional JWT Headers", 1766 ) 1767 jwt_payload: Optional[JwtPayload] = Field( 1768 None, 1769 description="JWT Payload used when signing JSON web token.", 1770 title="JWT Payload", 1771 ) 1772 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1773 None, 1774 description="Additional properties to be added to the JWT payload.", 1775 title="Additional JWT Payload Properties", 1776 ) 1777 passphrase: Optional[str] = Field( 1778 None, 1779 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1780 examples=["{{ config['passphrase'] }}"], 1781 title="Passphrase", 1782 ) 1783 request_option: Optional[RequestOption] = Field( 1784 None, 1785 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1786 title="Request Option", 1787 ) 1788 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1789 1790 1791class OAuthAuthenticator(BaseModel): 1792 type: Literal["OAuthAuthenticator"] 1793 client_id_name: Optional[str] = Field( 1794 "client_id", 1795 description="The name of the property to use to refresh the `access_token`.", 1796 examples=["custom_app_id"], 1797 title="Client ID Property Name", 1798 ) 1799 client_id: Optional[str] = Field( 1800 None, 1801 description="The OAuth client ID. Fill it in the user inputs.", 1802 examples=[ 1803 "{{ config['client_id'] }}", 1804 "{{ config['credentials']['client_id }}", 1805 ], 1806 title="Client ID", 1807 ) 1808 client_secret_name: Optional[str] = Field( 1809 "client_secret", 1810 description="The name of the property to use to refresh the `access_token`.", 1811 examples=["custom_app_secret"], 1812 title="Client Secret Property Name", 1813 ) 1814 client_secret: Optional[str] = Field( 1815 None, 1816 description="The OAuth client secret. Fill it in the user inputs.", 1817 examples=[ 1818 "{{ config['client_secret'] }}", 1819 "{{ config['credentials']['client_secret }}", 1820 ], 1821 title="Client Secret", 1822 ) 1823 refresh_token_name: Optional[str] = Field( 1824 "refresh_token", 1825 description="The name of the property to use to refresh the `access_token`.", 1826 examples=["custom_app_refresh_value"], 1827 title="Refresh Token Property Name", 1828 ) 1829 refresh_token: Optional[str] = Field( 1830 None, 1831 description="Credential artifact used to get a new access token.", 1832 examples=[ 1833 "{{ config['refresh_token'] }}", 1834 "{{ config['credentials]['refresh_token'] }}", 1835 ], 1836 title="Refresh Token", 1837 ) 1838 token_refresh_endpoint: Optional[str] = Field( 1839 None, 1840 description="The full URL to call to obtain a new access token.", 1841 examples=["https://connect.squareup.com/oauth2/token"], 1842 title="Token Refresh Endpoint", 1843 ) 1844 access_token_name: Optional[str] = Field( 1845 "access_token", 1846 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1847 examples=["access_token"], 1848 title="Access Token Property Name", 1849 ) 1850 access_token_value: Optional[str] = Field( 1851 None, 1852 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1853 examples=["secret_access_token_value"], 1854 title="Access Token Value", 1855 ) 1856 expires_in_name: Optional[str] = Field( 1857 "expires_in", 1858 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1859 examples=["expires_in"], 1860 title="Token Expiry Property Name", 1861 ) 1862 grant_type_name: Optional[str] = Field( 1863 "grant_type", 1864 description="The name of the property to use to refresh the `access_token`.", 1865 examples=["custom_grant_type"], 1866 title="Grant Type Property Name", 1867 ) 1868 grant_type: Optional[str] = Field( 1869 "refresh_token", 1870 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1871 examples=["refresh_token", "client_credentials"], 1872 title="Grant Type", 1873 ) 1874 refresh_request_body: Optional[Dict[str, Any]] = Field( 1875 None, 1876 description="Body of the request sent to get a new access token.", 1877 examples=[ 1878 { 1879 "applicationId": "{{ config['application_id'] }}", 1880 "applicationSecret": "{{ config['application_secret'] }}", 1881 "token": "{{ config['token'] }}", 1882 } 1883 ], 1884 title="Refresh Request Body", 1885 ) 1886 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1887 None, 1888 description="Headers of the request sent to get a new access token.", 1889 examples=[ 1890 { 1891 "Authorization": "<AUTH_TOKEN>", 1892 "Content-Type": "application/x-www-form-urlencoded", 1893 } 1894 ], 1895 title="Refresh Request Headers", 1896 ) 1897 scopes: Optional[List[str]] = Field( 1898 None, 1899 description="List of scopes that should be granted to the access token.", 1900 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1901 title="Scopes", 1902 ) 1903 token_expiry_date: Optional[str] = Field( 1904 None, 1905 description="The access token expiry date.", 1906 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1907 title="Token Expiry Date", 1908 ) 1909 token_expiry_date_format: Optional[str] = Field( 1910 None, 1911 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1912 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1913 title="Token Expiry Date Format", 1914 ) 1915 refresh_token_error_status_codes: Optional[List[int]] = Field( 1916 None, 1917 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1918 examples=[[400, 500]], 1919 title="Refresh Token Error Status Codes", 1920 ) 1921 refresh_token_error_key: Optional[str] = Field( 1922 None, 1923 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1924 examples=["error"], 1925 title="Refresh Token Error Key", 1926 ) 1927 refresh_token_error_values: Optional[List[str]] = Field( 1928 None, 1929 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1930 examples=[["invalid_grant", "invalid_permissions"]], 1931 title="Refresh Token Error Values", 1932 ) 1933 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1934 None, 1935 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1936 title="Refresh Token Updater", 1937 ) 1938 profile_assertion: Optional[JwtAuthenticator] = Field( 1939 None, 1940 description="The authenticator being used to authenticate the client authenticator.", 1941 title="Profile Assertion", 1942 ) 1943 use_profile_assertion: Optional[bool] = Field( 1944 False, 1945 description="Enable using profile assertion as a flow for OAuth authorization.", 1946 title="Use Profile Assertion", 1947 ) 1948 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1949 1950 1951class FixedWindowCallRatePolicy(BaseModel): 1952 class Config: 1953 extra = Extra.allow 1954 1955 type: Literal["FixedWindowCallRatePolicy"] 1956 period: str = Field( 1957 ..., description="The time interval for the rate limit window.", title="Period" 1958 ) 1959 call_limit: int = Field( 1960 ..., 1961 description="The maximum number of calls allowed within the period.", 1962 title="Call Limit", 1963 ) 1964 matchers: List[HttpRequestRegexMatcher] = Field( 1965 ..., 1966 description="List of matchers that define which requests this policy applies to.", 1967 title="Matchers", 1968 ) 1969 1970 1971class MovingWindowCallRatePolicy(BaseModel): 1972 class Config: 1973 extra = Extra.allow 1974 1975 type: Literal["MovingWindowCallRatePolicy"] 1976 rates: List[Rate] = Field( 1977 ..., 1978 description="List of rates that define the call limits for different time intervals.", 1979 title="Rates", 1980 ) 1981 matchers: List[HttpRequestRegexMatcher] = Field( 1982 ..., 1983 description="List of matchers that define which requests this policy applies to.", 1984 title="Matchers", 1985 ) 1986 1987 1988class UnlimitedCallRatePolicy(BaseModel): 1989 class Config: 1990 extra = Extra.allow 1991 1992 type: Literal["UnlimitedCallRatePolicy"] 1993 matchers: List[HttpRequestRegexMatcher] = Field( 1994 ..., 1995 description="List of matchers that define which requests this policy applies to.", 1996 title="Matchers", 1997 ) 1998 1999 2000class DefaultErrorHandler(BaseModel): 2001 type: Literal["DefaultErrorHandler"] 2002 backoff_strategies: Optional[ 2003 List[ 2004 Union[ 2005 ConstantBackoffStrategy, 2006 ExponentialBackoffStrategy, 2007 WaitTimeFromHeader, 2008 WaitUntilTimeFromHeader, 2009 CustomBackoffStrategy, 2010 ] 2011 ] 2012 ] = Field( 2013 None, 2014 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2015 title="Backoff Strategies", 2016 ) 2017 max_retries: Optional[int] = Field( 2018 5, 2019 description="The maximum number of time to retry a retryable request before giving up and failing.", 2020 examples=[5, 0, 10], 2021 title="Max Retry Count", 2022 ) 2023 response_filters: Optional[List[HttpResponseFilter]] = Field( 2024 None, 2025 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2026 title="Response Filters", 2027 ) 2028 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2029 2030 2031class DefaultPaginator(BaseModel): 2032 type: Literal["DefaultPaginator"] 2033 pagination_strategy: Union[ 2034 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2035 ] = Field( 2036 ..., 2037 description="Strategy defining how records are paginated.", 2038 title="Pagination Strategy", 2039 ) 2040 page_size_option: Optional[RequestOption] = Field( 2041 None, title="Inject Page Size Into Outgoing HTTP Request" 2042 ) 2043 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2044 None, 2045 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2046 title="Inject Page Token Into Outgoing HTTP Request", 2047 ) 2048 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2049 2050 2051class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2052 type: Literal["ApiKey"] 2053 inject_into: RequestOption = Field( 2054 ..., 2055 description="Configure how the API Key will be sent in requests to the source API.", 2056 examples=[ 2057 {"inject_into": "header", "field_name": "Authorization"}, 2058 {"inject_into": "request_parameter", "field_name": "authKey"}, 2059 ], 2060 title="Inject API Key Into Outgoing HTTP Request", 2061 ) 2062 api_token: Optional[str] = Field( 2063 "{{ session_token }}", 2064 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2065 examples=[ 2066 "{{ session_token }}", 2067 "Token {{ session_token }}", 2068 "Bearer {{ session_token }}", 2069 ], 2070 title="API Token Template", 2071 ) 2072 2073 2074class JsonSchemaPropertySelector(BaseModel): 2075 type: Literal["JsonSchemaPropertySelector"] 2076 transformations: Optional[ 2077 List[ 2078 Union[ 2079 AddFields, 2080 RemoveFields, 2081 KeysToLower, 2082 KeysToSnakeCase, 2083 FlattenFields, 2084 DpathFlattenFields, 2085 KeysReplace, 2086 CustomTransformation, 2087 ] 2088 ] 2089 ] = Field( 2090 None, 2091 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2092 title="Transformations", 2093 ) 2094 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2095 2096 2097class ListPartitionRouter(BaseModel): 2098 type: Literal["ListPartitionRouter"] 2099 cursor_field: str = Field( 2100 ..., 2101 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2102 examples=["section", "{{ config['section_key'] }}"], 2103 title="Current Partition Value Identifier", 2104 ) 2105 values: Union[str, List[str]] = Field( 2106 ..., 2107 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2108 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2109 title="Partition Values", 2110 ) 2111 request_option: Optional[RequestOption] = Field( 2112 None, 2113 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2114 title="Inject Partition Value Into Outgoing HTTP Request", 2115 ) 2116 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2117 2118 2119class RecordSelector(BaseModel): 2120 type: Literal["RecordSelector"] 2121 extractor: Union[DpathExtractor, CustomRecordExtractor] 2122 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2123 None, 2124 description="Responsible for filtering records to be emitted by the Source.", 2125 title="Record Filter", 2126 ) 2127 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2128 None, 2129 description="Responsible for normalization according to the schema.", 2130 title="Schema Normalization", 2131 ) 2132 transform_before_filtering: Optional[bool] = Field( 2133 None, 2134 description="If true, transformation will be applied before record filtering.", 2135 title="Transform Before Filtering", 2136 ) 2137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2138 2139 2140class PaginationReset(BaseModel): 2141 type: Literal["PaginationReset"] 2142 action: Action1 2143 limits: Optional[PaginationResetLimits] = None 2144 2145 2146class GzipDecoder(BaseModel): 2147 type: Literal["GzipDecoder"] 2148 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2149 2150 2151class RequestBodyGraphQL(BaseModel): 2152 type: Literal["RequestBodyGraphQL"] 2153 value: RequestBodyGraphQlQuery 2154 2155 2156class DpathValidator(BaseModel): 2157 type: Literal["DpathValidator"] 2158 field_path: List[str] = Field( 2159 ..., 2160 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2161 examples=[ 2162 ["data"], 2163 ["data", "records"], 2164 ["data", "{{ parameters.name }}"], 2165 ["data", "*", "record"], 2166 ], 2167 title="Field Path", 2168 ) 2169 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2170 ..., 2171 description="The condition that the specified config value will be evaluated against", 2172 title="Validation Strategy", 2173 ) 2174 2175 2176class PredicateValidator(BaseModel): 2177 type: Literal["PredicateValidator"] 2178 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2179 ..., 2180 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2181 examples=[ 2182 "test-value", 2183 "{{ config['api_version'] }}", 2184 "{{ config['tenant_id'] }}", 2185 123, 2186 ], 2187 title="Value", 2188 ) 2189 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2190 ..., 2191 description="The validation strategy to apply to the value.", 2192 title="Validation Strategy", 2193 ) 2194 2195 2196class ConfigAddFields(BaseModel): 2197 type: Literal["ConfigAddFields"] 2198 fields: List[AddedFieldDefinition] = Field( 2199 ..., 2200 description="A list of transformations (path and corresponding value) that will be added to the config.", 2201 title="Fields", 2202 ) 2203 condition: Optional[str] = Field( 2204 "", 2205 description="Fields will be added if expression is evaluated to True.", 2206 examples=[ 2207 "{{ config['environemnt'] == 'sandbox' }}", 2208 "{{ property is integer }}", 2209 "{{ property|length > 5 }}", 2210 "{{ property == 'some_string_to_match' }}", 2211 ], 2212 ) 2213 2214 2215class CompositeErrorHandler(BaseModel): 2216 type: Literal["CompositeErrorHandler"] 2217 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2218 Field( 2219 ..., 2220 description="List of error handlers to iterate on to determine how to handle a failed response.", 2221 title="Error Handlers", 2222 ) 2223 ) 2224 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2225 2226 2227class HTTPAPIBudget(BaseModel): 2228 class Config: 2229 extra = Extra.allow 2230 2231 type: Literal["HTTPAPIBudget"] 2232 policies: List[ 2233 Union[ 2234 FixedWindowCallRatePolicy, 2235 MovingWindowCallRatePolicy, 2236 UnlimitedCallRatePolicy, 2237 ] 2238 ] = Field( 2239 ..., 2240 description="List of call rate policies that define how many calls are allowed.", 2241 title="Policies", 2242 ) 2243 ratelimit_reset_header: Optional[str] = Field( 2244 "ratelimit-reset", 2245 description="The HTTP response header name that indicates when the rate limit resets.", 2246 title="Rate Limit Reset Header", 2247 ) 2248 ratelimit_remaining_header: Optional[str] = Field( 2249 "ratelimit-remaining", 2250 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2251 title="Rate Limit Remaining Header", 2252 ) 2253 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2254 [429], 2255 description="List of HTTP status codes that indicate a rate limit has been hit.", 2256 title="Status Codes for Rate Limit Hit", 2257 ) 2258 2259 2260class ZipfileDecoder(BaseModel): 2261 class Config: 2262 extra = Extra.allow 2263 2264 type: Literal["ZipfileDecoder"] 2265 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2266 ..., 2267 description="Parser to parse the decompressed data from the zipfile(s).", 2268 title="Parser", 2269 ) 2270 2271 2272class ConfigMigration(BaseModel): 2273 type: Literal["ConfigMigration"] 2274 description: Optional[str] = Field( 2275 None, description="The description/purpose of the config migration." 2276 ) 2277 transformations: List[ 2278 Union[ 2279 ConfigRemapField, 2280 ConfigAddFields, 2281 ConfigRemoveFields, 2282 CustomConfigTransformation, 2283 ] 2284 ] = Field( 2285 ..., 2286 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2287 title="Transformations", 2288 ) 2289 2290 2291class ConfigNormalizationRules(BaseModel): 2292 class Config: 2293 extra = Extra.forbid 2294 2295 type: Literal["ConfigNormalizationRules"] 2296 config_migrations: Optional[List[ConfigMigration]] = Field( 2297 [], 2298 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2299 title="Config Migrations", 2300 ) 2301 transformations: Optional[ 2302 List[ 2303 Union[ 2304 ConfigRemapField, 2305 ConfigAddFields, 2306 ConfigRemoveFields, 2307 CustomConfigTransformation, 2308 ] 2309 ] 2310 ] = Field( 2311 [], 2312 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2313 title="Transformations", 2314 ) 2315 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2316 [], 2317 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2318 title="Validations", 2319 ) 2320 2321 2322class Spec(BaseModel): 2323 type: Literal["Spec"] 2324 connection_specification: Dict[str, Any] = Field( 2325 ..., 2326 description="A connection specification describing how a the connector can be configured.", 2327 title="Connection Specification", 2328 ) 2329 documentation_url: Optional[str] = Field( 2330 None, 2331 description="URL of the connector's documentation page.", 2332 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2333 title="Documentation URL", 2334 ) 2335 advanced_auth: Optional[AuthFlow] = Field( 2336 None, 2337 description="Advanced specification for configuring the authentication flow.", 2338 title="Advanced Auth", 2339 ) 2340 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2341 None, title="Config Normalization Rules" 2342 ) 2343 2344 2345class DeclarativeSource1(BaseModel): 2346 class Config: 2347 extra = Extra.forbid 2348 2349 type: Literal["DeclarativeSource"] 2350 check: Union[CheckStream, CheckDynamicStream] 2351 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2352 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2353 version: str = Field( 2354 ..., 2355 description="The version of the Airbyte CDK used to build and test the source.", 2356 ) 2357 schemas: Optional[Schemas] = None 2358 definitions: Optional[Dict[str, Any]] = None 2359 spec: Optional[Spec] = None 2360 concurrency_level: Optional[ConcurrencyLevel] = None 2361 api_budget: Optional[HTTPAPIBudget] = None 2362 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2363 None, 2364 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2365 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2366 title="Maximum Concurrent Asynchronous Jobs", 2367 ) 2368 metadata: Optional[Dict[str, Any]] = Field( 2369 None, 2370 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2371 ) 2372 description: Optional[str] = Field( 2373 None, 2374 description="A description of the connector. It will be presented on the Source documentation page.", 2375 ) 2376 2377 2378class DeclarativeSource2(BaseModel): 2379 class Config: 2380 extra = Extra.forbid 2381 2382 type: Literal["DeclarativeSource"] 2383 check: Union[CheckStream, CheckDynamicStream] 2384 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2385 None 2386 ) 2387 dynamic_streams: List[DynamicDeclarativeStream] 2388 version: str = Field( 2389 ..., 2390 description="The version of the Airbyte CDK used to build and test the source.", 2391 ) 2392 schemas: Optional[Schemas] = None 2393 definitions: Optional[Dict[str, Any]] = None 2394 spec: Optional[Spec] = None 2395 concurrency_level: Optional[ConcurrencyLevel] = None 2396 api_budget: Optional[HTTPAPIBudget] = None 2397 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2398 None, 2399 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2400 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2401 title="Maximum Concurrent Asynchronous Jobs", 2402 ) 2403 metadata: Optional[Dict[str, Any]] = Field( 2404 None, 2405 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2406 ) 2407 description: Optional[str] = Field( 2408 None, 2409 description="A description of the connector. It will be presented on the Source documentation page.", 2410 ) 2411 2412 2413class DeclarativeSource(BaseModel): 2414 class Config: 2415 extra = Extra.forbid 2416 2417 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2418 ..., 2419 description="An API source that extracts data according to its declarative components.", 2420 title="DeclarativeSource", 2421 ) 2422 2423 2424class SelectiveAuthenticator(BaseModel): 2425 class Config: 2426 extra = Extra.allow 2427 2428 type: Literal["SelectiveAuthenticator"] 2429 authenticator_selection_path: List[str] = Field( 2430 ..., 2431 description="Path of the field in config with selected authenticator name", 2432 examples=[["auth"], ["auth", "type"]], 2433 title="Authenticator Selection Path", 2434 ) 2435 authenticators: Dict[ 2436 str, 2437 Union[ 2438 ApiKeyAuthenticator, 2439 BasicHttpAuthenticator, 2440 BearerAuthenticator, 2441 OAuthAuthenticator, 2442 JwtAuthenticator, 2443 SessionTokenAuthenticator, 2444 LegacySessionTokenAuthenticator, 2445 CustomAuthenticator, 2446 NoAuth, 2447 ], 2448 ] = Field( 2449 ..., 2450 description="Authenticators to select from.", 2451 examples=[ 2452 { 2453 "authenticators": { 2454 "token": "#/definitions/ApiKeyAuthenticator", 2455 "oauth": "#/definitions/OAuthAuthenticator", 2456 "jwt": "#/definitions/JwtAuthenticator", 2457 } 2458 } 2459 ], 2460 title="Authenticators", 2461 ) 2462 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2463 2464 2465class ConditionalStreams(BaseModel): 2466 type: Literal["ConditionalStreams"] 2467 condition: str = Field( 2468 ..., 2469 description="Condition that will be evaluated to determine if a set of streams should be available.", 2470 examples=["{{ config['is_sandbox'] }}"], 2471 title="Condition", 2472 ) 2473 streams: List[DeclarativeStream] = Field( 2474 ..., 2475 description="Streams that will be used during an operation based on the condition.", 2476 title="Streams", 2477 ) 2478 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2479 2480 2481class FileUploader(BaseModel): 2482 type: Literal["FileUploader"] 2483 requester: Union[HttpRequester, CustomRequester] = Field( 2484 ..., 2485 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2486 ) 2487 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2488 ..., 2489 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2490 ) 2491 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2492 None, 2493 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2494 ) 2495 filename_extractor: Optional[str] = Field( 2496 None, 2497 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2498 examples=[ 2499 "{{ record.id }}/{{ record.file_name }}/", 2500 "{{ record.id }}_{{ record.file_name }}/", 2501 ], 2502 ) 2503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2504 2505 2506class DeclarativeStream(BaseModel): 2507 class Config: 2508 extra = Extra.allow 2509 2510 type: Literal["DeclarativeStream"] 2511 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2512 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2513 ..., 2514 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2515 title="Retriever", 2516 ) 2517 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2518 None, 2519 description="Component used to fetch data incrementally based on a time field in the data.", 2520 title="Incremental Sync", 2521 ) 2522 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2523 schema_loader: Optional[ 2524 Union[ 2525 InlineSchemaLoader, 2526 DynamicSchemaLoader, 2527 JsonFileSchemaLoader, 2528 List[ 2529 Union[ 2530 InlineSchemaLoader, 2531 DynamicSchemaLoader, 2532 JsonFileSchemaLoader, 2533 CustomSchemaLoader, 2534 ] 2535 ], 2536 CustomSchemaLoader, 2537 ] 2538 ] = Field( 2539 None, 2540 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2541 title="Schema Loader", 2542 ) 2543 transformations: Optional[ 2544 List[ 2545 Union[ 2546 AddFields, 2547 RemoveFields, 2548 KeysToLower, 2549 KeysToSnakeCase, 2550 FlattenFields, 2551 DpathFlattenFields, 2552 KeysReplace, 2553 CustomTransformation, 2554 ] 2555 ] 2556 ] = Field( 2557 None, 2558 description="A list of transformations to be applied to each output record.", 2559 title="Transformations", 2560 ) 2561 state_migrations: Optional[ 2562 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2563 ] = Field( 2564 [], 2565 description="Array of state migrations to be applied on the input state", 2566 title="State Migrations", 2567 ) 2568 file_uploader: Optional[FileUploader] = Field( 2569 None, 2570 description="(experimental) Describes how to fetch a file", 2571 title="File Uploader", 2572 ) 2573 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2574 2575 2576class SessionTokenAuthenticator(BaseModel): 2577 type: Literal["SessionTokenAuthenticator"] 2578 login_requester: HttpRequester = Field( 2579 ..., 2580 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2581 examples=[ 2582 { 2583 "type": "HttpRequester", 2584 "url_base": "https://my_api.com", 2585 "path": "/login", 2586 "authenticator": { 2587 "type": "BasicHttpAuthenticator", 2588 "username": "{{ config.username }}", 2589 "password": "{{ config.password }}", 2590 }, 2591 } 2592 ], 2593 title="Login Requester", 2594 ) 2595 session_token_path: List[str] = Field( 2596 ..., 2597 description="The path in the response body returned from the login requester to the session token.", 2598 examples=[["access_token"], ["result", "token"]], 2599 title="Session Token Path", 2600 ) 2601 expiration_duration: Optional[str] = Field( 2602 None, 2603 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2604 examples=["PT1H", "P1D"], 2605 title="Expiration Duration", 2606 ) 2607 request_authentication: Union[ 2608 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2609 ] = Field( 2610 ..., 2611 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2612 title="Data Request Authentication", 2613 ) 2614 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2615 None, description="Component used to decode the response.", title="Decoder" 2616 ) 2617 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2618 2619 2620class HttpRequester(BaseModelWithDeprecations): 2621 type: Literal["HttpRequester"] 2622 url_base: Optional[str] = Field( 2623 None, 2624 deprecated=True, 2625 deprecation_message="Use `url` field instead.", 2626 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2627 examples=[ 2628 "https://connect.squareup.com/v2", 2629 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2630 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2631 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2632 ], 2633 title="API Base URL", 2634 ) 2635 url: Optional[str] = Field( 2636 None, 2637 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2638 examples=[ 2639 "https://connect.squareup.com/v2", 2640 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2641 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2642 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2643 ], 2644 title="API Endpoint URL", 2645 ) 2646 path: Optional[str] = Field( 2647 None, 2648 deprecated=True, 2649 deprecation_message="Use `url` field instead.", 2650 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2651 examples=[ 2652 "/products", 2653 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2654 "/trades/{{ config['symbol_id'] }}/history", 2655 ], 2656 title="URL Path", 2657 ) 2658 http_method: Optional[HttpMethod] = Field( 2659 HttpMethod.GET, 2660 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2661 examples=["GET", "POST"], 2662 title="HTTP Method", 2663 ) 2664 authenticator: Optional[ 2665 Union[ 2666 ApiKeyAuthenticator, 2667 BasicHttpAuthenticator, 2668 BearerAuthenticator, 2669 OAuthAuthenticator, 2670 JwtAuthenticator, 2671 SessionTokenAuthenticator, 2672 SelectiveAuthenticator, 2673 CustomAuthenticator, 2674 NoAuth, 2675 LegacySessionTokenAuthenticator, 2676 ] 2677 ] = Field( 2678 None, 2679 description="Authentication method to use for requests sent to the API.", 2680 title="Authenticator", 2681 ) 2682 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2683 None, 2684 deprecated=True, 2685 deprecation_message="Use `query_properties` field instead.", 2686 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2687 title="Fetch Properties from Endpoint", 2688 ) 2689 query_properties: Optional[QueryProperties] = Field( 2690 None, 2691 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2692 title="Query Properties", 2693 ) 2694 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2695 None, 2696 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2697 examples=[ 2698 {"unit": "day"}, 2699 { 2700 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2701 }, 2702 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2703 {"sort_by[asc]": "updated_at"}, 2704 ], 2705 title="Query Parameters", 2706 ) 2707 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2708 None, 2709 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2710 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2711 title="Request Headers", 2712 ) 2713 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2714 None, 2715 deprecated=True, 2716 deprecation_message="Use `request_body` field instead.", 2717 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2718 examples=[ 2719 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2720 ], 2721 title="Request Body Payload (Non-JSON)", 2722 ) 2723 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2724 None, 2725 deprecated=True, 2726 deprecation_message="Use `request_body` field instead.", 2727 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2728 examples=[ 2729 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2730 {"key": "{{ config['value'] }}"}, 2731 {"sort": {"field": "updated_at", "order": "ascending"}}, 2732 ], 2733 title="Request Body JSON Payload", 2734 ) 2735 request_body: Optional[ 2736 Union[ 2737 RequestBodyPlainText, 2738 RequestBodyUrlEncodedForm, 2739 RequestBodyJsonObject, 2740 RequestBodyGraphQL, 2741 ] 2742 ] = Field( 2743 None, 2744 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2745 title="Request Body", 2746 ) 2747 error_handler: Optional[ 2748 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2749 ] = Field( 2750 None, 2751 description="Error handler component that defines how to handle errors.", 2752 title="Error Handler", 2753 ) 2754 use_cache: Optional[bool] = Field( 2755 False, 2756 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2757 title="Use Cache", 2758 ) 2759 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2760 2761 2762class DynamicSchemaLoader(BaseModel): 2763 type: Literal["DynamicSchemaLoader"] 2764 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2765 ..., 2766 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2767 title="Retriever", 2768 ) 2769 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2770 None, 2771 description="Responsible for filtering fields to be added to json schema.", 2772 title="Schema Filter", 2773 ) 2774 schema_transformations: Optional[ 2775 List[ 2776 Union[ 2777 AddFields, 2778 RemoveFields, 2779 KeysToLower, 2780 KeysToSnakeCase, 2781 FlattenFields, 2782 DpathFlattenFields, 2783 KeysReplace, 2784 CustomTransformation, 2785 ] 2786 ] 2787 ] = Field( 2788 None, 2789 description="A list of transformations to be applied to the schema.", 2790 title="Schema Transformations", 2791 ) 2792 schema_type_identifier: SchemaTypeIdentifier 2793 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2794 2795 2796class ParentStreamConfig(BaseModel): 2797 type: Literal["ParentStreamConfig"] 2798 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2799 ..., description="Reference to the parent stream.", title="Parent Stream" 2800 ) 2801 parent_key: str = Field( 2802 ..., 2803 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2804 examples=["id", "{{ config['parent_record_id'] }}"], 2805 title="Parent Key", 2806 ) 2807 partition_field: str = Field( 2808 ..., 2809 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2810 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2811 title="Current Parent Key Value Identifier", 2812 ) 2813 request_option: Optional[RequestOption] = Field( 2814 None, 2815 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2816 title="Request Option", 2817 ) 2818 incremental_dependency: Optional[bool] = Field( 2819 False, 2820 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2821 title="Incremental Dependency", 2822 ) 2823 lazy_read_pointer: Optional[List[str]] = Field( 2824 [], 2825 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2826 title="Lazy Read Pointer", 2827 ) 2828 extra_fields: Optional[List[List[str]]] = Field( 2829 None, 2830 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2831 title="Extra Fields", 2832 ) 2833 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2834 2835 2836class PropertiesFromEndpoint(BaseModel): 2837 type: Literal["PropertiesFromEndpoint"] 2838 property_field_path: List[str] = Field( 2839 ..., 2840 description="Describes the path to the field that should be extracted", 2841 examples=[["name"]], 2842 ) 2843 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2844 ..., 2845 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2846 ) 2847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2848 2849 2850class QueryProperties(BaseModel): 2851 type: Literal["QueryProperties"] 2852 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2853 ..., 2854 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2855 title="Property List", 2856 ) 2857 always_include_properties: Optional[List[str]] = Field( 2858 None, 2859 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2860 title="Always Include Properties", 2861 ) 2862 property_chunking: Optional[PropertyChunking] = Field( 2863 None, 2864 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2865 title="Property Chunking", 2866 ) 2867 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2868 None, 2869 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2870 title="Property Selector", 2871 ) 2872 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2873 2874 2875class StateDelegatingStream(BaseModel): 2876 type: Literal["StateDelegatingStream"] 2877 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2878 full_refresh_stream: DeclarativeStream = Field( 2879 ..., 2880 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2881 title="Full Refresh Stream", 2882 ) 2883 incremental_stream: DeclarativeStream = Field( 2884 ..., 2885 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2886 title="Incremental Stream", 2887 ) 2888 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2889 2890 2891class SimpleRetriever(BaseModel): 2892 type: Literal["SimpleRetriever"] 2893 requester: Union[HttpRequester, CustomRequester] = Field( 2894 ..., 2895 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2896 ) 2897 decoder: Optional[ 2898 Union[ 2899 JsonDecoder, 2900 XmlDecoder, 2901 CsvDecoder, 2902 JsonlDecoder, 2903 GzipDecoder, 2904 IterableDecoder, 2905 ZipfileDecoder, 2906 CustomDecoder, 2907 ] 2908 ] = Field( 2909 None, 2910 description="Component decoding the response so records can be extracted.", 2911 title="HTTP Response Format", 2912 ) 2913 record_selector: RecordSelector = Field( 2914 ..., 2915 description="Component that describes how to extract records from a HTTP response.", 2916 ) 2917 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2918 None, 2919 description="Paginator component that describes how to navigate through the API's pages.", 2920 ) 2921 pagination_reset: Optional[PaginationReset] = Field( 2922 None, 2923 description="Describes what triggers pagination reset and how to handle it.", 2924 ) 2925 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2926 False, 2927 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2928 ) 2929 partition_router: Optional[ 2930 Union[ 2931 SubstreamPartitionRouter, 2932 ListPartitionRouter, 2933 GroupingPartitionRouter, 2934 CustomPartitionRouter, 2935 List[ 2936 Union[ 2937 SubstreamPartitionRouter, 2938 ListPartitionRouter, 2939 GroupingPartitionRouter, 2940 CustomPartitionRouter, 2941 ] 2942 ], 2943 ] 2944 ] = Field( 2945 None, 2946 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2947 title="Partition Router", 2948 ) 2949 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2950 2951 2952class AsyncRetriever(BaseModel): 2953 type: Literal["AsyncRetriever"] 2954 record_selector: RecordSelector = Field( 2955 ..., 2956 description="Component that describes how to extract records from a HTTP response.", 2957 ) 2958 status_mapping: AsyncJobStatusMap = Field( 2959 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2960 ) 2961 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2962 ..., description="Responsible for fetching the actual status of the async job." 2963 ) 2964 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2965 None, 2966 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2967 ) 2968 download_extractor: Optional[ 2969 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2970 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2971 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2972 ..., 2973 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2974 ) 2975 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2976 ..., 2977 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2978 ) 2979 polling_job_timeout: Optional[Union[int, str]] = Field( 2980 None, 2981 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2982 ) 2983 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2984 None, 2985 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2986 ) 2987 download_requester: Union[HttpRequester, CustomRequester] = Field( 2988 ..., 2989 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2990 ) 2991 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2992 None, 2993 description="Paginator component that describes how to navigate through the API's pages during download.", 2994 ) 2995 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2996 None, 2997 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2998 ) 2999 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3000 None, 3001 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3002 ) 3003 partition_router: Optional[ 3004 Union[ 3005 ListPartitionRouter, 3006 SubstreamPartitionRouter, 3007 GroupingPartitionRouter, 3008 CustomPartitionRouter, 3009 List[ 3010 Union[ 3011 ListPartitionRouter, 3012 SubstreamPartitionRouter, 3013 GroupingPartitionRouter, 3014 CustomPartitionRouter, 3015 ] 3016 ], 3017 ] 3018 ] = Field( 3019 [], 3020 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3021 title="Partition Router", 3022 ) 3023 decoder: Optional[ 3024 Union[ 3025 CsvDecoder, 3026 GzipDecoder, 3027 JsonDecoder, 3028 JsonlDecoder, 3029 IterableDecoder, 3030 XmlDecoder, 3031 ZipfileDecoder, 3032 CustomDecoder, 3033 ] 3034 ] = Field( 3035 None, 3036 description="Component decoding the response so records can be extracted.", 3037 title="HTTP Response Format", 3038 ) 3039 download_decoder: Optional[ 3040 Union[ 3041 CsvDecoder, 3042 GzipDecoder, 3043 JsonDecoder, 3044 JsonlDecoder, 3045 IterableDecoder, 3046 XmlDecoder, 3047 ZipfileDecoder, 3048 CustomDecoder, 3049 ] 3050 ] = Field( 3051 None, 3052 description="Component decoding the download response so records can be extracted.", 3053 title="Download HTTP Response Format", 3054 ) 3055 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3056 3057 3058class SubstreamPartitionRouter(BaseModel): 3059 type: Literal["SubstreamPartitionRouter"] 3060 parent_stream_configs: List[ParentStreamConfig] = Field( 3061 ..., 3062 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3063 title="Parent Stream Configs", 3064 ) 3065 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3066 3067 3068class GroupingPartitionRouter(BaseModel): 3069 type: Literal["GroupingPartitionRouter"] 3070 group_size: int = Field( 3071 ..., 3072 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3073 examples=[10, 50], 3074 title="Group Size", 3075 ) 3076 underlying_partition_router: Union[ 3077 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3078 ] = Field( 3079 ..., 3080 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3081 title="Underlying Partition Router", 3082 ) 3083 deduplicate: Optional[bool] = Field( 3084 True, 3085 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3086 title="Deduplicate Partitions", 3087 ) 3088 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3089 3090 3091class HttpComponentsResolver(BaseModel): 3092 type: Literal["HttpComponentsResolver"] 3093 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3094 ..., 3095 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3096 title="Retriever", 3097 ) 3098 components_mapping: List[ComponentMappingDefinition] 3099 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3100 3101 3102class DynamicDeclarativeStream(BaseModel): 3103 type: Literal["DynamicDeclarativeStream"] 3104 name: Optional[str] = Field( 3105 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3106 ) 3107 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3108 ..., description="Reference to the stream template.", title="Stream Template" 3109 ) 3110 components_resolver: Union[ 3111 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3112 ] = Field( 3113 ..., 3114 description="Component resolve and populates stream templates with components values.", 3115 title="Components Resolver", 3116 ) 3117 use_parent_parameters: Optional[bool] = Field( 3118 True, 3119 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3120 title="Use Parent Parameters", 3121 ) 3122 3123 3124ComplexFieldType.update_forward_refs() 3125GzipDecoder.update_forward_refs() 3126CompositeErrorHandler.update_forward_refs() 3127DeclarativeSource1.update_forward_refs() 3128DeclarativeSource2.update_forward_refs() 3129SelectiveAuthenticator.update_forward_refs() 3130ConditionalStreams.update_forward_refs() 3131FileUploader.update_forward_refs() 3132DeclarativeStream.update_forward_refs() 3133SessionTokenAuthenticator.update_forward_refs() 3134HttpRequester.update_forward_refs() 3135DynamicSchemaLoader.update_forward_refs() 3136ParentStreamConfig.update_forward_refs() 3137PropertiesFromEndpoint.update_forward_refs() 3138SimpleRetriever.update_forward_refs() 3139AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[Union[int, str]] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100, "{{ config['page_size'] }}"], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
178class CustomPaginationStrategy(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomPaginationStrategy"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomPaginationStrategy"], 187 title="Class Name", 188 ) 189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
192class CustomRecordExtractor(BaseModel): 193 class Config: 194 extra = Extra.allow 195 196 type: Literal["CustomRecordExtractor"] 197 class_name: str = Field( 198 ..., 199 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 200 examples=["source_railz.components.MyCustomRecordExtractor"], 201 title="Class Name", 202 ) 203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
206class CustomRecordFilter(BaseModel): 207 class Config: 208 extra = Extra.allow 209 210 type: Literal["CustomRecordFilter"] 211 class_name: str = Field( 212 ..., 213 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 214 examples=["source_railz.components.MyCustomCustomRecordFilter"], 215 title="Class Name", 216 ) 217 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
220class CustomRequester(BaseModel): 221 class Config: 222 extra = Extra.allow 223 224 type: Literal["CustomRequester"] 225 class_name: str = Field( 226 ..., 227 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 228 examples=["source_railz.components.MyCustomRecordExtractor"], 229 title="Class Name", 230 ) 231 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
234class CustomRetriever(BaseModel): 235 class Config: 236 extra = Extra.allow 237 238 type: Literal["CustomRetriever"] 239 class_name: str = Field( 240 ..., 241 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 242 examples=["source_railz.components.MyCustomRetriever"], 243 title="Class Name", 244 ) 245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
248class CustomPartitionRouter(BaseModel): 249 class Config: 250 extra = Extra.allow 251 252 type: Literal["CustomPartitionRouter"] 253 class_name: str = Field( 254 ..., 255 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 256 examples=["source_railz.components.MyCustomPartitionRouter"], 257 title="Class Name", 258 ) 259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
262class CustomSchemaLoader(BaseModel): 263 class Config: 264 extra = Extra.allow 265 266 type: Literal["CustomSchemaLoader"] 267 class_name: str = Field( 268 ..., 269 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 270 examples=["source_railz.components.MyCustomSchemaLoader"], 271 title="Class Name", 272 ) 273 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
276class CustomSchemaNormalization(BaseModel): 277 class Config: 278 extra = Extra.allow 279 280 type: Literal["CustomSchemaNormalization"] 281 class_name: str = Field( 282 ..., 283 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 284 examples=[ 285 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 286 ], 287 title="Class Name", 288 ) 289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
292class CustomStateMigration(BaseModel): 293 class Config: 294 extra = Extra.allow 295 296 type: Literal["CustomStateMigration"] 297 class_name: str = Field( 298 ..., 299 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 300 examples=["source_railz.components.MyCustomStateMigration"], 301 title="Class Name", 302 ) 303 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
306class CustomTransformation(BaseModel): 307 class Config: 308 extra = Extra.allow 309 310 type: Literal["CustomTransformation"] 311 class_name: str = Field( 312 ..., 313 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 314 examples=["source_railz.components.MyCustomTransformation"], 315 title="Class Name", 316 ) 317 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
337class Algorithm(Enum): 338 HS256 = "HS256" 339 HS384 = "HS384" 340 HS512 = "HS512" 341 ES256 = "ES256" 342 ES256K = "ES256K" 343 ES384 = "ES384" 344 ES512 = "ES512" 345 RS256 = "RS256" 346 RS384 = "RS384" 347 RS512 = "RS512" 348 PS256 = "PS256" 349 PS384 = "PS384" 350 PS512 = "PS512" 351 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
354class JwtHeaders(BaseModel): 355 class Config: 356 extra = Extra.forbid 357 358 kid: Optional[str] = Field( 359 None, 360 description="Private key ID for user account.", 361 examples=["{{ config['kid'] }}"], 362 title="Key Identifier", 363 ) 364 typ: Optional[str] = Field( 365 "JWT", 366 description="The media type of the complete JWT.", 367 examples=["JWT"], 368 title="Type", 369 ) 370 cty: Optional[str] = Field( 371 None, 372 description="Content type of JWT header.", 373 examples=["JWT"], 374 title="Content Type", 375 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
378class JwtPayload(BaseModel): 379 class Config: 380 extra = Extra.forbid 381 382 iss: Optional[str] = Field( 383 None, 384 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 385 examples=["{{ config['iss'] }}"], 386 title="Issuer", 387 ) 388 sub: Optional[str] = Field( 389 None, 390 description="The subject of the JWT. Commonly defined by the API.", 391 title="Subject", 392 ) 393 aud: Optional[str] = Field( 394 None, 395 description="The recipient that the JWT is intended for. Commonly defined by the API.", 396 examples=["appstoreconnect-v1"], 397 title="Audience", 398 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
401class RefreshTokenUpdater(BaseModel): 402 refresh_token_name: Optional[str] = Field( 403 "refresh_token", 404 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 405 examples=["refresh_token"], 406 title="Refresh Token Property Name", 407 ) 408 access_token_config_path: Optional[List[str]] = Field( 409 ["credentials", "access_token"], 410 description="Config path to the access token. Make sure the field actually exists in the config.", 411 examples=[["credentials", "access_token"], ["access_token"]], 412 title="Config Path To Access Token", 413 ) 414 refresh_token_config_path: Optional[List[str]] = Field( 415 ["credentials", "refresh_token"], 416 description="Config path to the access token. Make sure the field actually exists in the config.", 417 examples=[["credentials", "refresh_token"], ["refresh_token"]], 418 title="Config Path To Refresh Token", 419 ) 420 token_expiry_date_config_path: Optional[List[str]] = Field( 421 ["credentials", "token_expiry_date"], 422 description="Config path to the expiry date. Make sure actually exists in the config.", 423 examples=[["credentials", "token_expiry_date"]], 424 title="Config Path To Expiry Date", 425 ) 426 refresh_token_error_status_codes: Optional[List[int]] = Field( 427 [], 428 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 429 examples=[[400, 500]], 430 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 431 ) 432 refresh_token_error_key: Optional[str] = Field( 433 "", 434 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 435 examples=["error"], 436 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 437 ) 438 refresh_token_error_values: Optional[List[str]] = Field( 439 [], 440 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 441 examples=[["invalid_grant", "invalid_permissions"]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 443 )
class
Rate(pydantic.v1.main.BaseModel):
446class Rate(BaseModel): 447 class Config: 448 extra = Extra.allow 449 450 limit: Union[int, str] = Field( 451 ..., 452 description="The maximum number of calls allowed within the interval.", 453 title="Limit", 454 ) 455 interval: str = Field( 456 ..., 457 description="The time interval for the rate limit.", 458 examples=["PT1H", "P1D"], 459 title="Interval", 460 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
463class HttpRequestRegexMatcher(BaseModel): 464 class Config: 465 extra = Extra.allow 466 467 method: Optional[str] = Field( 468 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 469 ) 470 url_base: Optional[str] = Field( 471 None, 472 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 473 title="URL Base", 474 ) 475 url_path_pattern: Optional[str] = Field( 476 None, 477 description="A regular expression pattern to match the URL path.", 478 title="URL Path Pattern", 479 ) 480 params: Optional[Dict[str, Any]] = Field( 481 None, description="The query parameters to match.", title="Parameters" 482 ) 483 headers: Optional[Dict[str, Any]] = Field( 484 None, description="The headers to match.", title="Headers" 485 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
488class DpathExtractor(BaseModel): 489 type: Literal["DpathExtractor"] 490 field_path: List[str] = Field( 491 ..., 492 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 493 examples=[ 494 ["data"], 495 ["data", "records"], 496 ["data", "{{ parameters.name }}"], 497 ["data", "*", "record"], 498 ], 499 title="Field Path", 500 ) 501 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
509class ExponentialBackoffStrategy(BaseModel): 510 type: Literal["ExponentialBackoffStrategy"] 511 factor: Optional[Union[float, str]] = Field( 512 5, 513 description="Multiplicative constant applied on each retry.", 514 examples=[5, 5.5, "10"], 515 title="Factor", 516 ) 517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
520class GroupByKeyMergeStrategy(BaseModel): 521 type: Literal["GroupByKeyMergeStrategy"] 522 key: Union[str, List[str]] = Field( 523 ..., 524 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 525 examples=["id", ["parent_id", "end_date"]], 526 title="Key", 527 ) 528 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
540class Action(Enum): 541 SUCCESS = "SUCCESS" 542 FAIL = "FAIL" 543 RETRY = "RETRY" 544 IGNORE = "IGNORE" 545 RESET_PAGINATION = "RESET_PAGINATION" 546 RATE_LIMITED = "RATE_LIMITED" 547 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
550class FailureType(Enum): 551 system_error = "system_error" 552 config_error = "config_error" 553 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
556class HttpResponseFilter(BaseModel): 557 type: Literal["HttpResponseFilter"] 558 action: Optional[Action] = Field( 559 None, 560 description="Action to execute if a response matches the filter.", 561 examples=[ 562 "SUCCESS", 563 "FAIL", 564 "RETRY", 565 "IGNORE", 566 "RESET_PAGINATION", 567 "RATE_LIMITED", 568 "REFRESH_TOKEN_THEN_RETRY", 569 ], 570 title="Action", 571 ) 572 failure_type: Optional[FailureType] = Field( 573 None, 574 description="Failure type of traced exception if a response matches the filter.", 575 examples=["system_error", "config_error", "transient_error"], 576 title="Failure Type", 577 ) 578 error_message: Optional[str] = Field( 579 None, 580 description="Error Message to display if the response matches the filter.", 581 title="Error Message", 582 ) 583 error_message_contains: Optional[str] = Field( 584 None, 585 description="Match the response if its error message contains the substring.", 586 example=["This API operation is not enabled for this site"], 587 title="Error Message Substring", 588 ) 589 http_codes: Optional[List[int]] = Field( 590 None, 591 description="Match the response if its HTTP code is included in this list.", 592 examples=[[420, 429], [500]], 593 title="HTTP Codes", 594 unique_items=True, 595 ) 596 predicate: Optional[str] = Field( 597 None, 598 description="Match the response if the predicate evaluates to true.", 599 examples=[ 600 "{{ 'Too much requests' in response }}", 601 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 602 ], 603 title="Predicate", 604 ) 605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
608class ComplexFieldType(BaseModel): 609 field_type: str 610 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
613class TypesMap(BaseModel): 614 target_type: Union[str, List[str], ComplexFieldType] 615 current_type: Union[str, List[str]] 616 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
619class SchemaTypeIdentifier(BaseModel): 620 type: Optional[Literal["SchemaTypeIdentifier"]] = None 621 schema_pointer: Optional[List[str]] = Field( 622 [], 623 description="List of nested fields defining the schema field path to extract. Defaults to [].", 624 title="Schema Path", 625 ) 626 key_pointer: List[str] = Field( 627 ..., 628 description="List of potentially nested fields describing the full path of the field key to extract.", 629 title="Key Path", 630 ) 631 type_pointer: Optional[List[str]] = Field( 632 None, 633 description="List of potentially nested fields describing the full path of the field type to extract.", 634 title="Type Path", 635 ) 636 types_mapping: Optional[List[TypesMap]] = None 637 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
640class InlineSchemaLoader(BaseModel): 641 type: Literal["InlineSchemaLoader"] 642 schema_: Optional[Dict[str, Any]] = Field( 643 None, 644 alias="schema", 645 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 646 title="Schema", 647 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
650class JsonFileSchemaLoader(BaseModel): 651 type: Literal["JsonFileSchemaLoader"] 652 file_path: Optional[str] = Field( 653 None, 654 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 655 example=["./schemas/users.json"], 656 title="File Path", 657 ) 658 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
679class FlattenFields(BaseModel): 680 type: Literal["FlattenFields"] 681 flatten_lists: Optional[bool] = Field( 682 True, 683 description="Whether to flatten lists or leave it as is. Default is True.", 684 title="Flatten Lists", 685 ) 686 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
689class KeyTransformation(BaseModel): 690 type: Literal["KeyTransformation"] 691 prefix: Optional[str] = Field( 692 None, 693 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 694 examples=["flattened_"], 695 title="Key Prefix", 696 ) 697 suffix: Optional[str] = Field( 698 None, 699 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 700 examples=["_flattened"], 701 title="Key Suffix", 702 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
705class DpathFlattenFields(BaseModel): 706 type: Literal["DpathFlattenFields"] 707 field_path: List[str] = Field( 708 ..., 709 description="A path to field that needs to be flattened.", 710 examples=[["data"], ["data", "*", "field"]], 711 title="Field Path", 712 ) 713 delete_origin_value: Optional[bool] = Field( 714 None, 715 description="Whether to delete the origin value or keep it. Default is False.", 716 title="Delete Origin Value", 717 ) 718 replace_record: Optional[bool] = Field( 719 None, 720 description="Whether to replace the origin record or not. Default is False.", 721 title="Replace Origin Record", 722 ) 723 key_transformation: Optional[KeyTransformation] = Field( 724 None, 725 description="Transformation for object keys. If not provided, original key will be used.", 726 title="Key transformation", 727 ) 728 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
731class KeysReplace(BaseModel): 732 type: Literal["KeysReplace"] 733 old: str = Field( 734 ..., 735 description="Old value to replace.", 736 examples=[ 737 " ", 738 "{{ record.id }}", 739 "{{ config['id'] }}", 740 "{{ stream_slice['id'] }}", 741 ], 742 title="Old value", 743 ) 744 new: str = Field( 745 ..., 746 description="New value to set.", 747 examples=[ 748 "_", 749 "{{ record.id }}", 750 "{{ config['id'] }}", 751 "{{ stream_slice['id'] }}", 752 ], 753 title="New value", 754 ) 755 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
766class CustomDecoder(BaseModel): 767 class Config: 768 extra = Extra.allow 769 770 type: Literal["CustomDecoder"] 771 class_name: str = Field( 772 ..., 773 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 774 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 775 title="Class Name", 776 ) 777 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
780class MinMaxDatetime(BaseModel): 781 type: Literal["MinMaxDatetime"] 782 datetime: str = Field( 783 ..., 784 description="Datetime value.", 785 examples=[ 786 "2021-01-01", 787 "2021-01-01T00:00:00Z", 788 "{{ config['start_time'] }}", 789 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 790 ], 791 title="Datetime", 792 ) 793 datetime_format: Optional[str] = Field( 794 "", 795 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 796 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 797 title="Datetime Format", 798 ) 799 max_datetime: Optional[str] = Field( 800 None, 801 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 802 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 803 title="Max Datetime", 804 ) 805 min_datetime: Optional[str] = Field( 806 None, 807 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 808 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 809 title="Min Datetime", 810 ) 811 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
831class OauthConnectorInputSpecification(BaseModel): 832 class Config: 833 extra = Extra.allow 834 835 consent_url: str = Field( 836 ..., 837 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 838 examples=[ 839 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 840 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 841 ], 842 title="Consent URL", 843 ) 844 scope: Optional[str] = Field( 845 None, 846 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 847 examples=["user:read user:read_orders workspaces:read"], 848 title="Scopes", 849 ) 850 access_token_url: str = Field( 851 ..., 852 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 853 examples=[ 854 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 855 ], 856 title="Access Token URL", 857 ) 858 access_token_headers: Optional[Dict[str, Any]] = Field( 859 None, 860 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 861 examples=[ 862 { 863 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 864 } 865 ], 866 title="Access Token Headers", 867 ) 868 access_token_params: Optional[Dict[str, Any]] = Field( 869 None, 870 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 871 examples=[ 872 { 873 "{{ auth_code_key }}": "{{ auth_code_value }}", 874 "{{ client_id_key }}": "{{ client_id_value }}", 875 "{{ client_secret_key }}": "{{ client_secret_value }}", 876 } 877 ], 878 title="Access Token Query Params (Json Encoded)", 879 ) 880 extract_output: Optional[List[str]] = Field( 881 None, 882 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 883 examples=[["access_token", "refresh_token", "other_field"]], 884 title="Extract Output", 885 ) 886 state: Optional[State] = Field( 887 None, 888 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 889 examples=[{"min": 7, "max": 128}], 890 title="Configurable State Query Param", 891 ) 892 client_id_key: Optional[str] = Field( 893 None, 894 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 895 examples=["my_custom_client_id_key_name"], 896 title="Client ID Key Override", 897 ) 898 client_secret_key: Optional[str] = Field( 899 None, 900 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 901 examples=["my_custom_client_secret_key_name"], 902 title="Client Secret Key Override", 903 ) 904 scope_key: Optional[str] = Field( 905 None, 906 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 907 examples=["my_custom_scope_key_key_name"], 908 title="Scopes Key Override", 909 ) 910 state_key: Optional[str] = Field( 911 None, 912 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 913 examples=["my_custom_state_key_key_name"], 914 title="State Key Override", 915 ) 916 auth_code_key: Optional[str] = Field( 917 None, 918 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 919 examples=["my_custom_auth_code_key_name"], 920 title="Auth Code Key Override", 921 ) 922 redirect_uri_key: Optional[str] = Field( 923 None, 924 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 925 examples=["my_custom_redirect_uri_key_name"], 926 title="Redirect URI Key Override", 927 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
930class OAuthConfigSpecification(BaseModel): 931 class Config: 932 extra = Extra.allow 933 934 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 935 None, 936 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 937 examples=[ 938 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 939 { 940 "app_id": { 941 "type": "string", 942 "path_in_connector_config": ["info", "app_id"], 943 } 944 }, 945 ], 946 title="OAuth user input", 947 ) 948 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 949 None, 950 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 951 title="DeclarativeOAuth Connector Specification", 952 ) 953 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 954 None, 955 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 956 examples=[ 957 { 958 "refresh_token": { 959 "type": "string,", 960 "path_in_connector_config": ["credentials", "refresh_token"], 961 } 962 } 963 ], 964 title="OAuth output specification", 965 ) 966 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 967 None, 968 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 969 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 970 title="OAuth input specification", 971 ) 972 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 973 None, 974 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 975 examples=[ 976 { 977 "client_id": { 978 "type": "string,", 979 "path_in_connector_config": ["credentials", "client_id"], 980 }, 981 "client_secret": { 982 "type": "string,", 983 "path_in_connector_config": ["credentials", "client_secret"], 984 }, 985 } 986 ], 987 title="OAuth server output specification", 988 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
991class OffsetIncrement(BaseModel): 992 type: Literal["OffsetIncrement"] 993 page_size: Optional[Union[int, str]] = Field( 994 None, 995 description="The number of records to include in each pages.", 996 examples=[100, "{{ config['page_size'] }}"], 997 title="Limit", 998 ) 999 inject_on_first_request: Optional[bool] = Field( 1000 False, 1001 description="Using the `offset` with value `0` during the first request", 1002 title="Inject Offset on First Request", 1003 ) 1004 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1007class PageIncrement(BaseModel): 1008 type: Literal["PageIncrement"] 1009 page_size: Optional[Union[int, str]] = Field( 1010 None, 1011 description="The number of records to include in each pages.", 1012 examples=[100, "100", "{{ config['page_size'] }}"], 1013 title="Page Size", 1014 ) 1015 start_from_page: Optional[int] = Field( 1016 0, 1017 description="Index of the first page to request.", 1018 examples=[0, 1], 1019 title="Start From Page", 1020 ) 1021 inject_on_first_request: Optional[bool] = Field( 1022 False, 1023 description="Using the `page number` with value defined by `start_from_page` during the first request", 1024 title="Inject Page Number on First Request", 1025 ) 1026 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1029class PrimaryKey(BaseModel): 1030 __root__: Union[str, List[str], List[List[str]]] = Field( 1031 ..., 1032 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1033 examples=["id", ["code", "type"]], 1034 title="Primary Key", 1035 )
class
PropertyLimitType(enum.Enum):
1038class PropertyLimitType(Enum): 1039 characters = "characters" 1040 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1043class PropertyChunking(BaseModel): 1044 type: Literal["PropertyChunking"] 1045 property_limit_type: PropertyLimitType = Field( 1046 ..., 1047 description="The type used to determine the maximum number of properties per chunk", 1048 title="Property Limit Type", 1049 ) 1050 property_limit: Optional[int] = Field( 1051 None, 1052 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1053 title="Property Limit", 1054 ) 1055 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1056 None, 1057 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1058 title="Record Merge Strategy", 1059 ) 1060 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1063class RecordFilter(BaseModel): 1064 type: Literal["RecordFilter"] 1065 condition: Optional[str] = Field( 1066 "", 1067 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1068 examples=[ 1069 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1070 "{{ record.status in ['active', 'expired'] }}", 1071 ], 1072 title="Condition", 1073 ) 1074 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1082class RemoveFields(BaseModel): 1083 type: Literal["RemoveFields"] 1084 condition: Optional[str] = Field( 1085 "", 1086 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1087 examples=[ 1088 "{{ property|string == '' }}", 1089 "{{ property is integer }}", 1090 "{{ property|length > 5 }}", 1091 "{{ property == 'some_string_to_match' }}", 1092 ], 1093 ) 1094 field_pointers: List[List[str]] = Field( 1095 ..., 1096 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1097 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1098 title="Field Paths", 1099 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1106class InjectInto(Enum): 1107 request_parameter = "request_parameter" 1108 header = "header" 1109 body_data = "body_data" 1110 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1113class RequestOption(BaseModel): 1114 type: Literal["RequestOption"] 1115 inject_into: InjectInto = Field( 1116 ..., 1117 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1118 examples=["request_parameter", "header", "body_data", "body_json"], 1119 title="Inject Into", 1120 ) 1121 field_name: Optional[str] = Field( 1122 None, 1123 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1124 examples=["segment_id"], 1125 title="Field Name", 1126 ) 1127 field_path: Optional[List[str]] = Field( 1128 None, 1129 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1130 examples=[["data", "viewer", "id"]], 1131 title="Field Path", 1132 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1142class LegacySessionTokenAuthenticator(BaseModel): 1143 type: Literal["LegacySessionTokenAuthenticator"] 1144 header: str = Field( 1145 ..., 1146 description="The name of the session token header that will be injected in the request", 1147 examples=["X-Session"], 1148 title="Session Request Header", 1149 ) 1150 login_url: str = Field( 1151 ..., 1152 description="Path of the login URL (do not include the base URL)", 1153 examples=["session"], 1154 title="Login Path", 1155 ) 1156 session_token: Optional[str] = Field( 1157 None, 1158 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1159 example=["{{ config['session_token'] }}"], 1160 title="Session Token", 1161 ) 1162 session_token_response_key: str = Field( 1163 ..., 1164 description="Name of the key of the session token to be extracted from the response", 1165 examples=["id"], 1166 title="Response Token Response Key", 1167 ) 1168 username: Optional[str] = Field( 1169 None, 1170 description="Username used to authenticate and obtain a session token", 1171 examples=[" {{ config['username'] }}"], 1172 title="Username", 1173 ) 1174 password: Optional[str] = Field( 1175 "", 1176 description="Password used to authenticate and obtain a session token", 1177 examples=["{{ config['password'] }}", ""], 1178 title="Password", 1179 ) 1180 validate_session_url: str = Field( 1181 ..., 1182 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1183 examples=["user/current"], 1184 title="Validate Session Path", 1185 ) 1186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1214class ValueType(Enum): 1215 string = "string" 1216 number = "number" 1217 integer = "integer" 1218 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1221class WaitTimeFromHeader(BaseModel): 1222 type: Literal["WaitTimeFromHeader"] 1223 header: str = Field( 1224 ..., 1225 description="The name of the response header defining how long to wait before retrying.", 1226 examples=["Retry-After"], 1227 title="Response Header Name", 1228 ) 1229 regex: Optional[str] = Field( 1230 None, 1231 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1232 examples=["([-+]?\\d+)"], 1233 title="Extraction Regex", 1234 ) 1235 max_waiting_time_in_seconds: Optional[float] = Field( 1236 None, 1237 description="Given the value extracted from the header is greater than this value, stop the stream.", 1238 examples=[3600], 1239 title="Max Waiting Time in Seconds", 1240 ) 1241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1244class WaitUntilTimeFromHeader(BaseModel): 1245 type: Literal["WaitUntilTimeFromHeader"] 1246 header: str = Field( 1247 ..., 1248 description="The name of the response header defining how long to wait before retrying.", 1249 examples=["wait_time"], 1250 title="Response Header", 1251 ) 1252 min_wait: Optional[Union[float, str]] = Field( 1253 None, 1254 description="Minimum time to wait before retrying.", 1255 examples=[10, "60"], 1256 title="Minimum Wait Time", 1257 ) 1258 regex: Optional[str] = Field( 1259 None, 1260 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1261 examples=["([-+]?\\d+)"], 1262 title="Extraction Regex", 1263 ) 1264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1267class ComponentMappingDefinition(BaseModel): 1268 type: Literal["ComponentMappingDefinition"] 1269 field_path: List[str] = Field( 1270 ..., 1271 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1272 examples=[ 1273 ["name"], 1274 ["retriever", "requester", "url"], 1275 ["retriever", "requester", "{{ components_values.field }}"], 1276 ["*", "**", "name"], 1277 ], 1278 title="Field Path", 1279 ) 1280 value: str = Field( 1281 ..., 1282 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1283 examples=[ 1284 "{{ components_values['updates'] }}", 1285 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1286 "{{ config['segment_id'] }}", 1287 "{{ stream_slice['parent_id'] }}", 1288 "{{ stream_slice['extra_fields']['name'] }}", 1289 ], 1290 title="Value", 1291 ) 1292 value_type: Optional[ValueType] = Field( 1293 None, 1294 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1295 title="Value Type", 1296 ) 1297 create_or_update: Optional[bool] = Field( 1298 False, 1299 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1300 title="Create or Update", 1301 ) 1302 condition: Optional[str] = Field( 1303 None, 1304 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1305 examples=[ 1306 "{{ components_values.get('cursor_field', None) }}", 1307 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1308 ], 1309 title="Condition", 1310 ) 1311 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1314class StreamConfig(BaseModel): 1315 type: Literal["StreamConfig"] 1316 configs_pointer: List[str] = Field( 1317 ..., 1318 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1319 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1320 title="Configs Pointer", 1321 ) 1322 default_values: Optional[List[Dict[str, Any]]] = Field( 1323 None, 1324 description="A list of default values, each matching the structure expected from the parsed component value.", 1325 title="Default Values", 1326 ) 1327 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1330class ConfigComponentsResolver(BaseModel): 1331 type: Literal["ConfigComponentsResolver"] 1332 stream_config: Union[List[StreamConfig], StreamConfig] 1333 components_mapping: List[ComponentMappingDefinition] 1334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1337class StreamParametersDefinition(BaseModel): 1338 type: Literal["StreamParametersDefinition"] 1339 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1340 ..., 1341 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1342 examples=[ 1343 [ 1344 { 1345 "name": "test stream", 1346 "$parameters": {"entity": "test entity"}, 1347 "primary_key": "test key", 1348 } 1349 ] 1350 ], 1351 title="Stream Parameters", 1352 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1355class ParametrizedComponentsResolver(BaseModel): 1356 type: Literal["ParametrizedComponentsResolver"] 1357 stream_parameters: StreamParametersDefinition 1358 components_mapping: List[ComponentMappingDefinition] 1359 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1384class ValidateAdheresToSchema(BaseModel): 1385 type: Literal["ValidateAdheresToSchema"] 1386 base_schema: Union[str, Dict[str, Any]] = Field( 1387 ..., 1388 description="The base JSON schema against which the user-provided schema will be validated.", 1389 examples=[ 1390 "{{ config['report_validation_schema'] }}", 1391 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1392 { 1393 "$schema": "http://json-schema.org/draft-07/schema#", 1394 "title": "Person", 1395 "type": "object", 1396 "properties": { 1397 "name": {"type": "string", "description": "The person's name"}, 1398 "age": { 1399 "type": "integer", 1400 "minimum": 0, 1401 "description": "The person's age", 1402 }, 1403 }, 1404 "required": ["name", "age"], 1405 }, 1406 ], 1407 title="Base JSON Schema", 1408 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1411class CustomValidationStrategy(BaseModel): 1412 class Config: 1413 extra = Extra.allow 1414 1415 type: Literal["CustomValidationStrategy"] 1416 class_name: str = Field( 1417 ..., 1418 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1419 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1420 title="Class Name", 1421 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1424class ConfigRemapField(BaseModel): 1425 type: Literal["ConfigRemapField"] 1426 map: Union[Dict[str, Any], str] = Field( 1427 ..., 1428 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1429 examples=[ 1430 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1431 "{{ config['status_mapping'] }}", 1432 ], 1433 title="Value Mapping", 1434 ) 1435 field_path: List[str] = Field( 1436 ..., 1437 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1438 examples=[ 1439 ["status"], 1440 ["data", "status"], 1441 ["data", "{{ config.name }}", "status"], 1442 ["data", "*", "status"], 1443 ], 1444 title="Field Path", 1445 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1448class ConfigRemoveFields(BaseModel): 1449 type: Literal["ConfigRemoveFields"] 1450 field_pointers: List[List[str]] = Field( 1451 ..., 1452 description="A list of field pointers to be removed from the config.", 1453 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1454 title="Field Pointers", 1455 ) 1456 condition: Optional[str] = Field( 1457 "", 1458 description="Fields will be removed if expression is evaluated to True.", 1459 examples=[ 1460 "{{ config['environemnt'] == 'sandbox' }}", 1461 "{{ property is integer }}", 1462 "{{ property|length > 5 }}", 1463 "{{ property == 'some_string_to_match' }}", 1464 ], 1465 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1468class CustomConfigTransformation(BaseModel): 1469 type: Literal["CustomConfigTransformation"] 1470 class_name: str = Field( 1471 ..., 1472 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1473 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1474 ) 1475 parameters: Optional[Dict[str, Any]] = Field( 1476 None, 1477 alias="$parameters", 1478 description="Additional parameters to be passed to the custom config transformation.", 1479 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1482class AddedFieldDefinition(BaseModel): 1483 type: Literal["AddedFieldDefinition"] 1484 path: List[str] = Field( 1485 ..., 1486 description="List of strings defining the path where to add the value on the record.", 1487 examples=[["segment_id"], ["metadata", "segment_id"]], 1488 title="Path", 1489 ) 1490 value: str = Field( 1491 ..., 1492 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1493 examples=[ 1494 "{{ record['updates'] }}", 1495 "{{ record['MetaData']['LastUpdatedTime'] }}", 1496 "{{ stream_partition['segment_id'] }}", 1497 ], 1498 title="Value", 1499 ) 1500 value_type: Optional[ValueType] = Field( 1501 None, 1502 description="Type of the value. If not specified, the type will be inferred from the value.", 1503 title="Value Type", 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1508class AddFields(BaseModel): 1509 type: Literal["AddFields"] 1510 fields: List[AddedFieldDefinition] = Field( 1511 ..., 1512 description="List of transformations (path and corresponding value) that will be added to the record.", 1513 title="Fields", 1514 ) 1515 condition: Optional[str] = Field( 1516 "", 1517 description="Fields will be added if expression is evaluated to True.", 1518 examples=[ 1519 "{{ property|string == '' }}", 1520 "{{ property is integer }}", 1521 "{{ property|length > 5 }}", 1522 "{{ property == 'some_string_to_match' }}", 1523 ], 1524 ) 1525 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1528class ApiKeyAuthenticator(BaseModel): 1529 type: Literal["ApiKeyAuthenticator"] 1530 api_token: Optional[str] = Field( 1531 None, 1532 description="The API key to inject in the request. Fill it in the user inputs.", 1533 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1534 title="API Key", 1535 ) 1536 header: Optional[str] = Field( 1537 None, 1538 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1539 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1540 title="Header Name", 1541 ) 1542 inject_into: Optional[RequestOption] = Field( 1543 None, 1544 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1545 examples=[ 1546 {"inject_into": "header", "field_name": "Authorization"}, 1547 {"inject_into": "request_parameter", "field_name": "authKey"}, 1548 ], 1549 title="Inject API Key Into Outgoing HTTP Request", 1550 ) 1551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1554class AuthFlow(BaseModel): 1555 auth_flow_type: Optional[AuthFlowType] = Field( 1556 None, description="The type of auth to use", title="Auth flow type" 1557 ) 1558 predicate_key: Optional[List[str]] = Field( 1559 None, 1560 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1561 examples=[["credentials", "auth_type"]], 1562 title="Predicate key", 1563 ) 1564 predicate_value: Optional[str] = Field( 1565 None, 1566 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1567 examples=["Oauth"], 1568 title="Predicate value", 1569 ) 1570 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1573class CheckStream(BaseModel): 1574 type: Literal["CheckStream"] 1575 stream_names: Optional[List[str]] = Field( 1576 None, 1577 description="Names of the streams to try reading from when running a check operation.", 1578 examples=[["users"], ["users", "contacts"]], 1579 title="Stream Names", 1580 ) 1581 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1584class IncrementingCountCursor(BaseModel): 1585 type: Literal["IncrementingCountCursor"] 1586 cursor_field: str = Field( 1587 ..., 1588 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1589 examples=["created_at", "{{ config['record_cursor'] }}"], 1590 title="Cursor Field", 1591 ) 1592 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1593 None, 1594 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1595 title="Allow Catalog Defined Cursor Field", 1596 ) 1597 start_value: Optional[Union[str, int]] = Field( 1598 None, 1599 description="The value that determines the earliest record that should be synced.", 1600 examples=[0, "{{ config['start_value'] }}"], 1601 title="Start Value", 1602 ) 1603 start_value_option: Optional[RequestOption] = Field( 1604 None, 1605 description="Optionally configures how the start value will be sent in requests to the source API.", 1606 title="Inject Start Value Into Outgoing HTTP Request", 1607 ) 1608 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1611class DatetimeBasedCursor(BaseModel): 1612 type: Literal["DatetimeBasedCursor"] 1613 clamping: Optional[Clamping] = Field( 1614 None, 1615 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1616 title="Date Range Clamping", 1617 ) 1618 cursor_field: str = Field( 1619 ..., 1620 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1621 examples=["created_at", "{{ config['record_cursor'] }}"], 1622 title="Cursor Field", 1623 ) 1624 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1625 None, 1626 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1627 title="Allow Catalog Defined Cursor Field", 1628 ) 1629 cursor_datetime_formats: Optional[List[str]] = Field( 1630 None, 1631 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1632 examples=[ 1633 "%Y-%m-%d", 1634 "%Y-%m-%d %H:%M:%S", 1635 "%Y-%m-%dT%H:%M:%S", 1636 "%Y-%m-%dT%H:%M:%SZ", 1637 "%Y-%m-%dT%H:%M:%S%z", 1638 "%Y-%m-%dT%H:%M:%S.%fZ", 1639 "%Y-%m-%dT%H:%M:%S.%f%z", 1640 "%Y-%m-%d %H:%M:%S.%f+00:00", 1641 "%s", 1642 "%ms", 1643 ], 1644 title="Cursor Datetime Formats", 1645 ) 1646 start_datetime: Union[MinMaxDatetime, str] = Field( 1647 ..., 1648 description="The datetime that determines the earliest record that should be synced.", 1649 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1650 title="Start Datetime", 1651 ) 1652 start_time_option: Optional[RequestOption] = Field( 1653 None, 1654 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1655 title="Inject Start Time Into Outgoing HTTP Request", 1656 ) 1657 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1658 None, 1659 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1660 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1661 title="End Datetime", 1662 ) 1663 end_time_option: Optional[RequestOption] = Field( 1664 None, 1665 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1666 title="Inject End Time Into Outgoing HTTP Request", 1667 ) 1668 datetime_format: str = Field( 1669 ..., 1670 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1671 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1672 title="Outgoing Datetime Format", 1673 ) 1674 cursor_granularity: Optional[str] = Field( 1675 None, 1676 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1677 examples=["PT1S"], 1678 title="Cursor Granularity", 1679 ) 1680 is_data_feed: Optional[bool] = Field( 1681 None, 1682 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1683 title="Data Feed API", 1684 ) 1685 is_client_side_incremental: Optional[bool] = Field( 1686 None, 1687 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1688 title="Client-side Incremental Filtering", 1689 ) 1690 is_compare_strictly: Optional[bool] = Field( 1691 False, 1692 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1693 title="Strict Start-End Time Comparison", 1694 ) 1695 global_substream_cursor: Optional[bool] = Field( 1696 False, 1697 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1698 title="Global Substream Cursor", 1699 ) 1700 lookback_window: Optional[str] = Field( 1701 None, 1702 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1703 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1704 title="Lookback Window", 1705 ) 1706 partition_field_end: Optional[str] = Field( 1707 None, 1708 description="Name of the partition start time field.", 1709 examples=["ending_time"], 1710 title="Partition Field End", 1711 ) 1712 partition_field_start: Optional[str] = Field( 1713 None, 1714 description="Name of the partition end time field.", 1715 examples=["starting_time"], 1716 title="Partition Field Start", 1717 ) 1718 step: Optional[str] = Field( 1719 None, 1720 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1721 examples=["P1W", "{{ config['step_increment'] }}"], 1722 title="Step", 1723 ) 1724 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1727class JwtAuthenticator(BaseModel): 1728 type: Literal["JwtAuthenticator"] 1729 secret_key: str = Field( 1730 ..., 1731 description="Secret used to sign the JSON web token.", 1732 examples=["{{ config['secret_key'] }}"], 1733 title="Secret Key", 1734 ) 1735 base64_encode_secret_key: Optional[bool] = Field( 1736 False, 1737 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1738 title="Base64-encode Secret Key", 1739 ) 1740 algorithm: Algorithm = Field( 1741 ..., 1742 description="Algorithm used to sign the JSON web token.", 1743 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1744 title="Algorithm", 1745 ) 1746 token_duration: Optional[int] = Field( 1747 1200, 1748 description="The amount of time in seconds a JWT token can be valid after being issued.", 1749 examples=[1200, 3600], 1750 title="Token Duration", 1751 ) 1752 header_prefix: Optional[str] = Field( 1753 None, 1754 description="The prefix to be used within the Authentication header.", 1755 examples=["Bearer", "Basic"], 1756 title="Header Prefix", 1757 ) 1758 jwt_headers: Optional[JwtHeaders] = Field( 1759 None, 1760 description="JWT headers used when signing JSON web token.", 1761 title="JWT Headers", 1762 ) 1763 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1764 None, 1765 description="Additional headers to be included with the JWT headers object.", 1766 title="Additional JWT Headers", 1767 ) 1768 jwt_payload: Optional[JwtPayload] = Field( 1769 None, 1770 description="JWT Payload used when signing JSON web token.", 1771 title="JWT Payload", 1772 ) 1773 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1774 None, 1775 description="Additional properties to be added to the JWT payload.", 1776 title="Additional JWT Payload Properties", 1777 ) 1778 passphrase: Optional[str] = Field( 1779 None, 1780 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1781 examples=["{{ config['passphrase'] }}"], 1782 title="Passphrase", 1783 ) 1784 request_option: Optional[RequestOption] = Field( 1785 None, 1786 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1787 title="Request Option", 1788 ) 1789 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1792class OAuthAuthenticator(BaseModel): 1793 type: Literal["OAuthAuthenticator"] 1794 client_id_name: Optional[str] = Field( 1795 "client_id", 1796 description="The name of the property to use to refresh the `access_token`.", 1797 examples=["custom_app_id"], 1798 title="Client ID Property Name", 1799 ) 1800 client_id: Optional[str] = Field( 1801 None, 1802 description="The OAuth client ID. Fill it in the user inputs.", 1803 examples=[ 1804 "{{ config['client_id'] }}", 1805 "{{ config['credentials']['client_id }}", 1806 ], 1807 title="Client ID", 1808 ) 1809 client_secret_name: Optional[str] = Field( 1810 "client_secret", 1811 description="The name of the property to use to refresh the `access_token`.", 1812 examples=["custom_app_secret"], 1813 title="Client Secret Property Name", 1814 ) 1815 client_secret: Optional[str] = Field( 1816 None, 1817 description="The OAuth client secret. Fill it in the user inputs.", 1818 examples=[ 1819 "{{ config['client_secret'] }}", 1820 "{{ config['credentials']['client_secret }}", 1821 ], 1822 title="Client Secret", 1823 ) 1824 refresh_token_name: Optional[str] = Field( 1825 "refresh_token", 1826 description="The name of the property to use to refresh the `access_token`.", 1827 examples=["custom_app_refresh_value"], 1828 title="Refresh Token Property Name", 1829 ) 1830 refresh_token: Optional[str] = Field( 1831 None, 1832 description="Credential artifact used to get a new access token.", 1833 examples=[ 1834 "{{ config['refresh_token'] }}", 1835 "{{ config['credentials]['refresh_token'] }}", 1836 ], 1837 title="Refresh Token", 1838 ) 1839 token_refresh_endpoint: Optional[str] = Field( 1840 None, 1841 description="The full URL to call to obtain a new access token.", 1842 examples=["https://connect.squareup.com/oauth2/token"], 1843 title="Token Refresh Endpoint", 1844 ) 1845 access_token_name: Optional[str] = Field( 1846 "access_token", 1847 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1848 examples=["access_token"], 1849 title="Access Token Property Name", 1850 ) 1851 access_token_value: Optional[str] = Field( 1852 None, 1853 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1854 examples=["secret_access_token_value"], 1855 title="Access Token Value", 1856 ) 1857 expires_in_name: Optional[str] = Field( 1858 "expires_in", 1859 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1860 examples=["expires_in"], 1861 title="Token Expiry Property Name", 1862 ) 1863 grant_type_name: Optional[str] = Field( 1864 "grant_type", 1865 description="The name of the property to use to refresh the `access_token`.", 1866 examples=["custom_grant_type"], 1867 title="Grant Type Property Name", 1868 ) 1869 grant_type: Optional[str] = Field( 1870 "refresh_token", 1871 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1872 examples=["refresh_token", "client_credentials"], 1873 title="Grant Type", 1874 ) 1875 refresh_request_body: Optional[Dict[str, Any]] = Field( 1876 None, 1877 description="Body of the request sent to get a new access token.", 1878 examples=[ 1879 { 1880 "applicationId": "{{ config['application_id'] }}", 1881 "applicationSecret": "{{ config['application_secret'] }}", 1882 "token": "{{ config['token'] }}", 1883 } 1884 ], 1885 title="Refresh Request Body", 1886 ) 1887 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1888 None, 1889 description="Headers of the request sent to get a new access token.", 1890 examples=[ 1891 { 1892 "Authorization": "<AUTH_TOKEN>", 1893 "Content-Type": "application/x-www-form-urlencoded", 1894 } 1895 ], 1896 title="Refresh Request Headers", 1897 ) 1898 scopes: Optional[List[str]] = Field( 1899 None, 1900 description="List of scopes that should be granted to the access token.", 1901 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1902 title="Scopes", 1903 ) 1904 token_expiry_date: Optional[str] = Field( 1905 None, 1906 description="The access token expiry date.", 1907 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1908 title="Token Expiry Date", 1909 ) 1910 token_expiry_date_format: Optional[str] = Field( 1911 None, 1912 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1913 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1914 title="Token Expiry Date Format", 1915 ) 1916 refresh_token_error_status_codes: Optional[List[int]] = Field( 1917 None, 1918 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1919 examples=[[400, 500]], 1920 title="Refresh Token Error Status Codes", 1921 ) 1922 refresh_token_error_key: Optional[str] = Field( 1923 None, 1924 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1925 examples=["error"], 1926 title="Refresh Token Error Key", 1927 ) 1928 refresh_token_error_values: Optional[List[str]] = Field( 1929 None, 1930 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1931 examples=[["invalid_grant", "invalid_permissions"]], 1932 title="Refresh Token Error Values", 1933 ) 1934 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1935 None, 1936 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1937 title="Refresh Token Updater", 1938 ) 1939 profile_assertion: Optional[JwtAuthenticator] = Field( 1940 None, 1941 description="The authenticator being used to authenticate the client authenticator.", 1942 title="Profile Assertion", 1943 ) 1944 use_profile_assertion: Optional[bool] = Field( 1945 False, 1946 description="Enable using profile assertion as a flow for OAuth authorization.", 1947 title="Use Profile Assertion", 1948 ) 1949 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1952class FixedWindowCallRatePolicy(BaseModel): 1953 class Config: 1954 extra = Extra.allow 1955 1956 type: Literal["FixedWindowCallRatePolicy"] 1957 period: str = Field( 1958 ..., description="The time interval for the rate limit window.", title="Period" 1959 ) 1960 call_limit: int = Field( 1961 ..., 1962 description="The maximum number of calls allowed within the period.", 1963 title="Call Limit", 1964 ) 1965 matchers: List[HttpRequestRegexMatcher] = Field( 1966 ..., 1967 description="List of matchers that define which requests this policy applies to.", 1968 title="Matchers", 1969 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1972class MovingWindowCallRatePolicy(BaseModel): 1973 class Config: 1974 extra = Extra.allow 1975 1976 type: Literal["MovingWindowCallRatePolicy"] 1977 rates: List[Rate] = Field( 1978 ..., 1979 description="List of rates that define the call limits for different time intervals.", 1980 title="Rates", 1981 ) 1982 matchers: List[HttpRequestRegexMatcher] = Field( 1983 ..., 1984 description="List of matchers that define which requests this policy applies to.", 1985 title="Matchers", 1986 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1989class UnlimitedCallRatePolicy(BaseModel): 1990 class Config: 1991 extra = Extra.allow 1992 1993 type: Literal["UnlimitedCallRatePolicy"] 1994 matchers: List[HttpRequestRegexMatcher] = Field( 1995 ..., 1996 description="List of matchers that define which requests this policy applies to.", 1997 title="Matchers", 1998 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2001class DefaultErrorHandler(BaseModel): 2002 type: Literal["DefaultErrorHandler"] 2003 backoff_strategies: Optional[ 2004 List[ 2005 Union[ 2006 ConstantBackoffStrategy, 2007 ExponentialBackoffStrategy, 2008 WaitTimeFromHeader, 2009 WaitUntilTimeFromHeader, 2010 CustomBackoffStrategy, 2011 ] 2012 ] 2013 ] = Field( 2014 None, 2015 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2016 title="Backoff Strategies", 2017 ) 2018 max_retries: Optional[int] = Field( 2019 5, 2020 description="The maximum number of time to retry a retryable request before giving up and failing.", 2021 examples=[5, 0, 10], 2022 title="Max Retry Count", 2023 ) 2024 response_filters: Optional[List[HttpResponseFilter]] = Field( 2025 None, 2026 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2027 title="Response Filters", 2028 ) 2029 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2032class DefaultPaginator(BaseModel): 2033 type: Literal["DefaultPaginator"] 2034 pagination_strategy: Union[ 2035 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2036 ] = Field( 2037 ..., 2038 description="Strategy defining how records are paginated.", 2039 title="Pagination Strategy", 2040 ) 2041 page_size_option: Optional[RequestOption] = Field( 2042 None, title="Inject Page Size Into Outgoing HTTP Request" 2043 ) 2044 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2045 None, 2046 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2047 title="Inject Page Token Into Outgoing HTTP Request", 2048 ) 2049 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2052class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2053 type: Literal["ApiKey"] 2054 inject_into: RequestOption = Field( 2055 ..., 2056 description="Configure how the API Key will be sent in requests to the source API.", 2057 examples=[ 2058 {"inject_into": "header", "field_name": "Authorization"}, 2059 {"inject_into": "request_parameter", "field_name": "authKey"}, 2060 ], 2061 title="Inject API Key Into Outgoing HTTP Request", 2062 ) 2063 api_token: Optional[str] = Field( 2064 "{{ session_token }}", 2065 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2066 examples=[ 2067 "{{ session_token }}", 2068 "Token {{ session_token }}", 2069 "Bearer {{ session_token }}", 2070 ], 2071 title="API Token Template", 2072 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2075class JsonSchemaPropertySelector(BaseModel): 2076 type: Literal["JsonSchemaPropertySelector"] 2077 transformations: Optional[ 2078 List[ 2079 Union[ 2080 AddFields, 2081 RemoveFields, 2082 KeysToLower, 2083 KeysToSnakeCase, 2084 FlattenFields, 2085 DpathFlattenFields, 2086 KeysReplace, 2087 CustomTransformation, 2088 ] 2089 ] 2090 ] = Field( 2091 None, 2092 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2093 title="Transformations", 2094 ) 2095 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2098class ListPartitionRouter(BaseModel): 2099 type: Literal["ListPartitionRouter"] 2100 cursor_field: str = Field( 2101 ..., 2102 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2103 examples=["section", "{{ config['section_key'] }}"], 2104 title="Current Partition Value Identifier", 2105 ) 2106 values: Union[str, List[str]] = Field( 2107 ..., 2108 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2109 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2110 title="Partition Values", 2111 ) 2112 request_option: Optional[RequestOption] = Field( 2113 None, 2114 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2115 title="Inject Partition Value Into Outgoing HTTP Request", 2116 ) 2117 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2120class RecordSelector(BaseModel): 2121 type: Literal["RecordSelector"] 2122 extractor: Union[DpathExtractor, CustomRecordExtractor] 2123 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2124 None, 2125 description="Responsible for filtering records to be emitted by the Source.", 2126 title="Record Filter", 2127 ) 2128 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2129 None, 2130 description="Responsible for normalization according to the schema.", 2131 title="Schema Normalization", 2132 ) 2133 transform_before_filtering: Optional[bool] = Field( 2134 None, 2135 description="If true, transformation will be applied before record filtering.", 2136 title="Transform Before Filtering", 2137 ) 2138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
PaginationReset(pydantic.v1.main.BaseModel):
2141class PaginationReset(BaseModel): 2142 type: Literal["PaginationReset"] 2143 action: Action1 2144 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2147class GzipDecoder(BaseModel): 2148 type: Literal["GzipDecoder"] 2149 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2152class RequestBodyGraphQL(BaseModel): 2153 type: Literal["RequestBodyGraphQL"] 2154 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2157class DpathValidator(BaseModel): 2158 type: Literal["DpathValidator"] 2159 field_path: List[str] = Field( 2160 ..., 2161 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2162 examples=[ 2163 ["data"], 2164 ["data", "records"], 2165 ["data", "{{ parameters.name }}"], 2166 ["data", "*", "record"], 2167 ], 2168 title="Field Path", 2169 ) 2170 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2171 ..., 2172 description="The condition that the specified config value will be evaluated against", 2173 title="Validation Strategy", 2174 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2177class PredicateValidator(BaseModel): 2178 type: Literal["PredicateValidator"] 2179 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2180 ..., 2181 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2182 examples=[ 2183 "test-value", 2184 "{{ config['api_version'] }}", 2185 "{{ config['tenant_id'] }}", 2186 123, 2187 ], 2188 title="Value", 2189 ) 2190 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2191 ..., 2192 description="The validation strategy to apply to the value.", 2193 title="Validation Strategy", 2194 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2197class ConfigAddFields(BaseModel): 2198 type: Literal["ConfigAddFields"] 2199 fields: List[AddedFieldDefinition] = Field( 2200 ..., 2201 description="A list of transformations (path and corresponding value) that will be added to the config.", 2202 title="Fields", 2203 ) 2204 condition: Optional[str] = Field( 2205 "", 2206 description="Fields will be added if expression is evaluated to True.", 2207 examples=[ 2208 "{{ config['environemnt'] == 'sandbox' }}", 2209 "{{ property is integer }}", 2210 "{{ property|length > 5 }}", 2211 "{{ property == 'some_string_to_match' }}", 2212 ], 2213 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2216class CompositeErrorHandler(BaseModel): 2217 type: Literal["CompositeErrorHandler"] 2218 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2219 Field( 2220 ..., 2221 description="List of error handlers to iterate on to determine how to handle a failed response.", 2222 title="Error Handlers", 2223 ) 2224 ) 2225 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2228class HTTPAPIBudget(BaseModel): 2229 class Config: 2230 extra = Extra.allow 2231 2232 type: Literal["HTTPAPIBudget"] 2233 policies: List[ 2234 Union[ 2235 FixedWindowCallRatePolicy, 2236 MovingWindowCallRatePolicy, 2237 UnlimitedCallRatePolicy, 2238 ] 2239 ] = Field( 2240 ..., 2241 description="List of call rate policies that define how many calls are allowed.", 2242 title="Policies", 2243 ) 2244 ratelimit_reset_header: Optional[str] = Field( 2245 "ratelimit-reset", 2246 description="The HTTP response header name that indicates when the rate limit resets.", 2247 title="Rate Limit Reset Header", 2248 ) 2249 ratelimit_remaining_header: Optional[str] = Field( 2250 "ratelimit-remaining", 2251 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2252 title="Rate Limit Remaining Header", 2253 ) 2254 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2255 [429], 2256 description="List of HTTP status codes that indicate a rate limit has been hit.", 2257 title="Status Codes for Rate Limit Hit", 2258 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2261class ZipfileDecoder(BaseModel): 2262 class Config: 2263 extra = Extra.allow 2264 2265 type: Literal["ZipfileDecoder"] 2266 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2267 ..., 2268 description="Parser to parse the decompressed data from the zipfile(s).", 2269 title="Parser", 2270 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2273class ConfigMigration(BaseModel): 2274 type: Literal["ConfigMigration"] 2275 description: Optional[str] = Field( 2276 None, description="The description/purpose of the config migration." 2277 ) 2278 transformations: List[ 2279 Union[ 2280 ConfigRemapField, 2281 ConfigAddFields, 2282 ConfigRemoveFields, 2283 CustomConfigTransformation, 2284 ] 2285 ] = Field( 2286 ..., 2287 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2288 title="Transformations", 2289 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2292class ConfigNormalizationRules(BaseModel): 2293 class Config: 2294 extra = Extra.forbid 2295 2296 type: Literal["ConfigNormalizationRules"] 2297 config_migrations: Optional[List[ConfigMigration]] = Field( 2298 [], 2299 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2300 title="Config Migrations", 2301 ) 2302 transformations: Optional[ 2303 List[ 2304 Union[ 2305 ConfigRemapField, 2306 ConfigAddFields, 2307 ConfigRemoveFields, 2308 CustomConfigTransformation, 2309 ] 2310 ] 2311 ] = Field( 2312 [], 2313 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2314 title="Transformations", 2315 ) 2316 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2317 [], 2318 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2319 title="Validations", 2320 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2323class Spec(BaseModel): 2324 type: Literal["Spec"] 2325 connection_specification: Dict[str, Any] = Field( 2326 ..., 2327 description="A connection specification describing how a the connector can be configured.", 2328 title="Connection Specification", 2329 ) 2330 documentation_url: Optional[str] = Field( 2331 None, 2332 description="URL of the connector's documentation page.", 2333 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2334 title="Documentation URL", 2335 ) 2336 advanced_auth: Optional[AuthFlow] = Field( 2337 None, 2338 description="Advanced specification for configuring the authentication flow.", 2339 title="Advanced Auth", 2340 ) 2341 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2342 None, title="Config Normalization Rules" 2343 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2346class DeclarativeSource1(BaseModel): 2347 class Config: 2348 extra = Extra.forbid 2349 2350 type: Literal["DeclarativeSource"] 2351 check: Union[CheckStream, CheckDynamicStream] 2352 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2353 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2354 version: str = Field( 2355 ..., 2356 description="The version of the Airbyte CDK used to build and test the source.", 2357 ) 2358 schemas: Optional[Schemas] = None 2359 definitions: Optional[Dict[str, Any]] = None 2360 spec: Optional[Spec] = None 2361 concurrency_level: Optional[ConcurrencyLevel] = None 2362 api_budget: Optional[HTTPAPIBudget] = None 2363 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2364 None, 2365 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2366 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2367 title="Maximum Concurrent Asynchronous Jobs", 2368 ) 2369 metadata: Optional[Dict[str, Any]] = Field( 2370 None, 2371 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2372 ) 2373 description: Optional[str] = Field( 2374 None, 2375 description="A description of the connector. It will be presented on the Source documentation page.", 2376 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2379class DeclarativeSource2(BaseModel): 2380 class Config: 2381 extra = Extra.forbid 2382 2383 type: Literal["DeclarativeSource"] 2384 check: Union[CheckStream, CheckDynamicStream] 2385 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2386 None 2387 ) 2388 dynamic_streams: List[DynamicDeclarativeStream] 2389 version: str = Field( 2390 ..., 2391 description="The version of the Airbyte CDK used to build and test the source.", 2392 ) 2393 schemas: Optional[Schemas] = None 2394 definitions: Optional[Dict[str, Any]] = None 2395 spec: Optional[Spec] = None 2396 concurrency_level: Optional[ConcurrencyLevel] = None 2397 api_budget: Optional[HTTPAPIBudget] = None 2398 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2399 None, 2400 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2401 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2402 title="Maximum Concurrent Asynchronous Jobs", 2403 ) 2404 metadata: Optional[Dict[str, Any]] = Field( 2405 None, 2406 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2407 ) 2408 description: Optional[str] = Field( 2409 None, 2410 description="A description of the connector. It will be presented on the Source documentation page.", 2411 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2414class DeclarativeSource(BaseModel): 2415 class Config: 2416 extra = Extra.forbid 2417 2418 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2419 ..., 2420 description="An API source that extracts data according to its declarative components.", 2421 title="DeclarativeSource", 2422 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2425class SelectiveAuthenticator(BaseModel): 2426 class Config: 2427 extra = Extra.allow 2428 2429 type: Literal["SelectiveAuthenticator"] 2430 authenticator_selection_path: List[str] = Field( 2431 ..., 2432 description="Path of the field in config with selected authenticator name", 2433 examples=[["auth"], ["auth", "type"]], 2434 title="Authenticator Selection Path", 2435 ) 2436 authenticators: Dict[ 2437 str, 2438 Union[ 2439 ApiKeyAuthenticator, 2440 BasicHttpAuthenticator, 2441 BearerAuthenticator, 2442 OAuthAuthenticator, 2443 JwtAuthenticator, 2444 SessionTokenAuthenticator, 2445 LegacySessionTokenAuthenticator, 2446 CustomAuthenticator, 2447 NoAuth, 2448 ], 2449 ] = Field( 2450 ..., 2451 description="Authenticators to select from.", 2452 examples=[ 2453 { 2454 "authenticators": { 2455 "token": "#/definitions/ApiKeyAuthenticator", 2456 "oauth": "#/definitions/OAuthAuthenticator", 2457 "jwt": "#/definitions/JwtAuthenticator", 2458 } 2459 } 2460 ], 2461 title="Authenticators", 2462 ) 2463 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2466class ConditionalStreams(BaseModel): 2467 type: Literal["ConditionalStreams"] 2468 condition: str = Field( 2469 ..., 2470 description="Condition that will be evaluated to determine if a set of streams should be available.", 2471 examples=["{{ config['is_sandbox'] }}"], 2472 title="Condition", 2473 ) 2474 streams: List[DeclarativeStream] = Field( 2475 ..., 2476 description="Streams that will be used during an operation based on the condition.", 2477 title="Streams", 2478 ) 2479 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2482class FileUploader(BaseModel): 2483 type: Literal["FileUploader"] 2484 requester: Union[HttpRequester, CustomRequester] = Field( 2485 ..., 2486 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2487 ) 2488 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2489 ..., 2490 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2491 ) 2492 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2493 None, 2494 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2495 ) 2496 filename_extractor: Optional[str] = Field( 2497 None, 2498 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2499 examples=[ 2500 "{{ record.id }}/{{ record.file_name }}/", 2501 "{{ record.id }}_{{ record.file_name }}/", 2502 ], 2503 ) 2504 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2507class DeclarativeStream(BaseModel): 2508 class Config: 2509 extra = Extra.allow 2510 2511 type: Literal["DeclarativeStream"] 2512 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2513 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2514 ..., 2515 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2516 title="Retriever", 2517 ) 2518 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2519 None, 2520 description="Component used to fetch data incrementally based on a time field in the data.", 2521 title="Incremental Sync", 2522 ) 2523 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2524 schema_loader: Optional[ 2525 Union[ 2526 InlineSchemaLoader, 2527 DynamicSchemaLoader, 2528 JsonFileSchemaLoader, 2529 List[ 2530 Union[ 2531 InlineSchemaLoader, 2532 DynamicSchemaLoader, 2533 JsonFileSchemaLoader, 2534 CustomSchemaLoader, 2535 ] 2536 ], 2537 CustomSchemaLoader, 2538 ] 2539 ] = Field( 2540 None, 2541 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2542 title="Schema Loader", 2543 ) 2544 transformations: Optional[ 2545 List[ 2546 Union[ 2547 AddFields, 2548 RemoveFields, 2549 KeysToLower, 2550 KeysToSnakeCase, 2551 FlattenFields, 2552 DpathFlattenFields, 2553 KeysReplace, 2554 CustomTransformation, 2555 ] 2556 ] 2557 ] = Field( 2558 None, 2559 description="A list of transformations to be applied to each output record.", 2560 title="Transformations", 2561 ) 2562 state_migrations: Optional[ 2563 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2564 ] = Field( 2565 [], 2566 description="Array of state migrations to be applied on the input state", 2567 title="State Migrations", 2568 ) 2569 file_uploader: Optional[FileUploader] = Field( 2570 None, 2571 description="(experimental) Describes how to fetch a file", 2572 title="File Uploader", 2573 ) 2574 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2577class SessionTokenAuthenticator(BaseModel): 2578 type: Literal["SessionTokenAuthenticator"] 2579 login_requester: HttpRequester = Field( 2580 ..., 2581 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2582 examples=[ 2583 { 2584 "type": "HttpRequester", 2585 "url_base": "https://my_api.com", 2586 "path": "/login", 2587 "authenticator": { 2588 "type": "BasicHttpAuthenticator", 2589 "username": "{{ config.username }}", 2590 "password": "{{ config.password }}", 2591 }, 2592 } 2593 ], 2594 title="Login Requester", 2595 ) 2596 session_token_path: List[str] = Field( 2597 ..., 2598 description="The path in the response body returned from the login requester to the session token.", 2599 examples=[["access_token"], ["result", "token"]], 2600 title="Session Token Path", 2601 ) 2602 expiration_duration: Optional[str] = Field( 2603 None, 2604 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2605 examples=["PT1H", "P1D"], 2606 title="Expiration Duration", 2607 ) 2608 request_authentication: Union[ 2609 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2610 ] = Field( 2611 ..., 2612 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2613 title="Data Request Authentication", 2614 ) 2615 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2616 None, description="Component used to decode the response.", title="Decoder" 2617 ) 2618 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2621class HttpRequester(BaseModelWithDeprecations): 2622 type: Literal["HttpRequester"] 2623 url_base: Optional[str] = Field( 2624 None, 2625 deprecated=True, 2626 deprecation_message="Use `url` field instead.", 2627 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2628 examples=[ 2629 "https://connect.squareup.com/v2", 2630 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2631 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2632 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2633 ], 2634 title="API Base URL", 2635 ) 2636 url: Optional[str] = Field( 2637 None, 2638 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2639 examples=[ 2640 "https://connect.squareup.com/v2", 2641 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2642 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2643 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2644 ], 2645 title="API Endpoint URL", 2646 ) 2647 path: Optional[str] = Field( 2648 None, 2649 deprecated=True, 2650 deprecation_message="Use `url` field instead.", 2651 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2652 examples=[ 2653 "/products", 2654 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2655 "/trades/{{ config['symbol_id'] }}/history", 2656 ], 2657 title="URL Path", 2658 ) 2659 http_method: Optional[HttpMethod] = Field( 2660 HttpMethod.GET, 2661 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2662 examples=["GET", "POST"], 2663 title="HTTP Method", 2664 ) 2665 authenticator: Optional[ 2666 Union[ 2667 ApiKeyAuthenticator, 2668 BasicHttpAuthenticator, 2669 BearerAuthenticator, 2670 OAuthAuthenticator, 2671 JwtAuthenticator, 2672 SessionTokenAuthenticator, 2673 SelectiveAuthenticator, 2674 CustomAuthenticator, 2675 NoAuth, 2676 LegacySessionTokenAuthenticator, 2677 ] 2678 ] = Field( 2679 None, 2680 description="Authentication method to use for requests sent to the API.", 2681 title="Authenticator", 2682 ) 2683 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2684 None, 2685 deprecated=True, 2686 deprecation_message="Use `query_properties` field instead.", 2687 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2688 title="Fetch Properties from Endpoint", 2689 ) 2690 query_properties: Optional[QueryProperties] = Field( 2691 None, 2692 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2693 title="Query Properties", 2694 ) 2695 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2696 None, 2697 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2698 examples=[ 2699 {"unit": "day"}, 2700 { 2701 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2702 }, 2703 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2704 {"sort_by[asc]": "updated_at"}, 2705 ], 2706 title="Query Parameters", 2707 ) 2708 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2709 None, 2710 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2711 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2712 title="Request Headers", 2713 ) 2714 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2715 None, 2716 deprecated=True, 2717 deprecation_message="Use `request_body` field instead.", 2718 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2719 examples=[ 2720 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2721 ], 2722 title="Request Body Payload (Non-JSON)", 2723 ) 2724 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2725 None, 2726 deprecated=True, 2727 deprecation_message="Use `request_body` field instead.", 2728 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2729 examples=[ 2730 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2731 {"key": "{{ config['value'] }}"}, 2732 {"sort": {"field": "updated_at", "order": "ascending"}}, 2733 ], 2734 title="Request Body JSON Payload", 2735 ) 2736 request_body: Optional[ 2737 Union[ 2738 RequestBodyPlainText, 2739 RequestBodyUrlEncodedForm, 2740 RequestBodyJsonObject, 2741 RequestBodyGraphQL, 2742 ] 2743 ] = Field( 2744 None, 2745 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2746 title="Request Body", 2747 ) 2748 error_handler: Optional[ 2749 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2750 ] = Field( 2751 None, 2752 description="Error handler component that defines how to handle errors.", 2753 title="Error Handler", 2754 ) 2755 use_cache: Optional[bool] = Field( 2756 False, 2757 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2758 title="Use Cache", 2759 ) 2760 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2763class DynamicSchemaLoader(BaseModel): 2764 type: Literal["DynamicSchemaLoader"] 2765 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2766 ..., 2767 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2768 title="Retriever", 2769 ) 2770 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2771 None, 2772 description="Responsible for filtering fields to be added to json schema.", 2773 title="Schema Filter", 2774 ) 2775 schema_transformations: Optional[ 2776 List[ 2777 Union[ 2778 AddFields, 2779 RemoveFields, 2780 KeysToLower, 2781 KeysToSnakeCase, 2782 FlattenFields, 2783 DpathFlattenFields, 2784 KeysReplace, 2785 CustomTransformation, 2786 ] 2787 ] 2788 ] = Field( 2789 None, 2790 description="A list of transformations to be applied to the schema.", 2791 title="Schema Transformations", 2792 ) 2793 schema_type_identifier: SchemaTypeIdentifier 2794 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2797class ParentStreamConfig(BaseModel): 2798 type: Literal["ParentStreamConfig"] 2799 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2800 ..., description="Reference to the parent stream.", title="Parent Stream" 2801 ) 2802 parent_key: str = Field( 2803 ..., 2804 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2805 examples=["id", "{{ config['parent_record_id'] }}"], 2806 title="Parent Key", 2807 ) 2808 partition_field: str = Field( 2809 ..., 2810 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2811 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2812 title="Current Parent Key Value Identifier", 2813 ) 2814 request_option: Optional[RequestOption] = Field( 2815 None, 2816 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2817 title="Request Option", 2818 ) 2819 incremental_dependency: Optional[bool] = Field( 2820 False, 2821 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2822 title="Incremental Dependency", 2823 ) 2824 lazy_read_pointer: Optional[List[str]] = Field( 2825 [], 2826 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2827 title="Lazy Read Pointer", 2828 ) 2829 extra_fields: Optional[List[List[str]]] = Field( 2830 None, 2831 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2832 title="Extra Fields", 2833 ) 2834 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2837class PropertiesFromEndpoint(BaseModel): 2838 type: Literal["PropertiesFromEndpoint"] 2839 property_field_path: List[str] = Field( 2840 ..., 2841 description="Describes the path to the field that should be extracted", 2842 examples=[["name"]], 2843 ) 2844 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2845 ..., 2846 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2847 ) 2848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2851class QueryProperties(BaseModel): 2852 type: Literal["QueryProperties"] 2853 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2854 ..., 2855 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2856 title="Property List", 2857 ) 2858 always_include_properties: Optional[List[str]] = Field( 2859 None, 2860 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2861 title="Always Include Properties", 2862 ) 2863 property_chunking: Optional[PropertyChunking] = Field( 2864 None, 2865 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2866 title="Property Chunking", 2867 ) 2868 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2869 None, 2870 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2871 title="Property Selector", 2872 ) 2873 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2876class StateDelegatingStream(BaseModel): 2877 type: Literal["StateDelegatingStream"] 2878 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2879 full_refresh_stream: DeclarativeStream = Field( 2880 ..., 2881 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2882 title="Full Refresh Stream", 2883 ) 2884 incremental_stream: DeclarativeStream = Field( 2885 ..., 2886 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2887 title="Incremental Stream", 2888 ) 2889 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2892class SimpleRetriever(BaseModel): 2893 type: Literal["SimpleRetriever"] 2894 requester: Union[HttpRequester, CustomRequester] = Field( 2895 ..., 2896 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2897 ) 2898 decoder: Optional[ 2899 Union[ 2900 JsonDecoder, 2901 XmlDecoder, 2902 CsvDecoder, 2903 JsonlDecoder, 2904 GzipDecoder, 2905 IterableDecoder, 2906 ZipfileDecoder, 2907 CustomDecoder, 2908 ] 2909 ] = Field( 2910 None, 2911 description="Component decoding the response so records can be extracted.", 2912 title="HTTP Response Format", 2913 ) 2914 record_selector: RecordSelector = Field( 2915 ..., 2916 description="Component that describes how to extract records from a HTTP response.", 2917 ) 2918 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2919 None, 2920 description="Paginator component that describes how to navigate through the API's pages.", 2921 ) 2922 pagination_reset: Optional[PaginationReset] = Field( 2923 None, 2924 description="Describes what triggers pagination reset and how to handle it.", 2925 ) 2926 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2927 False, 2928 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2929 ) 2930 partition_router: Optional[ 2931 Union[ 2932 SubstreamPartitionRouter, 2933 ListPartitionRouter, 2934 GroupingPartitionRouter, 2935 CustomPartitionRouter, 2936 List[ 2937 Union[ 2938 SubstreamPartitionRouter, 2939 ListPartitionRouter, 2940 GroupingPartitionRouter, 2941 CustomPartitionRouter, 2942 ] 2943 ], 2944 ] 2945 ] = Field( 2946 None, 2947 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2948 title="Partition Router", 2949 ) 2950 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2953class AsyncRetriever(BaseModel): 2954 type: Literal["AsyncRetriever"] 2955 record_selector: RecordSelector = Field( 2956 ..., 2957 description="Component that describes how to extract records from a HTTP response.", 2958 ) 2959 status_mapping: AsyncJobStatusMap = Field( 2960 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2961 ) 2962 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2963 ..., description="Responsible for fetching the actual status of the async job." 2964 ) 2965 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2966 None, 2967 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2968 ) 2969 download_extractor: Optional[ 2970 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2971 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2972 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2973 ..., 2974 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2975 ) 2976 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2977 ..., 2978 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2979 ) 2980 polling_job_timeout: Optional[Union[int, str]] = Field( 2981 None, 2982 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2983 ) 2984 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2985 None, 2986 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2987 ) 2988 download_requester: Union[HttpRequester, CustomRequester] = Field( 2989 ..., 2990 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2991 ) 2992 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2993 None, 2994 description="Paginator component that describes how to navigate through the API's pages during download.", 2995 ) 2996 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2997 None, 2998 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2999 ) 3000 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3001 None, 3002 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3003 ) 3004 partition_router: Optional[ 3005 Union[ 3006 ListPartitionRouter, 3007 SubstreamPartitionRouter, 3008 GroupingPartitionRouter, 3009 CustomPartitionRouter, 3010 List[ 3011 Union[ 3012 ListPartitionRouter, 3013 SubstreamPartitionRouter, 3014 GroupingPartitionRouter, 3015 CustomPartitionRouter, 3016 ] 3017 ], 3018 ] 3019 ] = Field( 3020 [], 3021 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3022 title="Partition Router", 3023 ) 3024 decoder: Optional[ 3025 Union[ 3026 CsvDecoder, 3027 GzipDecoder, 3028 JsonDecoder, 3029 JsonlDecoder, 3030 IterableDecoder, 3031 XmlDecoder, 3032 ZipfileDecoder, 3033 CustomDecoder, 3034 ] 3035 ] = Field( 3036 None, 3037 description="Component decoding the response so records can be extracted.", 3038 title="HTTP Response Format", 3039 ) 3040 download_decoder: Optional[ 3041 Union[ 3042 CsvDecoder, 3043 GzipDecoder, 3044 JsonDecoder, 3045 JsonlDecoder, 3046 IterableDecoder, 3047 XmlDecoder, 3048 ZipfileDecoder, 3049 CustomDecoder, 3050 ] 3051 ] = Field( 3052 None, 3053 description="Component decoding the download response so records can be extracted.", 3054 title="Download HTTP Response Format", 3055 ) 3056 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3059class SubstreamPartitionRouter(BaseModel): 3060 type: Literal["SubstreamPartitionRouter"] 3061 parent_stream_configs: List[ParentStreamConfig] = Field( 3062 ..., 3063 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3064 title="Parent Stream Configs", 3065 ) 3066 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3069class GroupingPartitionRouter(BaseModel): 3070 type: Literal["GroupingPartitionRouter"] 3071 group_size: int = Field( 3072 ..., 3073 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3074 examples=[10, 50], 3075 title="Group Size", 3076 ) 3077 underlying_partition_router: Union[ 3078 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3079 ] = Field( 3080 ..., 3081 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3082 title="Underlying Partition Router", 3083 ) 3084 deduplicate: Optional[bool] = Field( 3085 True, 3086 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3087 title="Deduplicate Partitions", 3088 ) 3089 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3092class HttpComponentsResolver(BaseModel): 3093 type: Literal["HttpComponentsResolver"] 3094 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3095 ..., 3096 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3097 title="Retriever", 3098 ) 3099 components_mapping: List[ComponentMappingDefinition] 3100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3103class DynamicDeclarativeStream(BaseModel): 3104 type: Literal["DynamicDeclarativeStream"] 3105 name: Optional[str] = Field( 3106 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3107 ) 3108 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3109 ..., description="Reference to the stream template.", title="Stream Template" 3110 ) 3111 components_resolver: Union[ 3112 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3113 ] = Field( 3114 ..., 3115 description="Component resolve and populates stream templates with components values.", 3116 title="Components Resolver", 3117 ) 3118 use_parent_parameters: Optional[bool] = Field( 3119 True, 3120 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3121 title="Use Parent Parameters", 3122 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]