airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class DynamicStreamCheckConfig(BaseModel): 50 type: Literal["DynamicStreamCheckConfig"] 51 dynamic_stream_name: str = Field( 52 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 53 ) 54 stream_count: Optional[int] = Field( 55 0, 56 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 57 title="Stream Count", 58 ) 59 60 61class CheckDynamicStream(BaseModel): 62 type: Literal["CheckDynamicStream"] 63 stream_count: int = Field( 64 ..., 65 description="Numbers of the streams to try reading from when running a check operation.", 66 title="Stream Count", 67 ) 68 use_check_availability: Optional[bool] = Field( 69 True, 70 description="Enables stream check availability. This field is automatically set by the CDK.", 71 title="Use Check Availability", 72 ) 73 74 75class ConcurrencyLevel(BaseModel): 76 type: Optional[Literal["ConcurrencyLevel"]] = None 77 default_concurrency: Union[int, str] = Field( 78 ..., 79 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 80 examples=[10, "{{ config['num_workers'] or 10 }}"], 81 title="Default Concurrency", 82 ) 83 max_concurrency: Optional[int] = Field( 84 None, 85 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 86 examples=[20, 100], 87 title="Max Concurrency", 88 ) 89 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 90 91 92class ConstantBackoffStrategy(BaseModel): 93 type: Literal["ConstantBackoffStrategy"] 94 backoff_time_in_seconds: Union[float, str] = Field( 95 ..., 96 description="Backoff time in seconds.", 97 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 98 title="Backoff Time", 99 ) 100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 101 102 103class CursorPagination(BaseModel): 104 type: Literal["CursorPagination"] 105 cursor_value: str = Field( 106 ..., 107 description="Value of the cursor defining the next page to fetch.", 108 examples=[ 109 "{{ headers.link.next.cursor }}", 110 "{{ last_record['key'] }}", 111 "{{ response['nextPage'] }}", 112 ], 113 title="Cursor Value", 114 ) 115 page_size: Optional[int] = Field( 116 None, 117 description="The number of records to include in each pages.", 118 examples=[100], 119 title="Page Size", 120 ) 121 stop_condition: Optional[str] = Field( 122 None, 123 description="Template string evaluating when to stop paginating.", 124 examples=[ 125 "{{ response.data.has_more is false }}", 126 "{{ 'next' not in headers['link'] }}", 127 ], 128 title="Stop Condition", 129 ) 130 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 131 132 133class CustomAuthenticator(BaseModel): 134 class Config: 135 extra = Extra.allow 136 137 type: Literal["CustomAuthenticator"] 138 class_name: str = Field( 139 ..., 140 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 141 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 142 title="Class Name", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomBackoffStrategy(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomBackoffStrategy"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.MyCustomBackoffStrategy"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomErrorHandler(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomErrorHandler"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomErrorHandler"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomPaginationStrategy(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomPaginationStrategy"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomPaginationStrategy"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomRecordExtractor(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomRecordExtractor"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomRecordExtractor"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordFilter(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordFilter"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomCustomRecordFilter"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRequester(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRequester"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomRecordExtractor"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRetriever(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRetriever"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRetriever"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomPartitionRouter(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomPartitionRouter"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomPartitionRouter"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomSchemaLoader(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomSchemaLoader"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomSchemaLoader"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaNormalization(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaNormalization"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 281 examples=[ 282 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 283 ], 284 title="Class Name", 285 ) 286 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 287 288 289class CustomStateMigration(BaseModel): 290 class Config: 291 extra = Extra.allow 292 293 type: Literal["CustomStateMigration"] 294 class_name: str = Field( 295 ..., 296 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 297 examples=["source_railz.components.MyCustomStateMigration"], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomTransformation(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomTransformation"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomTransformation"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class LegacyToPerPartitionStateMigration(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 322 323 324class Clamping(BaseModel): 325 target: str = Field( 326 ..., 327 description="The period of time that datetime windows will be clamped by", 328 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 329 title="Target", 330 ) 331 target_details: Optional[Dict[str, Any]] = None 332 333 334class Algorithm(Enum): 335 HS256 = "HS256" 336 HS384 = "HS384" 337 HS512 = "HS512" 338 ES256 = "ES256" 339 ES256K = "ES256K" 340 ES384 = "ES384" 341 ES512 = "ES512" 342 RS256 = "RS256" 343 RS384 = "RS384" 344 RS512 = "RS512" 345 PS256 = "PS256" 346 PS384 = "PS384" 347 PS512 = "PS512" 348 EdDSA = "EdDSA" 349 350 351class JwtHeaders(BaseModel): 352 class Config: 353 extra = Extra.forbid 354 355 kid: Optional[str] = Field( 356 None, 357 description="Private key ID for user account.", 358 examples=["{{ config['kid'] }}"], 359 title="Key Identifier", 360 ) 361 typ: Optional[str] = Field( 362 "JWT", 363 description="The media type of the complete JWT.", 364 examples=["JWT"], 365 title="Type", 366 ) 367 cty: Optional[str] = Field( 368 None, 369 description="Content type of JWT header.", 370 examples=["JWT"], 371 title="Content Type", 372 ) 373 374 375class JwtPayload(BaseModel): 376 class Config: 377 extra = Extra.forbid 378 379 iss: Optional[str] = Field( 380 None, 381 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 382 examples=["{{ config['iss'] }}"], 383 title="Issuer", 384 ) 385 sub: Optional[str] = Field( 386 None, 387 description="The subject of the JWT. Commonly defined by the API.", 388 title="Subject", 389 ) 390 aud: Optional[str] = Field( 391 None, 392 description="The recipient that the JWT is intended for. Commonly defined by the API.", 393 examples=["appstoreconnect-v1"], 394 title="Audience", 395 ) 396 397 398class RefreshTokenUpdater(BaseModel): 399 refresh_token_name: Optional[str] = Field( 400 "refresh_token", 401 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 402 examples=["refresh_token"], 403 title="Refresh Token Property Name", 404 ) 405 access_token_config_path: Optional[List[str]] = Field( 406 ["credentials", "access_token"], 407 description="Config path to the access token. Make sure the field actually exists in the config.", 408 examples=[["credentials", "access_token"], ["access_token"]], 409 title="Config Path To Access Token", 410 ) 411 refresh_token_config_path: Optional[List[str]] = Field( 412 ["credentials", "refresh_token"], 413 description="Config path to the access token. Make sure the field actually exists in the config.", 414 examples=[["credentials", "refresh_token"], ["refresh_token"]], 415 title="Config Path To Refresh Token", 416 ) 417 token_expiry_date_config_path: Optional[List[str]] = Field( 418 ["credentials", "token_expiry_date"], 419 description="Config path to the expiry date. Make sure actually exists in the config.", 420 examples=[["credentials", "token_expiry_date"]], 421 title="Config Path To Expiry Date", 422 ) 423 refresh_token_error_status_codes: Optional[List[int]] = Field( 424 [], 425 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 426 examples=[[400, 500]], 427 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 428 ) 429 refresh_token_error_key: Optional[str] = Field( 430 "", 431 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 432 examples=["error"], 433 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 434 ) 435 refresh_token_error_values: Optional[List[str]] = Field( 436 [], 437 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 438 examples=[["invalid_grant", "invalid_permissions"]], 439 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 440 ) 441 442 443class Rate(BaseModel): 444 class Config: 445 extra = Extra.allow 446 447 limit: Union[int, str] = Field( 448 ..., 449 description="The maximum number of calls allowed within the interval.", 450 title="Limit", 451 ) 452 interval: str = Field( 453 ..., 454 description="The time interval for the rate limit.", 455 examples=["PT1H", "P1D"], 456 title="Interval", 457 ) 458 459 460class HttpRequestRegexMatcher(BaseModel): 461 class Config: 462 extra = Extra.allow 463 464 method: Optional[str] = Field( 465 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 466 ) 467 url_base: Optional[str] = Field( 468 None, 469 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 470 title="URL Base", 471 ) 472 url_path_pattern: Optional[str] = Field( 473 None, 474 description="A regular expression pattern to match the URL path.", 475 title="URL Path Pattern", 476 ) 477 params: Optional[Dict[str, Any]] = Field( 478 None, description="The query parameters to match.", title="Parameters" 479 ) 480 headers: Optional[Dict[str, Any]] = Field( 481 None, description="The headers to match.", title="Headers" 482 ) 483 484 485class DpathExtractor(BaseModel): 486 type: Literal["DpathExtractor"] 487 field_path: List[str] = Field( 488 ..., 489 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 490 examples=[ 491 ["data"], 492 ["data", "records"], 493 ["data", "{{ parameters.name }}"], 494 ["data", "*", "record"], 495 ], 496 title="Field Path", 497 ) 498 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 499 500 501class ResponseToFileExtractor(BaseModel): 502 type: Literal["ResponseToFileExtractor"] 503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 504 505 506class ExponentialBackoffStrategy(BaseModel): 507 type: Literal["ExponentialBackoffStrategy"] 508 factor: Optional[Union[float, str]] = Field( 509 5, 510 description="Multiplicative constant applied on each retry.", 511 examples=[5, 5.5, "10"], 512 title="Factor", 513 ) 514 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 515 516 517class GroupByKeyMergeStrategy(BaseModel): 518 type: Literal["GroupByKeyMergeStrategy"] 519 key: Union[str, List[str]] = Field( 520 ..., 521 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 522 examples=["id", ["parent_id", "end_date"]], 523 title="Key", 524 ) 525 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 526 527 528class SessionTokenRequestBearerAuthenticator(BaseModel): 529 type: Literal["Bearer"] 530 531 532class HttpMethod(Enum): 533 GET = "GET" 534 POST = "POST" 535 536 537class Action(Enum): 538 SUCCESS = "SUCCESS" 539 FAIL = "FAIL" 540 RETRY = "RETRY" 541 IGNORE = "IGNORE" 542 RESET_PAGINATION = "RESET_PAGINATION" 543 RATE_LIMITED = "RATE_LIMITED" 544 545 546class FailureType(Enum): 547 system_error = "system_error" 548 config_error = "config_error" 549 transient_error = "transient_error" 550 551 552class HttpResponseFilter(BaseModel): 553 type: Literal["HttpResponseFilter"] 554 action: Optional[Action] = Field( 555 None, 556 description="Action to execute if a response matches the filter.", 557 examples=[ 558 "SUCCESS", 559 "FAIL", 560 "RETRY", 561 "IGNORE", 562 "RESET_PAGINATION", 563 "RATE_LIMITED", 564 ], 565 title="Action", 566 ) 567 failure_type: Optional[FailureType] = Field( 568 None, 569 description="Failure type of traced exception if a response matches the filter.", 570 examples=["system_error", "config_error", "transient_error"], 571 title="Failure Type", 572 ) 573 error_message: Optional[str] = Field( 574 None, 575 description="Error Message to display if the response matches the filter.", 576 title="Error Message", 577 ) 578 error_message_contains: Optional[str] = Field( 579 None, 580 description="Match the response if its error message contains the substring.", 581 example=["This API operation is not enabled for this site"], 582 title="Error Message Substring", 583 ) 584 http_codes: Optional[List[int]] = Field( 585 None, 586 description="Match the response if its HTTP code is included in this list.", 587 examples=[[420, 429], [500]], 588 title="HTTP Codes", 589 unique_items=True, 590 ) 591 predicate: Optional[str] = Field( 592 None, 593 description="Match the response if the predicate evaluates to true.", 594 examples=[ 595 "{{ 'Too much requests' in response }}", 596 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 597 ], 598 title="Predicate", 599 ) 600 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 601 602 603class ComplexFieldType(BaseModel): 604 field_type: str 605 items: Optional[Union[str, ComplexFieldType]] = None 606 607 608class TypesMap(BaseModel): 609 target_type: Union[str, List[str], ComplexFieldType] 610 current_type: Union[str, List[str]] 611 condition: Optional[str] = None 612 613 614class SchemaTypeIdentifier(BaseModel): 615 type: Optional[Literal["SchemaTypeIdentifier"]] = None 616 schema_pointer: Optional[List[str]] = Field( 617 [], 618 description="List of nested fields defining the schema field path to extract. Defaults to [].", 619 title="Schema Path", 620 ) 621 key_pointer: List[str] = Field( 622 ..., 623 description="List of potentially nested fields describing the full path of the field key to extract.", 624 title="Key Path", 625 ) 626 type_pointer: Optional[List[str]] = Field( 627 None, 628 description="List of potentially nested fields describing the full path of the field type to extract.", 629 title="Type Path", 630 ) 631 types_mapping: Optional[List[TypesMap]] = None 632 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 633 634 635class InlineSchemaLoader(BaseModel): 636 type: Literal["InlineSchemaLoader"] 637 schema_: Optional[Dict[str, Any]] = Field( 638 None, 639 alias="schema", 640 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 641 title="Schema", 642 ) 643 644 645class JsonFileSchemaLoader(BaseModel): 646 type: Literal["JsonFileSchemaLoader"] 647 file_path: Optional[str] = Field( 648 None, 649 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 650 example=["./schemas/users.json"], 651 title="File Path", 652 ) 653 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 654 655 656class JsonDecoder(BaseModel): 657 type: Literal["JsonDecoder"] 658 659 660class JsonlDecoder(BaseModel): 661 type: Literal["JsonlDecoder"] 662 663 664class KeysToLower(BaseModel): 665 type: Literal["KeysToLower"] 666 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 667 668 669class KeysToSnakeCase(BaseModel): 670 type: Literal["KeysToSnakeCase"] 671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 672 673 674class FlattenFields(BaseModel): 675 type: Literal["FlattenFields"] 676 flatten_lists: Optional[bool] = Field( 677 True, 678 description="Whether to flatten lists or leave it as is. Default is True.", 679 title="Flatten Lists", 680 ) 681 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 682 683 684class KeyTransformation(BaseModel): 685 type: Literal["KeyTransformation"] 686 prefix: Optional[str] = Field( 687 None, 688 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 689 examples=["flattened_"], 690 title="Key Prefix", 691 ) 692 suffix: Optional[str] = Field( 693 None, 694 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 695 examples=["_flattened"], 696 title="Key Suffix", 697 ) 698 699 700class DpathFlattenFields(BaseModel): 701 type: Literal["DpathFlattenFields"] 702 field_path: List[str] = Field( 703 ..., 704 description="A path to field that needs to be flattened.", 705 examples=[["data"], ["data", "*", "field"]], 706 title="Field Path", 707 ) 708 delete_origin_value: Optional[bool] = Field( 709 None, 710 description="Whether to delete the origin value or keep it. Default is False.", 711 title="Delete Origin Value", 712 ) 713 replace_record: Optional[bool] = Field( 714 None, 715 description="Whether to replace the origin record or not. Default is False.", 716 title="Replace Origin Record", 717 ) 718 key_transformation: Optional[KeyTransformation] = Field( 719 None, 720 description="Transformation for object keys. If not provided, original key will be used.", 721 title="Key transformation", 722 ) 723 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 724 725 726class KeysReplace(BaseModel): 727 type: Literal["KeysReplace"] 728 old: str = Field( 729 ..., 730 description="Old value to replace.", 731 examples=[ 732 " ", 733 "{{ record.id }}", 734 "{{ config['id'] }}", 735 "{{ stream_slice['id'] }}", 736 ], 737 title="Old value", 738 ) 739 new: str = Field( 740 ..., 741 description="New value to set.", 742 examples=[ 743 "_", 744 "{{ record.id }}", 745 "{{ config['id'] }}", 746 "{{ stream_slice['id'] }}", 747 ], 748 title="New value", 749 ) 750 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 751 752 753class IterableDecoder(BaseModel): 754 type: Literal["IterableDecoder"] 755 756 757class XmlDecoder(BaseModel): 758 type: Literal["XmlDecoder"] 759 760 761class CustomDecoder(BaseModel): 762 class Config: 763 extra = Extra.allow 764 765 type: Literal["CustomDecoder"] 766 class_name: str = Field( 767 ..., 768 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 769 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 770 title="Class Name", 771 ) 772 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 773 774 775class MinMaxDatetime(BaseModel): 776 type: Literal["MinMaxDatetime"] 777 datetime: str = Field( 778 ..., 779 description="Datetime value.", 780 examples=[ 781 "2021-01-01", 782 "2021-01-01T00:00:00Z", 783 "{{ config['start_time'] }}", 784 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 785 ], 786 title="Datetime", 787 ) 788 datetime_format: Optional[str] = Field( 789 "", 790 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 791 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 792 title="Datetime Format", 793 ) 794 max_datetime: Optional[str] = Field( 795 None, 796 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 797 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 798 title="Max Datetime", 799 ) 800 min_datetime: Optional[str] = Field( 801 None, 802 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 803 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 804 title="Min Datetime", 805 ) 806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 807 808 809class NoAuth(BaseModel): 810 type: Literal["NoAuth"] 811 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 812 813 814class NoPagination(BaseModel): 815 type: Literal["NoPagination"] 816 817 818class State(BaseModel): 819 class Config: 820 extra = Extra.allow 821 822 min: int 823 max: int 824 825 826class OauthConnectorInputSpecification(BaseModel): 827 class Config: 828 extra = Extra.allow 829 830 consent_url: str = Field( 831 ..., 832 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 833 examples=[ 834 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 835 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 836 ], 837 title="Consent URL", 838 ) 839 scope: Optional[str] = Field( 840 None, 841 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 842 examples=["user:read user:read_orders workspaces:read"], 843 title="Scopes", 844 ) 845 access_token_url: str = Field( 846 ..., 847 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 848 examples=[ 849 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 850 ], 851 title="Access Token URL", 852 ) 853 access_token_headers: Optional[Dict[str, Any]] = Field( 854 None, 855 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 856 examples=[ 857 { 858 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 859 } 860 ], 861 title="Access Token Headers", 862 ) 863 access_token_params: Optional[Dict[str, Any]] = Field( 864 None, 865 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 866 examples=[ 867 { 868 "{{ auth_code_key }}": "{{ auth_code_value }}", 869 "{{ client_id_key }}": "{{ client_id_value }}", 870 "{{ client_secret_key }}": "{{ client_secret_value }}", 871 } 872 ], 873 title="Access Token Query Params (Json Encoded)", 874 ) 875 extract_output: Optional[List[str]] = Field( 876 None, 877 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 878 examples=[["access_token", "refresh_token", "other_field"]], 879 title="Extract Output", 880 ) 881 state: Optional[State] = Field( 882 None, 883 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 884 examples=[{"min": 7, "max": 128}], 885 title="Configurable State Query Param", 886 ) 887 client_id_key: Optional[str] = Field( 888 None, 889 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 890 examples=["my_custom_client_id_key_name"], 891 title="Client ID Key Override", 892 ) 893 client_secret_key: Optional[str] = Field( 894 None, 895 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 896 examples=["my_custom_client_secret_key_name"], 897 title="Client Secret Key Override", 898 ) 899 scope_key: Optional[str] = Field( 900 None, 901 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 902 examples=["my_custom_scope_key_key_name"], 903 title="Scopes Key Override", 904 ) 905 state_key: Optional[str] = Field( 906 None, 907 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 908 examples=["my_custom_state_key_key_name"], 909 title="State Key Override", 910 ) 911 auth_code_key: Optional[str] = Field( 912 None, 913 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 914 examples=["my_custom_auth_code_key_name"], 915 title="Auth Code Key Override", 916 ) 917 redirect_uri_key: Optional[str] = Field( 918 None, 919 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 920 examples=["my_custom_redirect_uri_key_name"], 921 title="Redirect URI Key Override", 922 ) 923 924 925class OAuthConfigSpecification(BaseModel): 926 class Config: 927 extra = Extra.allow 928 929 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 930 None, 931 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 932 examples=[ 933 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 934 { 935 "app_id": { 936 "type": "string", 937 "path_in_connector_config": ["info", "app_id"], 938 } 939 }, 940 ], 941 title="OAuth user input", 942 ) 943 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 944 None, 945 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 946 title="DeclarativeOAuth Connector Specification", 947 ) 948 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 949 None, 950 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 951 examples=[ 952 { 953 "refresh_token": { 954 "type": "string,", 955 "path_in_connector_config": ["credentials", "refresh_token"], 956 } 957 } 958 ], 959 title="OAuth output specification", 960 ) 961 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 962 None, 963 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 964 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 965 title="OAuth input specification", 966 ) 967 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 968 None, 969 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 970 examples=[ 971 { 972 "client_id": { 973 "type": "string,", 974 "path_in_connector_config": ["credentials", "client_id"], 975 }, 976 "client_secret": { 977 "type": "string,", 978 "path_in_connector_config": ["credentials", "client_secret"], 979 }, 980 } 981 ], 982 title="OAuth server output specification", 983 ) 984 985 986class OffsetIncrement(BaseModel): 987 type: Literal["OffsetIncrement"] 988 page_size: Optional[Union[int, str]] = Field( 989 None, 990 description="The number of records to include in each pages.", 991 examples=[100, "{{ config['page_size'] }}"], 992 title="Limit", 993 ) 994 inject_on_first_request: Optional[bool] = Field( 995 False, 996 description="Using the `offset` with value `0` during the first request", 997 title="Inject Offset on First Request", 998 ) 999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1000 1001 1002class PageIncrement(BaseModel): 1003 type: Literal["PageIncrement"] 1004 page_size: Optional[Union[int, str]] = Field( 1005 None, 1006 description="The number of records to include in each pages.", 1007 examples=[100, "100", "{{ config['page_size'] }}"], 1008 title="Page Size", 1009 ) 1010 start_from_page: Optional[int] = Field( 1011 0, 1012 description="Index of the first page to request.", 1013 examples=[0, 1], 1014 title="Start From Page", 1015 ) 1016 inject_on_first_request: Optional[bool] = Field( 1017 False, 1018 description="Using the `page number` with value defined by `start_from_page` during the first request", 1019 title="Inject Page Number on First Request", 1020 ) 1021 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1022 1023 1024class PrimaryKey(BaseModel): 1025 __root__: Union[str, List[str], List[List[str]]] = Field( 1026 ..., 1027 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1028 examples=["id", ["code", "type"]], 1029 title="Primary Key", 1030 ) 1031 1032 1033class PropertyLimitType(Enum): 1034 characters = "characters" 1035 property_count = "property_count" 1036 1037 1038class PropertyChunking(BaseModel): 1039 type: Literal["PropertyChunking"] 1040 property_limit_type: PropertyLimitType = Field( 1041 ..., 1042 description="The type used to determine the maximum number of properties per chunk", 1043 title="Property Limit Type", 1044 ) 1045 property_limit: Optional[int] = Field( 1046 None, 1047 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1048 title="Property Limit", 1049 ) 1050 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1051 None, 1052 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1053 title="Record Merge Strategy", 1054 ) 1055 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1056 1057 1058class RecordFilter(BaseModel): 1059 type: Literal["RecordFilter"] 1060 condition: Optional[str] = Field( 1061 "", 1062 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1063 examples=[ 1064 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1065 "{{ record.status in ['active', 'expired'] }}", 1066 ], 1067 title="Condition", 1068 ) 1069 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1070 1071 1072class SchemaNormalization(Enum): 1073 Default = "Default" 1074 None_ = "None" 1075 1076 1077class RemoveFields(BaseModel): 1078 type: Literal["RemoveFields"] 1079 condition: Optional[str] = Field( 1080 "", 1081 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1082 examples=[ 1083 "{{ property|string == '' }}", 1084 "{{ property is integer }}", 1085 "{{ property|length > 5 }}", 1086 "{{ property == 'some_string_to_match' }}", 1087 ], 1088 ) 1089 field_pointers: List[List[str]] = Field( 1090 ..., 1091 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1092 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1093 title="Field Paths", 1094 ) 1095 1096 1097class RequestPath(BaseModel): 1098 type: Literal["RequestPath"] 1099 1100 1101class InjectInto(Enum): 1102 request_parameter = "request_parameter" 1103 header = "header" 1104 body_data = "body_data" 1105 body_json = "body_json" 1106 1107 1108class RequestOption(BaseModel): 1109 type: Literal["RequestOption"] 1110 inject_into: InjectInto = Field( 1111 ..., 1112 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1113 examples=["request_parameter", "header", "body_data", "body_json"], 1114 title="Inject Into", 1115 ) 1116 field_name: Optional[str] = Field( 1117 None, 1118 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1119 examples=["segment_id"], 1120 title="Field Name", 1121 ) 1122 field_path: Optional[List[str]] = Field( 1123 None, 1124 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1125 examples=[["data", "viewer", "id"]], 1126 title="Field Path", 1127 ) 1128 1129 1130class Schemas(BaseModel): 1131 pass 1132 1133 class Config: 1134 extra = Extra.allow 1135 1136 1137class LegacySessionTokenAuthenticator(BaseModel): 1138 type: Literal["LegacySessionTokenAuthenticator"] 1139 header: str = Field( 1140 ..., 1141 description="The name of the session token header that will be injected in the request", 1142 examples=["X-Session"], 1143 title="Session Request Header", 1144 ) 1145 login_url: str = Field( 1146 ..., 1147 description="Path of the login URL (do not include the base URL)", 1148 examples=["session"], 1149 title="Login Path", 1150 ) 1151 session_token: Optional[str] = Field( 1152 None, 1153 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1154 example=["{{ config['session_token'] }}"], 1155 title="Session Token", 1156 ) 1157 session_token_response_key: str = Field( 1158 ..., 1159 description="Name of the key of the session token to be extracted from the response", 1160 examples=["id"], 1161 title="Response Token Response Key", 1162 ) 1163 username: Optional[str] = Field( 1164 None, 1165 description="Username used to authenticate and obtain a session token", 1166 examples=[" {{ config['username'] }}"], 1167 title="Username", 1168 ) 1169 password: Optional[str] = Field( 1170 "", 1171 description="Password used to authenticate and obtain a session token", 1172 examples=["{{ config['password'] }}", ""], 1173 title="Password", 1174 ) 1175 validate_session_url: str = Field( 1176 ..., 1177 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1178 examples=["user/current"], 1179 title="Validate Session Path", 1180 ) 1181 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1182 1183 1184class Action1(Enum): 1185 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1186 RESET = "RESET" 1187 1188 1189class PaginationResetLimits(BaseModel): 1190 type: Literal["PaginationResetLimits"] 1191 number_of_records: Optional[int] = None 1192 1193 1194class CsvDecoder(BaseModel): 1195 type: Literal["CsvDecoder"] 1196 encoding: Optional[str] = "utf-8" 1197 delimiter: Optional[str] = "," 1198 set_values_to_none: Optional[List[str]] = None 1199 1200 1201class AsyncJobStatusMap(BaseModel): 1202 type: Optional[Literal["AsyncJobStatusMap"]] = None 1203 running: List[str] 1204 completed: List[str] 1205 failed: List[str] 1206 timeout: List[str] 1207 1208 1209class ValueType(Enum): 1210 string = "string" 1211 number = "number" 1212 integer = "integer" 1213 boolean = "boolean" 1214 1215 1216class WaitTimeFromHeader(BaseModel): 1217 type: Literal["WaitTimeFromHeader"] 1218 header: str = Field( 1219 ..., 1220 description="The name of the response header defining how long to wait before retrying.", 1221 examples=["Retry-After"], 1222 title="Response Header Name", 1223 ) 1224 regex: Optional[str] = Field( 1225 None, 1226 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1227 examples=["([-+]?\\d+)"], 1228 title="Extraction Regex", 1229 ) 1230 max_waiting_time_in_seconds: Optional[float] = Field( 1231 None, 1232 description="Given the value extracted from the header is greater than this value, stop the stream.", 1233 examples=[3600], 1234 title="Max Waiting Time in Seconds", 1235 ) 1236 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1237 1238 1239class WaitUntilTimeFromHeader(BaseModel): 1240 type: Literal["WaitUntilTimeFromHeader"] 1241 header: str = Field( 1242 ..., 1243 description="The name of the response header defining how long to wait before retrying.", 1244 examples=["wait_time"], 1245 title="Response Header", 1246 ) 1247 min_wait: Optional[Union[float, str]] = Field( 1248 None, 1249 description="Minimum time to wait before retrying.", 1250 examples=[10, "60"], 1251 title="Minimum Wait Time", 1252 ) 1253 regex: Optional[str] = Field( 1254 None, 1255 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1256 examples=["([-+]?\\d+)"], 1257 title="Extraction Regex", 1258 ) 1259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1260 1261 1262class ComponentMappingDefinition(BaseModel): 1263 type: Literal["ComponentMappingDefinition"] 1264 field_path: List[str] = Field( 1265 ..., 1266 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1267 examples=[ 1268 ["name"], 1269 ["retriever", "requester", "url"], 1270 ["retriever", "requester", "{{ components_values.field }}"], 1271 ["*", "**", "name"], 1272 ], 1273 title="Field Path", 1274 ) 1275 value: str = Field( 1276 ..., 1277 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1278 examples=[ 1279 "{{ components_values['updates'] }}", 1280 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1281 "{{ config['segment_id'] }}", 1282 "{{ stream_slice['parent_id'] }}", 1283 "{{ stream_slice['extra_fields']['name'] }}", 1284 ], 1285 title="Value", 1286 ) 1287 value_type: Optional[ValueType] = Field( 1288 None, 1289 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1290 title="Value Type", 1291 ) 1292 create_or_update: Optional[bool] = Field( 1293 False, 1294 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1295 title="Create or Update", 1296 ) 1297 condition: Optional[str] = Field( 1298 None, 1299 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1300 examples=[ 1301 "{{ components_values.get('cursor_field', None) }}", 1302 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1303 ], 1304 title="Condition", 1305 ) 1306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1307 1308 1309class StreamConfig(BaseModel): 1310 type: Literal["StreamConfig"] 1311 configs_pointer: List[str] = Field( 1312 ..., 1313 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1314 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1315 title="Configs Pointer", 1316 ) 1317 default_values: Optional[List[Dict[str, Any]]] = Field( 1318 None, 1319 description="A list of default values, each matching the structure expected from the parsed component value.", 1320 title="Default Values", 1321 ) 1322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1323 1324 1325class ConfigComponentsResolver(BaseModel): 1326 type: Literal["ConfigComponentsResolver"] 1327 stream_config: Union[List[StreamConfig], StreamConfig] 1328 components_mapping: List[ComponentMappingDefinition] 1329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1330 1331 1332class StreamParametersDefinition(BaseModel): 1333 type: Literal["StreamParametersDefinition"] 1334 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1335 ..., 1336 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1337 examples=[ 1338 [ 1339 { 1340 "name": "test stream", 1341 "$parameters": {"entity": "test entity"}, 1342 "primary_key": "test key", 1343 } 1344 ] 1345 ], 1346 title="Stream Parameters", 1347 ) 1348 1349 1350class ParametrizedComponentsResolver(BaseModel): 1351 type: Literal["ParametrizedComponentsResolver"] 1352 stream_parameters: StreamParametersDefinition 1353 components_mapping: List[ComponentMappingDefinition] 1354 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1355 1356 1357class RequestBodyPlainText(BaseModel): 1358 type: Literal["RequestBodyPlainText"] 1359 value: str 1360 1361 1362class RequestBodyUrlEncodedForm(BaseModel): 1363 type: Literal["RequestBodyUrlEncodedForm"] 1364 value: Dict[str, str] 1365 1366 1367class RequestBodyJsonObject(BaseModel): 1368 type: Literal["RequestBodyJsonObject"] 1369 value: Dict[str, Any] 1370 1371 1372class RequestBodyGraphQlQuery(BaseModel): 1373 class Config: 1374 extra = Extra.allow 1375 1376 query: str = Field(..., description="The GraphQL query to be executed") 1377 1378 1379class ValidateAdheresToSchema(BaseModel): 1380 type: Literal["ValidateAdheresToSchema"] 1381 base_schema: Union[str, Dict[str, Any]] = Field( 1382 ..., 1383 description="The base JSON schema against which the user-provided schema will be validated.", 1384 examples=[ 1385 "{{ config['report_validation_schema'] }}", 1386 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1387 { 1388 "$schema": "http://json-schema.org/draft-07/schema#", 1389 "title": "Person", 1390 "type": "object", 1391 "properties": { 1392 "name": {"type": "string", "description": "The person's name"}, 1393 "age": { 1394 "type": "integer", 1395 "minimum": 0, 1396 "description": "The person's age", 1397 }, 1398 }, 1399 "required": ["name", "age"], 1400 }, 1401 ], 1402 title="Base JSON Schema", 1403 ) 1404 1405 1406class CustomValidationStrategy(BaseModel): 1407 class Config: 1408 extra = Extra.allow 1409 1410 type: Literal["CustomValidationStrategy"] 1411 class_name: str = Field( 1412 ..., 1413 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1414 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1415 title="Class Name", 1416 ) 1417 1418 1419class ConfigRemapField(BaseModel): 1420 type: Literal["ConfigRemapField"] 1421 map: Union[Dict[str, Any], str] = Field( 1422 ..., 1423 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1424 examples=[ 1425 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1426 "{{ config['status_mapping'] }}", 1427 ], 1428 title="Value Mapping", 1429 ) 1430 field_path: List[str] = Field( 1431 ..., 1432 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1433 examples=[ 1434 ["status"], 1435 ["data", "status"], 1436 ["data", "{{ config.name }}", "status"], 1437 ["data", "*", "status"], 1438 ], 1439 title="Field Path", 1440 ) 1441 1442 1443class ConfigRemoveFields(BaseModel): 1444 type: Literal["ConfigRemoveFields"] 1445 field_pointers: List[List[str]] = Field( 1446 ..., 1447 description="A list of field pointers to be removed from the config.", 1448 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1449 title="Field Pointers", 1450 ) 1451 condition: Optional[str] = Field( 1452 "", 1453 description="Fields will be removed if expression is evaluated to True.", 1454 examples=[ 1455 "{{ config['environemnt'] == 'sandbox' }}", 1456 "{{ property is integer }}", 1457 "{{ property|length > 5 }}", 1458 "{{ property == 'some_string_to_match' }}", 1459 ], 1460 ) 1461 1462 1463class CustomConfigTransformation(BaseModel): 1464 type: Literal["CustomConfigTransformation"] 1465 class_name: str = Field( 1466 ..., 1467 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1468 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1469 ) 1470 parameters: Optional[Dict[str, Any]] = Field( 1471 None, 1472 alias="$parameters", 1473 description="Additional parameters to be passed to the custom config transformation.", 1474 ) 1475 1476 1477class AddedFieldDefinition(BaseModel): 1478 type: Literal["AddedFieldDefinition"] 1479 path: List[str] = Field( 1480 ..., 1481 description="List of strings defining the path where to add the value on the record.", 1482 examples=[["segment_id"], ["metadata", "segment_id"]], 1483 title="Path", 1484 ) 1485 value: str = Field( 1486 ..., 1487 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1488 examples=[ 1489 "{{ record['updates'] }}", 1490 "{{ record['MetaData']['LastUpdatedTime'] }}", 1491 "{{ stream_partition['segment_id'] }}", 1492 ], 1493 title="Value", 1494 ) 1495 value_type: Optional[ValueType] = Field( 1496 None, 1497 description="Type of the value. If not specified, the type will be inferred from the value.", 1498 title="Value Type", 1499 ) 1500 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1501 1502 1503class AddFields(BaseModel): 1504 type: Literal["AddFields"] 1505 fields: List[AddedFieldDefinition] = Field( 1506 ..., 1507 description="List of transformations (path and corresponding value) that will be added to the record.", 1508 title="Fields", 1509 ) 1510 condition: Optional[str] = Field( 1511 "", 1512 description="Fields will be added if expression is evaluated to True.", 1513 examples=[ 1514 "{{ property|string == '' }}", 1515 "{{ property is integer }}", 1516 "{{ property|length > 5 }}", 1517 "{{ property == 'some_string_to_match' }}", 1518 ], 1519 ) 1520 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1521 1522 1523class ApiKeyAuthenticator(BaseModel): 1524 type: Literal["ApiKeyAuthenticator"] 1525 api_token: Optional[str] = Field( 1526 None, 1527 description="The API key to inject in the request. Fill it in the user inputs.", 1528 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1529 title="API Key", 1530 ) 1531 header: Optional[str] = Field( 1532 None, 1533 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1534 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1535 title="Header Name", 1536 ) 1537 inject_into: Optional[RequestOption] = Field( 1538 None, 1539 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1540 examples=[ 1541 {"inject_into": "header", "field_name": "Authorization"}, 1542 {"inject_into": "request_parameter", "field_name": "authKey"}, 1543 ], 1544 title="Inject API Key Into Outgoing HTTP Request", 1545 ) 1546 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1547 1548 1549class AuthFlow(BaseModel): 1550 auth_flow_type: Optional[AuthFlowType] = Field( 1551 None, description="The type of auth to use", title="Auth flow type" 1552 ) 1553 predicate_key: Optional[List[str]] = Field( 1554 None, 1555 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1556 examples=[["credentials", "auth_type"]], 1557 title="Predicate key", 1558 ) 1559 predicate_value: Optional[str] = Field( 1560 None, 1561 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1562 examples=["Oauth"], 1563 title="Predicate value", 1564 ) 1565 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1566 1567 1568class CheckStream(BaseModel): 1569 type: Literal["CheckStream"] 1570 stream_names: Optional[List[str]] = Field( 1571 None, 1572 description="Names of the streams to try reading from when running a check operation.", 1573 examples=[["users"], ["users", "contacts"]], 1574 title="Stream Names", 1575 ) 1576 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1577 1578 1579class IncrementingCountCursor(BaseModel): 1580 type: Literal["IncrementingCountCursor"] 1581 cursor_field: str = Field( 1582 ..., 1583 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1584 examples=["created_at", "{{ config['record_cursor'] }}"], 1585 title="Cursor Field", 1586 ) 1587 start_value: Optional[Union[str, int]] = Field( 1588 None, 1589 description="The value that determines the earliest record that should be synced.", 1590 examples=[0, "{{ config['start_value'] }}"], 1591 title="Start Value", 1592 ) 1593 start_value_option: Optional[RequestOption] = Field( 1594 None, 1595 description="Optionally configures how the start value will be sent in requests to the source API.", 1596 title="Inject Start Value Into Outgoing HTTP Request", 1597 ) 1598 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1599 1600 1601class DatetimeBasedCursor(BaseModel): 1602 type: Literal["DatetimeBasedCursor"] 1603 clamping: Optional[Clamping] = Field( 1604 None, 1605 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1606 title="Date Range Clamping", 1607 ) 1608 cursor_field: str = Field( 1609 ..., 1610 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1611 examples=["created_at", "{{ config['record_cursor'] }}"], 1612 title="Cursor Field", 1613 ) 1614 cursor_datetime_formats: Optional[List[str]] = Field( 1615 None, 1616 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1617 examples=[ 1618 "%Y-%m-%d", 1619 "%Y-%m-%d %H:%M:%S", 1620 "%Y-%m-%dT%H:%M:%S", 1621 "%Y-%m-%dT%H:%M:%SZ", 1622 "%Y-%m-%dT%H:%M:%S%z", 1623 "%Y-%m-%dT%H:%M:%S.%fZ", 1624 "%Y-%m-%dT%H:%M:%S.%f%z", 1625 "%Y-%m-%d %H:%M:%S.%f+00:00", 1626 "%s", 1627 "%ms", 1628 ], 1629 title="Cursor Datetime Formats", 1630 ) 1631 start_datetime: Union[MinMaxDatetime, str] = Field( 1632 ..., 1633 description="The datetime that determines the earliest record that should be synced.", 1634 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1635 title="Start Datetime", 1636 ) 1637 start_time_option: Optional[RequestOption] = Field( 1638 None, 1639 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1640 title="Inject Start Time Into Outgoing HTTP Request", 1641 ) 1642 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1643 None, 1644 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1645 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1646 title="End Datetime", 1647 ) 1648 end_time_option: Optional[RequestOption] = Field( 1649 None, 1650 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1651 title="Inject End Time Into Outgoing HTTP Request", 1652 ) 1653 datetime_format: str = Field( 1654 ..., 1655 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1656 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1657 title="Outgoing Datetime Format", 1658 ) 1659 cursor_granularity: Optional[str] = Field( 1660 None, 1661 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1662 examples=["PT1S"], 1663 title="Cursor Granularity", 1664 ) 1665 is_data_feed: Optional[bool] = Field( 1666 None, 1667 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1668 title="Data Feed API", 1669 ) 1670 is_client_side_incremental: Optional[bool] = Field( 1671 None, 1672 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1673 title="Client-side Incremental Filtering", 1674 ) 1675 is_compare_strictly: Optional[bool] = Field( 1676 False, 1677 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1678 title="Strict Start-End Time Comparison", 1679 ) 1680 global_substream_cursor: Optional[bool] = Field( 1681 False, 1682 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1683 title="Global Substream Cursor", 1684 ) 1685 lookback_window: Optional[str] = Field( 1686 None, 1687 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1688 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1689 title="Lookback Window", 1690 ) 1691 partition_field_end: Optional[str] = Field( 1692 None, 1693 description="Name of the partition start time field.", 1694 examples=["ending_time"], 1695 title="Partition Field End", 1696 ) 1697 partition_field_start: Optional[str] = Field( 1698 None, 1699 description="Name of the partition end time field.", 1700 examples=["starting_time"], 1701 title="Partition Field Start", 1702 ) 1703 step: Optional[str] = Field( 1704 None, 1705 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1706 examples=["P1W", "{{ config['step_increment'] }}"], 1707 title="Step", 1708 ) 1709 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1710 1711 1712class JwtAuthenticator(BaseModel): 1713 type: Literal["JwtAuthenticator"] 1714 secret_key: str = Field( 1715 ..., 1716 description="Secret used to sign the JSON web token.", 1717 examples=["{{ config['secret_key'] }}"], 1718 title="Secret Key", 1719 ) 1720 base64_encode_secret_key: Optional[bool] = Field( 1721 False, 1722 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1723 title="Base64-encode Secret Key", 1724 ) 1725 algorithm: Algorithm = Field( 1726 ..., 1727 description="Algorithm used to sign the JSON web token.", 1728 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1729 title="Algorithm", 1730 ) 1731 token_duration: Optional[int] = Field( 1732 1200, 1733 description="The amount of time in seconds a JWT token can be valid after being issued.", 1734 examples=[1200, 3600], 1735 title="Token Duration", 1736 ) 1737 header_prefix: Optional[str] = Field( 1738 None, 1739 description="The prefix to be used within the Authentication header.", 1740 examples=["Bearer", "Basic"], 1741 title="Header Prefix", 1742 ) 1743 jwt_headers: Optional[JwtHeaders] = Field( 1744 None, 1745 description="JWT headers used when signing JSON web token.", 1746 title="JWT Headers", 1747 ) 1748 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1749 None, 1750 description="Additional headers to be included with the JWT headers object.", 1751 title="Additional JWT Headers", 1752 ) 1753 jwt_payload: Optional[JwtPayload] = Field( 1754 None, 1755 description="JWT Payload used when signing JSON web token.", 1756 title="JWT Payload", 1757 ) 1758 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1759 None, 1760 description="Additional properties to be added to the JWT payload.", 1761 title="Additional JWT Payload Properties", 1762 ) 1763 passphrase: Optional[str] = Field( 1764 None, 1765 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1766 examples=["{{ config['passphrase'] }}"], 1767 title="Passphrase", 1768 ) 1769 request_option: Optional[RequestOption] = Field( 1770 None, 1771 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1772 title="Request Option", 1773 ) 1774 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1775 1776 1777class OAuthAuthenticator(BaseModel): 1778 type: Literal["OAuthAuthenticator"] 1779 client_id_name: Optional[str] = Field( 1780 "client_id", 1781 description="The name of the property to use to refresh the `access_token`.", 1782 examples=["custom_app_id"], 1783 title="Client ID Property Name", 1784 ) 1785 client_id: Optional[str] = Field( 1786 None, 1787 description="The OAuth client ID. Fill it in the user inputs.", 1788 examples=[ 1789 "{{ config['client_id'] }}", 1790 "{{ config['credentials']['client_id }}", 1791 ], 1792 title="Client ID", 1793 ) 1794 client_secret_name: Optional[str] = Field( 1795 "client_secret", 1796 description="The name of the property to use to refresh the `access_token`.", 1797 examples=["custom_app_secret"], 1798 title="Client Secret Property Name", 1799 ) 1800 client_secret: Optional[str] = Field( 1801 None, 1802 description="The OAuth client secret. Fill it in the user inputs.", 1803 examples=[ 1804 "{{ config['client_secret'] }}", 1805 "{{ config['credentials']['client_secret }}", 1806 ], 1807 title="Client Secret", 1808 ) 1809 refresh_token_name: Optional[str] = Field( 1810 "refresh_token", 1811 description="The name of the property to use to refresh the `access_token`.", 1812 examples=["custom_app_refresh_value"], 1813 title="Refresh Token Property Name", 1814 ) 1815 refresh_token: Optional[str] = Field( 1816 None, 1817 description="Credential artifact used to get a new access token.", 1818 examples=[ 1819 "{{ config['refresh_token'] }}", 1820 "{{ config['credentials]['refresh_token'] }}", 1821 ], 1822 title="Refresh Token", 1823 ) 1824 token_refresh_endpoint: Optional[str] = Field( 1825 None, 1826 description="The full URL to call to obtain a new access token.", 1827 examples=["https://connect.squareup.com/oauth2/token"], 1828 title="Token Refresh Endpoint", 1829 ) 1830 access_token_name: Optional[str] = Field( 1831 "access_token", 1832 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1833 examples=["access_token"], 1834 title="Access Token Property Name", 1835 ) 1836 access_token_value: Optional[str] = Field( 1837 None, 1838 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1839 examples=["secret_access_token_value"], 1840 title="Access Token Value", 1841 ) 1842 expires_in_name: Optional[str] = Field( 1843 "expires_in", 1844 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1845 examples=["expires_in"], 1846 title="Token Expiry Property Name", 1847 ) 1848 grant_type_name: Optional[str] = Field( 1849 "grant_type", 1850 description="The name of the property to use to refresh the `access_token`.", 1851 examples=["custom_grant_type"], 1852 title="Grant Type Property Name", 1853 ) 1854 grant_type: Optional[str] = Field( 1855 "refresh_token", 1856 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1857 examples=["refresh_token", "client_credentials"], 1858 title="Grant Type", 1859 ) 1860 refresh_request_body: Optional[Dict[str, Any]] = Field( 1861 None, 1862 description="Body of the request sent to get a new access token.", 1863 examples=[ 1864 { 1865 "applicationId": "{{ config['application_id'] }}", 1866 "applicationSecret": "{{ config['application_secret'] }}", 1867 "token": "{{ config['token'] }}", 1868 } 1869 ], 1870 title="Refresh Request Body", 1871 ) 1872 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1873 None, 1874 description="Headers of the request sent to get a new access token.", 1875 examples=[ 1876 { 1877 "Authorization": "<AUTH_TOKEN>", 1878 "Content-Type": "application/x-www-form-urlencoded", 1879 } 1880 ], 1881 title="Refresh Request Headers", 1882 ) 1883 scopes: Optional[List[str]] = Field( 1884 None, 1885 description="List of scopes that should be granted to the access token.", 1886 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1887 title="Scopes", 1888 ) 1889 token_expiry_date: Optional[str] = Field( 1890 None, 1891 description="The access token expiry date.", 1892 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1893 title="Token Expiry Date", 1894 ) 1895 token_expiry_date_format: Optional[str] = Field( 1896 None, 1897 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1898 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1899 title="Token Expiry Date Format", 1900 ) 1901 refresh_token_error_status_codes: Optional[List[int]] = Field( 1902 None, 1903 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1904 examples=[[400, 500]], 1905 title="Refresh Token Error Status Codes", 1906 ) 1907 refresh_token_error_key: Optional[str] = Field( 1908 None, 1909 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1910 examples=["error"], 1911 title="Refresh Token Error Key", 1912 ) 1913 refresh_token_error_values: Optional[List[str]] = Field( 1914 None, 1915 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1916 examples=[["invalid_grant", "invalid_permissions"]], 1917 title="Refresh Token Error Values", 1918 ) 1919 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1920 None, 1921 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1922 title="Refresh Token Updater", 1923 ) 1924 profile_assertion: Optional[JwtAuthenticator] = Field( 1925 None, 1926 description="The authenticator being used to authenticate the client authenticator.", 1927 title="Profile Assertion", 1928 ) 1929 use_profile_assertion: Optional[bool] = Field( 1930 False, 1931 description="Enable using profile assertion as a flow for OAuth authorization.", 1932 title="Use Profile Assertion", 1933 ) 1934 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1935 1936 1937class FixedWindowCallRatePolicy(BaseModel): 1938 class Config: 1939 extra = Extra.allow 1940 1941 type: Literal["FixedWindowCallRatePolicy"] 1942 period: str = Field( 1943 ..., description="The time interval for the rate limit window.", title="Period" 1944 ) 1945 call_limit: int = Field( 1946 ..., 1947 description="The maximum number of calls allowed within the period.", 1948 title="Call Limit", 1949 ) 1950 matchers: List[HttpRequestRegexMatcher] = Field( 1951 ..., 1952 description="List of matchers that define which requests this policy applies to.", 1953 title="Matchers", 1954 ) 1955 1956 1957class MovingWindowCallRatePolicy(BaseModel): 1958 class Config: 1959 extra = Extra.allow 1960 1961 type: Literal["MovingWindowCallRatePolicy"] 1962 rates: List[Rate] = Field( 1963 ..., 1964 description="List of rates that define the call limits for different time intervals.", 1965 title="Rates", 1966 ) 1967 matchers: List[HttpRequestRegexMatcher] = Field( 1968 ..., 1969 description="List of matchers that define which requests this policy applies to.", 1970 title="Matchers", 1971 ) 1972 1973 1974class UnlimitedCallRatePolicy(BaseModel): 1975 class Config: 1976 extra = Extra.allow 1977 1978 type: Literal["UnlimitedCallRatePolicy"] 1979 matchers: List[HttpRequestRegexMatcher] = Field( 1980 ..., 1981 description="List of matchers that define which requests this policy applies to.", 1982 title="Matchers", 1983 ) 1984 1985 1986class DefaultErrorHandler(BaseModel): 1987 type: Literal["DefaultErrorHandler"] 1988 backoff_strategies: Optional[ 1989 List[ 1990 Union[ 1991 ConstantBackoffStrategy, 1992 ExponentialBackoffStrategy, 1993 WaitTimeFromHeader, 1994 WaitUntilTimeFromHeader, 1995 CustomBackoffStrategy, 1996 ] 1997 ] 1998 ] = Field( 1999 None, 2000 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2001 title="Backoff Strategies", 2002 ) 2003 max_retries: Optional[int] = Field( 2004 5, 2005 description="The maximum number of time to retry a retryable request before giving up and failing.", 2006 examples=[5, 0, 10], 2007 title="Max Retry Count", 2008 ) 2009 response_filters: Optional[List[HttpResponseFilter]] = Field( 2010 None, 2011 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2012 title="Response Filters", 2013 ) 2014 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2015 2016 2017class DefaultPaginator(BaseModel): 2018 type: Literal["DefaultPaginator"] 2019 pagination_strategy: Union[ 2020 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2021 ] = Field( 2022 ..., 2023 description="Strategy defining how records are paginated.", 2024 title="Pagination Strategy", 2025 ) 2026 page_size_option: Optional[RequestOption] = Field( 2027 None, title="Inject Page Size Into Outgoing HTTP Request" 2028 ) 2029 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2030 None, 2031 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2032 title="Inject Page Token Into Outgoing HTTP Request", 2033 ) 2034 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2035 2036 2037class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2038 type: Literal["ApiKey"] 2039 inject_into: RequestOption = Field( 2040 ..., 2041 description="Configure how the API Key will be sent in requests to the source API.", 2042 examples=[ 2043 {"inject_into": "header", "field_name": "Authorization"}, 2044 {"inject_into": "request_parameter", "field_name": "authKey"}, 2045 ], 2046 title="Inject API Key Into Outgoing HTTP Request", 2047 ) 2048 2049 2050class JsonSchemaPropertySelector(BaseModel): 2051 type: Literal["JsonSchemaPropertySelector"] 2052 transformations: Optional[ 2053 List[ 2054 Union[ 2055 AddFields, 2056 RemoveFields, 2057 KeysToLower, 2058 KeysToSnakeCase, 2059 FlattenFields, 2060 DpathFlattenFields, 2061 KeysReplace, 2062 CustomTransformation, 2063 ] 2064 ] 2065 ] = Field( 2066 None, 2067 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2068 title="Transformations", 2069 ) 2070 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2071 2072 2073class ListPartitionRouter(BaseModel): 2074 type: Literal["ListPartitionRouter"] 2075 cursor_field: str = Field( 2076 ..., 2077 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2078 examples=["section", "{{ config['section_key'] }}"], 2079 title="Current Partition Value Identifier", 2080 ) 2081 values: Union[str, List[str]] = Field( 2082 ..., 2083 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2084 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2085 title="Partition Values", 2086 ) 2087 request_option: Optional[RequestOption] = Field( 2088 None, 2089 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2090 title="Inject Partition Value Into Outgoing HTTP Request", 2091 ) 2092 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2093 2094 2095class RecordSelector(BaseModel): 2096 type: Literal["RecordSelector"] 2097 extractor: Union[DpathExtractor, CustomRecordExtractor] 2098 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2099 None, 2100 description="Responsible for filtering records to be emitted by the Source.", 2101 title="Record Filter", 2102 ) 2103 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2104 None, 2105 description="Responsible for normalization according to the schema.", 2106 title="Schema Normalization", 2107 ) 2108 transform_before_filtering: Optional[bool] = Field( 2109 None, 2110 description="If true, transformation will be applied before record filtering.", 2111 title="Transform Before Filtering", 2112 ) 2113 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2114 2115 2116class PaginationReset(BaseModel): 2117 type: Literal["PaginationReset"] 2118 action: Action1 2119 limits: Optional[PaginationResetLimits] = None 2120 2121 2122class GzipDecoder(BaseModel): 2123 type: Literal["GzipDecoder"] 2124 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2125 2126 2127class RequestBodyGraphQL(BaseModel): 2128 type: Literal["RequestBodyGraphQL"] 2129 value: RequestBodyGraphQlQuery 2130 2131 2132class DpathValidator(BaseModel): 2133 type: Literal["DpathValidator"] 2134 field_path: List[str] = Field( 2135 ..., 2136 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2137 examples=[ 2138 ["data"], 2139 ["data", "records"], 2140 ["data", "{{ parameters.name }}"], 2141 ["data", "*", "record"], 2142 ], 2143 title="Field Path", 2144 ) 2145 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2146 ..., 2147 description="The condition that the specified config value will be evaluated against", 2148 title="Validation Strategy", 2149 ) 2150 2151 2152class PredicateValidator(BaseModel): 2153 type: Literal["PredicateValidator"] 2154 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2155 ..., 2156 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2157 examples=[ 2158 "test-value", 2159 "{{ config['api_version'] }}", 2160 "{{ config['tenant_id'] }}", 2161 123, 2162 ], 2163 title="Value", 2164 ) 2165 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2166 ..., 2167 description="The validation strategy to apply to the value.", 2168 title="Validation Strategy", 2169 ) 2170 2171 2172class ConfigAddFields(BaseModel): 2173 type: Literal["ConfigAddFields"] 2174 fields: List[AddedFieldDefinition] = Field( 2175 ..., 2176 description="A list of transformations (path and corresponding value) that will be added to the config.", 2177 title="Fields", 2178 ) 2179 condition: Optional[str] = Field( 2180 "", 2181 description="Fields will be added if expression is evaluated to True.", 2182 examples=[ 2183 "{{ config['environemnt'] == 'sandbox' }}", 2184 "{{ property is integer }}", 2185 "{{ property|length > 5 }}", 2186 "{{ property == 'some_string_to_match' }}", 2187 ], 2188 ) 2189 2190 2191class CompositeErrorHandler(BaseModel): 2192 type: Literal["CompositeErrorHandler"] 2193 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2194 Field( 2195 ..., 2196 description="List of error handlers to iterate on to determine how to handle a failed response.", 2197 title="Error Handlers", 2198 ) 2199 ) 2200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2201 2202 2203class HTTPAPIBudget(BaseModel): 2204 class Config: 2205 extra = Extra.allow 2206 2207 type: Literal["HTTPAPIBudget"] 2208 policies: List[ 2209 Union[ 2210 FixedWindowCallRatePolicy, 2211 MovingWindowCallRatePolicy, 2212 UnlimitedCallRatePolicy, 2213 ] 2214 ] = Field( 2215 ..., 2216 description="List of call rate policies that define how many calls are allowed.", 2217 title="Policies", 2218 ) 2219 ratelimit_reset_header: Optional[str] = Field( 2220 "ratelimit-reset", 2221 description="The HTTP response header name that indicates when the rate limit resets.", 2222 title="Rate Limit Reset Header", 2223 ) 2224 ratelimit_remaining_header: Optional[str] = Field( 2225 "ratelimit-remaining", 2226 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2227 title="Rate Limit Remaining Header", 2228 ) 2229 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2230 [429], 2231 description="List of HTTP status codes that indicate a rate limit has been hit.", 2232 title="Status Codes for Rate Limit Hit", 2233 ) 2234 2235 2236class ZipfileDecoder(BaseModel): 2237 class Config: 2238 extra = Extra.allow 2239 2240 type: Literal["ZipfileDecoder"] 2241 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2242 ..., 2243 description="Parser to parse the decompressed data from the zipfile(s).", 2244 title="Parser", 2245 ) 2246 2247 2248class ConfigMigration(BaseModel): 2249 type: Literal["ConfigMigration"] 2250 description: Optional[str] = Field( 2251 None, description="The description/purpose of the config migration." 2252 ) 2253 transformations: List[ 2254 Union[ 2255 ConfigRemapField, 2256 ConfigAddFields, 2257 ConfigRemoveFields, 2258 CustomConfigTransformation, 2259 ] 2260 ] = Field( 2261 ..., 2262 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2263 title="Transformations", 2264 ) 2265 2266 2267class ConfigNormalizationRules(BaseModel): 2268 class Config: 2269 extra = Extra.forbid 2270 2271 type: Literal["ConfigNormalizationRules"] 2272 config_migrations: Optional[List[ConfigMigration]] = Field( 2273 [], 2274 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2275 title="Config Migrations", 2276 ) 2277 transformations: Optional[ 2278 List[ 2279 Union[ 2280 ConfigRemapField, 2281 ConfigAddFields, 2282 ConfigRemoveFields, 2283 CustomConfigTransformation, 2284 ] 2285 ] 2286 ] = Field( 2287 [], 2288 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2289 title="Transformations", 2290 ) 2291 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2292 [], 2293 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2294 title="Validations", 2295 ) 2296 2297 2298class Spec(BaseModel): 2299 type: Literal["Spec"] 2300 connection_specification: Dict[str, Any] = Field( 2301 ..., 2302 description="A connection specification describing how a the connector can be configured.", 2303 title="Connection Specification", 2304 ) 2305 documentation_url: Optional[str] = Field( 2306 None, 2307 description="URL of the connector's documentation page.", 2308 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2309 title="Documentation URL", 2310 ) 2311 advanced_auth: Optional[AuthFlow] = Field( 2312 None, 2313 description="Advanced specification for configuring the authentication flow.", 2314 title="Advanced Auth", 2315 ) 2316 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2317 None, title="Config Normalization Rules" 2318 ) 2319 2320 2321class DeclarativeSource1(BaseModel): 2322 class Config: 2323 extra = Extra.forbid 2324 2325 type: Literal["DeclarativeSource"] 2326 check: Union[CheckStream, CheckDynamicStream] 2327 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2328 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2329 version: str = Field( 2330 ..., 2331 description="The version of the Airbyte CDK used to build and test the source.", 2332 ) 2333 schemas: Optional[Schemas] = None 2334 definitions: Optional[Dict[str, Any]] = None 2335 spec: Optional[Spec] = None 2336 concurrency_level: Optional[ConcurrencyLevel] = None 2337 api_budget: Optional[HTTPAPIBudget] = None 2338 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2339 None, 2340 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2341 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2342 title="Maximum Concurrent Asynchronous Jobs", 2343 ) 2344 metadata: Optional[Dict[str, Any]] = Field( 2345 None, 2346 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2347 ) 2348 description: Optional[str] = Field( 2349 None, 2350 description="A description of the connector. It will be presented on the Source documentation page.", 2351 ) 2352 2353 2354class DeclarativeSource2(BaseModel): 2355 class Config: 2356 extra = Extra.forbid 2357 2358 type: Literal["DeclarativeSource"] 2359 check: Union[CheckStream, CheckDynamicStream] 2360 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2361 None 2362 ) 2363 dynamic_streams: List[DynamicDeclarativeStream] 2364 version: str = Field( 2365 ..., 2366 description="The version of the Airbyte CDK used to build and test the source.", 2367 ) 2368 schemas: Optional[Schemas] = None 2369 definitions: Optional[Dict[str, Any]] = None 2370 spec: Optional[Spec] = None 2371 concurrency_level: Optional[ConcurrencyLevel] = None 2372 api_budget: Optional[HTTPAPIBudget] = None 2373 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2374 None, 2375 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2376 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2377 title="Maximum Concurrent Asynchronous Jobs", 2378 ) 2379 metadata: Optional[Dict[str, Any]] = Field( 2380 None, 2381 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2382 ) 2383 description: Optional[str] = Field( 2384 None, 2385 description="A description of the connector. It will be presented on the Source documentation page.", 2386 ) 2387 2388 2389class DeclarativeSource(BaseModel): 2390 class Config: 2391 extra = Extra.forbid 2392 2393 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2394 ..., 2395 description="An API source that extracts data according to its declarative components.", 2396 title="DeclarativeSource", 2397 ) 2398 2399 2400class SelectiveAuthenticator(BaseModel): 2401 class Config: 2402 extra = Extra.allow 2403 2404 type: Literal["SelectiveAuthenticator"] 2405 authenticator_selection_path: List[str] = Field( 2406 ..., 2407 description="Path of the field in config with selected authenticator name", 2408 examples=[["auth"], ["auth", "type"]], 2409 title="Authenticator Selection Path", 2410 ) 2411 authenticators: Dict[ 2412 str, 2413 Union[ 2414 ApiKeyAuthenticator, 2415 BasicHttpAuthenticator, 2416 BearerAuthenticator, 2417 OAuthAuthenticator, 2418 JwtAuthenticator, 2419 SessionTokenAuthenticator, 2420 LegacySessionTokenAuthenticator, 2421 CustomAuthenticator, 2422 NoAuth, 2423 ], 2424 ] = Field( 2425 ..., 2426 description="Authenticators to select from.", 2427 examples=[ 2428 { 2429 "authenticators": { 2430 "token": "#/definitions/ApiKeyAuthenticator", 2431 "oauth": "#/definitions/OAuthAuthenticator", 2432 "jwt": "#/definitions/JwtAuthenticator", 2433 } 2434 } 2435 ], 2436 title="Authenticators", 2437 ) 2438 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2439 2440 2441class ConditionalStreams(BaseModel): 2442 type: Literal["ConditionalStreams"] 2443 condition: str = Field( 2444 ..., 2445 description="Condition that will be evaluated to determine if a set of streams should be available.", 2446 examples=["{{ config['is_sandbox'] }}"], 2447 title="Condition", 2448 ) 2449 streams: List[DeclarativeStream] = Field( 2450 ..., 2451 description="Streams that will be used during an operation based on the condition.", 2452 title="Streams", 2453 ) 2454 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2455 2456 2457class FileUploader(BaseModel): 2458 type: Literal["FileUploader"] 2459 requester: Union[HttpRequester, CustomRequester] = Field( 2460 ..., 2461 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2462 ) 2463 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2464 ..., 2465 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2466 ) 2467 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2468 None, 2469 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2470 ) 2471 filename_extractor: Optional[str] = Field( 2472 None, 2473 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2474 examples=[ 2475 "{{ record.id }}/{{ record.file_name }}/", 2476 "{{ record.id }}_{{ record.file_name }}/", 2477 ], 2478 ) 2479 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2480 2481 2482class DeclarativeStream(BaseModel): 2483 class Config: 2484 extra = Extra.allow 2485 2486 type: Literal["DeclarativeStream"] 2487 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2488 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2489 ..., 2490 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2491 title="Retriever", 2492 ) 2493 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2494 None, 2495 description="Component used to fetch data incrementally based on a time field in the data.", 2496 title="Incremental Sync", 2497 ) 2498 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2499 schema_loader: Optional[ 2500 Union[ 2501 InlineSchemaLoader, 2502 DynamicSchemaLoader, 2503 JsonFileSchemaLoader, 2504 List[ 2505 Union[ 2506 InlineSchemaLoader, 2507 DynamicSchemaLoader, 2508 JsonFileSchemaLoader, 2509 CustomSchemaLoader, 2510 ] 2511 ], 2512 CustomSchemaLoader, 2513 ] 2514 ] = Field( 2515 None, 2516 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2517 title="Schema Loader", 2518 ) 2519 transformations: Optional[ 2520 List[ 2521 Union[ 2522 AddFields, 2523 RemoveFields, 2524 KeysToLower, 2525 KeysToSnakeCase, 2526 FlattenFields, 2527 DpathFlattenFields, 2528 KeysReplace, 2529 CustomTransformation, 2530 ] 2531 ] 2532 ] = Field( 2533 None, 2534 description="A list of transformations to be applied to each output record.", 2535 title="Transformations", 2536 ) 2537 state_migrations: Optional[ 2538 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2539 ] = Field( 2540 [], 2541 description="Array of state migrations to be applied on the input state", 2542 title="State Migrations", 2543 ) 2544 file_uploader: Optional[FileUploader] = Field( 2545 None, 2546 description="(experimental) Describes how to fetch a file", 2547 title="File Uploader", 2548 ) 2549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2550 2551 2552class SessionTokenAuthenticator(BaseModel): 2553 type: Literal["SessionTokenAuthenticator"] 2554 login_requester: HttpRequester = Field( 2555 ..., 2556 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2557 examples=[ 2558 { 2559 "type": "HttpRequester", 2560 "url_base": "https://my_api.com", 2561 "path": "/login", 2562 "authenticator": { 2563 "type": "BasicHttpAuthenticator", 2564 "username": "{{ config.username }}", 2565 "password": "{{ config.password }}", 2566 }, 2567 } 2568 ], 2569 title="Login Requester", 2570 ) 2571 session_token_path: List[str] = Field( 2572 ..., 2573 description="The path in the response body returned from the login requester to the session token.", 2574 examples=[["access_token"], ["result", "token"]], 2575 title="Session Token Path", 2576 ) 2577 expiration_duration: Optional[str] = Field( 2578 None, 2579 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2580 examples=["PT1H", "P1D"], 2581 title="Expiration Duration", 2582 ) 2583 request_authentication: Union[ 2584 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2585 ] = Field( 2586 ..., 2587 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2588 title="Data Request Authentication", 2589 ) 2590 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2591 None, description="Component used to decode the response.", title="Decoder" 2592 ) 2593 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2594 2595 2596class HttpRequester(BaseModelWithDeprecations): 2597 type: Literal["HttpRequester"] 2598 url_base: Optional[str] = Field( 2599 None, 2600 deprecated=True, 2601 deprecation_message="Use `url` field instead.", 2602 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2603 examples=[ 2604 "https://connect.squareup.com/v2", 2605 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2606 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2607 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2608 ], 2609 title="API Base URL", 2610 ) 2611 url: Optional[str] = Field( 2612 None, 2613 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2614 examples=[ 2615 "https://connect.squareup.com/v2", 2616 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2617 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2618 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2619 ], 2620 title="API Endpoint URL", 2621 ) 2622 path: Optional[str] = Field( 2623 None, 2624 deprecated=True, 2625 deprecation_message="Use `url` field instead.", 2626 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2627 examples=[ 2628 "/products", 2629 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2630 "/trades/{{ config['symbol_id'] }}/history", 2631 ], 2632 title="URL Path", 2633 ) 2634 http_method: Optional[HttpMethod] = Field( 2635 HttpMethod.GET, 2636 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2637 examples=["GET", "POST"], 2638 title="HTTP Method", 2639 ) 2640 authenticator: Optional[ 2641 Union[ 2642 ApiKeyAuthenticator, 2643 BasicHttpAuthenticator, 2644 BearerAuthenticator, 2645 OAuthAuthenticator, 2646 JwtAuthenticator, 2647 SessionTokenAuthenticator, 2648 SelectiveAuthenticator, 2649 CustomAuthenticator, 2650 NoAuth, 2651 LegacySessionTokenAuthenticator, 2652 ] 2653 ] = Field( 2654 None, 2655 description="Authentication method to use for requests sent to the API.", 2656 title="Authenticator", 2657 ) 2658 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2659 None, 2660 deprecated=True, 2661 deprecation_message="Use `query_properties` field instead.", 2662 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2663 title="Fetch Properties from Endpoint", 2664 ) 2665 query_properties: Optional[QueryProperties] = Field( 2666 None, 2667 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2668 title="Query Properties", 2669 ) 2670 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2671 None, 2672 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2673 examples=[ 2674 {"unit": "day"}, 2675 { 2676 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2677 }, 2678 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2679 {"sort_by[asc]": "updated_at"}, 2680 ], 2681 title="Query Parameters", 2682 ) 2683 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2684 None, 2685 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2686 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2687 title="Request Headers", 2688 ) 2689 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2690 None, 2691 deprecated=True, 2692 deprecation_message="Use `request_body` field instead.", 2693 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2694 examples=[ 2695 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2696 ], 2697 title="Request Body Payload (Non-JSON)", 2698 ) 2699 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2700 None, 2701 deprecated=True, 2702 deprecation_message="Use `request_body` field instead.", 2703 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2704 examples=[ 2705 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2706 {"key": "{{ config['value'] }}"}, 2707 {"sort": {"field": "updated_at", "order": "ascending"}}, 2708 ], 2709 title="Request Body JSON Payload", 2710 ) 2711 request_body: Optional[ 2712 Union[ 2713 RequestBodyPlainText, 2714 RequestBodyUrlEncodedForm, 2715 RequestBodyJsonObject, 2716 RequestBodyGraphQL, 2717 ] 2718 ] = Field( 2719 None, 2720 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2721 title="Request Body", 2722 ) 2723 error_handler: Optional[ 2724 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2725 ] = Field( 2726 None, 2727 description="Error handler component that defines how to handle errors.", 2728 title="Error Handler", 2729 ) 2730 use_cache: Optional[bool] = Field( 2731 False, 2732 description="Enables stream requests caching. This field is automatically set by the CDK.", 2733 title="Use Cache", 2734 ) 2735 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2736 2737 2738class DynamicSchemaLoader(BaseModel): 2739 type: Literal["DynamicSchemaLoader"] 2740 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2741 ..., 2742 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2743 title="Retriever", 2744 ) 2745 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2746 None, 2747 description="Responsible for filtering fields to be added to json schema.", 2748 title="Schema Filter", 2749 ) 2750 schema_transformations: Optional[ 2751 List[ 2752 Union[ 2753 AddFields, 2754 RemoveFields, 2755 KeysToLower, 2756 KeysToSnakeCase, 2757 FlattenFields, 2758 DpathFlattenFields, 2759 KeysReplace, 2760 CustomTransformation, 2761 ] 2762 ] 2763 ] = Field( 2764 None, 2765 description="A list of transformations to be applied to the schema.", 2766 title="Schema Transformations", 2767 ) 2768 schema_type_identifier: SchemaTypeIdentifier 2769 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2770 2771 2772class ParentStreamConfig(BaseModel): 2773 type: Literal["ParentStreamConfig"] 2774 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2775 ..., description="Reference to the parent stream.", title="Parent Stream" 2776 ) 2777 parent_key: str = Field( 2778 ..., 2779 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2780 examples=["id", "{{ config['parent_record_id'] }}"], 2781 title="Parent Key", 2782 ) 2783 partition_field: str = Field( 2784 ..., 2785 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2786 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2787 title="Current Parent Key Value Identifier", 2788 ) 2789 request_option: Optional[RequestOption] = Field( 2790 None, 2791 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2792 title="Request Option", 2793 ) 2794 incremental_dependency: Optional[bool] = Field( 2795 False, 2796 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2797 title="Incremental Dependency", 2798 ) 2799 lazy_read_pointer: Optional[List[str]] = Field( 2800 [], 2801 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2802 title="Lazy Read Pointer", 2803 ) 2804 extra_fields: Optional[List[List[str]]] = Field( 2805 None, 2806 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2807 title="Extra Fields", 2808 ) 2809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2810 2811 2812class PropertiesFromEndpoint(BaseModel): 2813 type: Literal["PropertiesFromEndpoint"] 2814 property_field_path: List[str] = Field( 2815 ..., 2816 description="Describes the path to the field that should be extracted", 2817 examples=[["name"]], 2818 ) 2819 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2820 ..., 2821 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2822 ) 2823 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2824 2825 2826class QueryProperties(BaseModel): 2827 type: Literal["QueryProperties"] 2828 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2829 ..., 2830 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2831 title="Property List", 2832 ) 2833 always_include_properties: Optional[List[str]] = Field( 2834 None, 2835 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2836 title="Always Include Properties", 2837 ) 2838 property_chunking: Optional[PropertyChunking] = Field( 2839 None, 2840 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2841 title="Property Chunking", 2842 ) 2843 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2844 None, 2845 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2846 title="Property Selector", 2847 ) 2848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2849 2850 2851class StateDelegatingStream(BaseModel): 2852 type: Literal["StateDelegatingStream"] 2853 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2854 full_refresh_stream: DeclarativeStream = Field( 2855 ..., 2856 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2857 title="Full Refresh Stream", 2858 ) 2859 incremental_stream: DeclarativeStream = Field( 2860 ..., 2861 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2862 title="Incremental Stream", 2863 ) 2864 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2865 2866 2867class SimpleRetriever(BaseModel): 2868 type: Literal["SimpleRetriever"] 2869 requester: Union[HttpRequester, CustomRequester] = Field( 2870 ..., 2871 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2872 ) 2873 decoder: Optional[ 2874 Union[ 2875 JsonDecoder, 2876 XmlDecoder, 2877 CsvDecoder, 2878 JsonlDecoder, 2879 GzipDecoder, 2880 IterableDecoder, 2881 ZipfileDecoder, 2882 CustomDecoder, 2883 ] 2884 ] = Field( 2885 None, 2886 description="Component decoding the response so records can be extracted.", 2887 title="HTTP Response Format", 2888 ) 2889 record_selector: RecordSelector = Field( 2890 ..., 2891 description="Component that describes how to extract records from a HTTP response.", 2892 ) 2893 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2894 None, 2895 description="Paginator component that describes how to navigate through the API's pages.", 2896 ) 2897 pagination_reset: Optional[PaginationReset] = Field( 2898 None, 2899 description="Describes what triggers pagination reset and how to handle it.", 2900 ) 2901 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2902 False, 2903 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2904 ) 2905 partition_router: Optional[ 2906 Union[ 2907 SubstreamPartitionRouter, 2908 ListPartitionRouter, 2909 GroupingPartitionRouter, 2910 CustomPartitionRouter, 2911 List[ 2912 Union[ 2913 SubstreamPartitionRouter, 2914 ListPartitionRouter, 2915 GroupingPartitionRouter, 2916 CustomPartitionRouter, 2917 ] 2918 ], 2919 ] 2920 ] = Field( 2921 None, 2922 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2923 title="Partition Router", 2924 ) 2925 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2926 2927 2928class AsyncRetriever(BaseModel): 2929 type: Literal["AsyncRetriever"] 2930 record_selector: RecordSelector = Field( 2931 ..., 2932 description="Component that describes how to extract records from a HTTP response.", 2933 ) 2934 status_mapping: AsyncJobStatusMap = Field( 2935 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2936 ) 2937 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2938 ..., description="Responsible for fetching the actual status of the async job." 2939 ) 2940 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2941 None, 2942 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2943 ) 2944 download_extractor: Optional[ 2945 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2946 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2947 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2948 ..., 2949 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2950 ) 2951 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2952 ..., 2953 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2954 ) 2955 polling_job_timeout: Optional[Union[int, str]] = Field( 2956 None, 2957 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2958 ) 2959 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2960 None, 2961 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2962 ) 2963 download_requester: Union[HttpRequester, CustomRequester] = Field( 2964 ..., 2965 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2966 ) 2967 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2968 None, 2969 description="Paginator component that describes how to navigate through the API's pages during download.", 2970 ) 2971 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2972 None, 2973 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2974 ) 2975 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2976 None, 2977 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2978 ) 2979 partition_router: Optional[ 2980 Union[ 2981 ListPartitionRouter, 2982 SubstreamPartitionRouter, 2983 GroupingPartitionRouter, 2984 CustomPartitionRouter, 2985 List[ 2986 Union[ 2987 ListPartitionRouter, 2988 SubstreamPartitionRouter, 2989 GroupingPartitionRouter, 2990 CustomPartitionRouter, 2991 ] 2992 ], 2993 ] 2994 ] = Field( 2995 [], 2996 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2997 title="Partition Router", 2998 ) 2999 decoder: Optional[ 3000 Union[ 3001 CsvDecoder, 3002 GzipDecoder, 3003 JsonDecoder, 3004 JsonlDecoder, 3005 IterableDecoder, 3006 XmlDecoder, 3007 ZipfileDecoder, 3008 CustomDecoder, 3009 ] 3010 ] = Field( 3011 None, 3012 description="Component decoding the response so records can be extracted.", 3013 title="HTTP Response Format", 3014 ) 3015 download_decoder: Optional[ 3016 Union[ 3017 CsvDecoder, 3018 GzipDecoder, 3019 JsonDecoder, 3020 JsonlDecoder, 3021 IterableDecoder, 3022 XmlDecoder, 3023 ZipfileDecoder, 3024 CustomDecoder, 3025 ] 3026 ] = Field( 3027 None, 3028 description="Component decoding the download response so records can be extracted.", 3029 title="Download HTTP Response Format", 3030 ) 3031 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3032 3033 3034class SubstreamPartitionRouter(BaseModel): 3035 type: Literal["SubstreamPartitionRouter"] 3036 parent_stream_configs: List[ParentStreamConfig] = Field( 3037 ..., 3038 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3039 title="Parent Stream Configs", 3040 ) 3041 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3042 3043 3044class GroupingPartitionRouter(BaseModel): 3045 type: Literal["GroupingPartitionRouter"] 3046 group_size: int = Field( 3047 ..., 3048 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3049 examples=[10, 50], 3050 title="Group Size", 3051 ) 3052 underlying_partition_router: Union[ 3053 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3054 ] = Field( 3055 ..., 3056 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3057 title="Underlying Partition Router", 3058 ) 3059 deduplicate: Optional[bool] = Field( 3060 True, 3061 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3062 title="Deduplicate Partitions", 3063 ) 3064 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3065 3066 3067class HttpComponentsResolver(BaseModel): 3068 type: Literal["HttpComponentsResolver"] 3069 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3070 ..., 3071 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3072 title="Retriever", 3073 ) 3074 components_mapping: List[ComponentMappingDefinition] 3075 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3076 3077 3078class DynamicDeclarativeStream(BaseModel): 3079 type: Literal["DynamicDeclarativeStream"] 3080 name: Optional[str] = Field( 3081 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3082 ) 3083 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3084 ..., description="Reference to the stream template.", title="Stream Template" 3085 ) 3086 components_resolver: Union[ 3087 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3088 ] = Field( 3089 ..., 3090 description="Component resolve and populates stream templates with components values.", 3091 title="Components Resolver", 3092 ) 3093 use_parent_parameters: Optional[bool] = Field( 3094 True, 3095 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3096 title="Use Parent Parameters", 3097 ) 3098 3099 3100ComplexFieldType.update_forward_refs() 3101GzipDecoder.update_forward_refs() 3102CompositeErrorHandler.update_forward_refs() 3103DeclarativeSource1.update_forward_refs() 3104DeclarativeSource2.update_forward_refs() 3105SelectiveAuthenticator.update_forward_refs() 3106ConditionalStreams.update_forward_refs() 3107FileUploader.update_forward_refs() 3108DeclarativeStream.update_forward_refs() 3109SessionTokenAuthenticator.update_forward_refs() 3110HttpRequester.update_forward_refs() 3111DynamicSchemaLoader.update_forward_refs() 3112ParentStreamConfig.update_forward_refs() 3113PropertiesFromEndpoint.update_forward_refs() 3114SimpleRetriever.update_forward_refs() 3115AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
50class DynamicStreamCheckConfig(BaseModel): 51 type: Literal["DynamicStreamCheckConfig"] 52 dynamic_stream_name: str = Field( 53 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 54 ) 55 stream_count: Optional[int] = Field( 56 0, 57 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 58 title="Stream Count", 59 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
62class CheckDynamicStream(BaseModel): 63 type: Literal["CheckDynamicStream"] 64 stream_count: int = Field( 65 ..., 66 description="Numbers of the streams to try reading from when running a check operation.", 67 title="Stream Count", 68 ) 69 use_check_availability: Optional[bool] = Field( 70 True, 71 description="Enables stream check availability. This field is automatically set by the CDK.", 72 title="Use Check Availability", 73 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
76class ConcurrencyLevel(BaseModel): 77 type: Optional[Literal["ConcurrencyLevel"]] = None 78 default_concurrency: Union[int, str] = Field( 79 ..., 80 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 81 examples=[10, "{{ config['num_workers'] or 10 }}"], 82 title="Default Concurrency", 83 ) 84 max_concurrency: Optional[int] = Field( 85 None, 86 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 87 examples=[20, 100], 88 title="Max Concurrency", 89 ) 90 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
93class ConstantBackoffStrategy(BaseModel): 94 type: Literal["ConstantBackoffStrategy"] 95 backoff_time_in_seconds: Union[float, str] = Field( 96 ..., 97 description="Backoff time in seconds.", 98 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 99 title="Backoff Time", 100 ) 101 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
104class CursorPagination(BaseModel): 105 type: Literal["CursorPagination"] 106 cursor_value: str = Field( 107 ..., 108 description="Value of the cursor defining the next page to fetch.", 109 examples=[ 110 "{{ headers.link.next.cursor }}", 111 "{{ last_record['key'] }}", 112 "{{ response['nextPage'] }}", 113 ], 114 title="Cursor Value", 115 ) 116 page_size: Optional[int] = Field( 117 None, 118 description="The number of records to include in each pages.", 119 examples=[100], 120 title="Page Size", 121 ) 122 stop_condition: Optional[str] = Field( 123 None, 124 description="Template string evaluating when to stop paginating.", 125 examples=[ 126 "{{ response.data.has_more is false }}", 127 "{{ 'next' not in headers['link'] }}", 128 ], 129 title="Stop Condition", 130 ) 131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
134class CustomAuthenticator(BaseModel): 135 class Config: 136 extra = Extra.allow 137 138 type: Literal["CustomAuthenticator"] 139 class_name: str = Field( 140 ..., 141 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 142 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 143 title="Class Name", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
148class CustomBackoffStrategy(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomBackoffStrategy"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.MyCustomBackoffStrategy"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
162class CustomErrorHandler(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomErrorHandler"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomErrorHandler"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
176class CustomPaginationStrategy(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomPaginationStrategy"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomPaginationStrategy"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
190class CustomRecordExtractor(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomRecordExtractor"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomRecordExtractor"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
204class CustomRecordFilter(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordFilter"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomCustomRecordFilter"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
218class CustomRequester(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRequester"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomRecordExtractor"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
232class CustomRetriever(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRetriever"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRetriever"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
246class CustomPartitionRouter(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomPartitionRouter"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomPartitionRouter"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
260class CustomSchemaLoader(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomSchemaLoader"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomSchemaLoader"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
274class CustomSchemaNormalization(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaNormalization"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 282 examples=[ 283 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 284 ], 285 title="Class Name", 286 ) 287 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
290class CustomStateMigration(BaseModel): 291 class Config: 292 extra = Extra.allow 293 294 type: Literal["CustomStateMigration"] 295 class_name: str = Field( 296 ..., 297 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 298 examples=["source_railz.components.MyCustomStateMigration"], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
304class CustomTransformation(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomTransformation"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomTransformation"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
335class Algorithm(Enum): 336 HS256 = "HS256" 337 HS384 = "HS384" 338 HS512 = "HS512" 339 ES256 = "ES256" 340 ES256K = "ES256K" 341 ES384 = "ES384" 342 ES512 = "ES512" 343 RS256 = "RS256" 344 RS384 = "RS384" 345 RS512 = "RS512" 346 PS256 = "PS256" 347 PS384 = "PS384" 348 PS512 = "PS512" 349 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
352class JwtHeaders(BaseModel): 353 class Config: 354 extra = Extra.forbid 355 356 kid: Optional[str] = Field( 357 None, 358 description="Private key ID for user account.", 359 examples=["{{ config['kid'] }}"], 360 title="Key Identifier", 361 ) 362 typ: Optional[str] = Field( 363 "JWT", 364 description="The media type of the complete JWT.", 365 examples=["JWT"], 366 title="Type", 367 ) 368 cty: Optional[str] = Field( 369 None, 370 description="Content type of JWT header.", 371 examples=["JWT"], 372 title="Content Type", 373 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
376class JwtPayload(BaseModel): 377 class Config: 378 extra = Extra.forbid 379 380 iss: Optional[str] = Field( 381 None, 382 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 383 examples=["{{ config['iss'] }}"], 384 title="Issuer", 385 ) 386 sub: Optional[str] = Field( 387 None, 388 description="The subject of the JWT. Commonly defined by the API.", 389 title="Subject", 390 ) 391 aud: Optional[str] = Field( 392 None, 393 description="The recipient that the JWT is intended for. Commonly defined by the API.", 394 examples=["appstoreconnect-v1"], 395 title="Audience", 396 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
399class RefreshTokenUpdater(BaseModel): 400 refresh_token_name: Optional[str] = Field( 401 "refresh_token", 402 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 403 examples=["refresh_token"], 404 title="Refresh Token Property Name", 405 ) 406 access_token_config_path: Optional[List[str]] = Field( 407 ["credentials", "access_token"], 408 description="Config path to the access token. Make sure the field actually exists in the config.", 409 examples=[["credentials", "access_token"], ["access_token"]], 410 title="Config Path To Access Token", 411 ) 412 refresh_token_config_path: Optional[List[str]] = Field( 413 ["credentials", "refresh_token"], 414 description="Config path to the access token. Make sure the field actually exists in the config.", 415 examples=[["credentials", "refresh_token"], ["refresh_token"]], 416 title="Config Path To Refresh Token", 417 ) 418 token_expiry_date_config_path: Optional[List[str]] = Field( 419 ["credentials", "token_expiry_date"], 420 description="Config path to the expiry date. Make sure actually exists in the config.", 421 examples=[["credentials", "token_expiry_date"]], 422 title="Config Path To Expiry Date", 423 ) 424 refresh_token_error_status_codes: Optional[List[int]] = Field( 425 [], 426 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 427 examples=[[400, 500]], 428 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 429 ) 430 refresh_token_error_key: Optional[str] = Field( 431 "", 432 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 433 examples=["error"], 434 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 435 ) 436 refresh_token_error_values: Optional[List[str]] = Field( 437 [], 438 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 439 examples=[["invalid_grant", "invalid_permissions"]], 440 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 441 )
class
Rate(pydantic.v1.main.BaseModel):
444class Rate(BaseModel): 445 class Config: 446 extra = Extra.allow 447 448 limit: Union[int, str] = Field( 449 ..., 450 description="The maximum number of calls allowed within the interval.", 451 title="Limit", 452 ) 453 interval: str = Field( 454 ..., 455 description="The time interval for the rate limit.", 456 examples=["PT1H", "P1D"], 457 title="Interval", 458 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
461class HttpRequestRegexMatcher(BaseModel): 462 class Config: 463 extra = Extra.allow 464 465 method: Optional[str] = Field( 466 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 467 ) 468 url_base: Optional[str] = Field( 469 None, 470 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 471 title="URL Base", 472 ) 473 url_path_pattern: Optional[str] = Field( 474 None, 475 description="A regular expression pattern to match the URL path.", 476 title="URL Path Pattern", 477 ) 478 params: Optional[Dict[str, Any]] = Field( 479 None, description="The query parameters to match.", title="Parameters" 480 ) 481 headers: Optional[Dict[str, Any]] = Field( 482 None, description="The headers to match.", title="Headers" 483 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
486class DpathExtractor(BaseModel): 487 type: Literal["DpathExtractor"] 488 field_path: List[str] = Field( 489 ..., 490 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 491 examples=[ 492 ["data"], 493 ["data", "records"], 494 ["data", "{{ parameters.name }}"], 495 ["data", "*", "record"], 496 ], 497 title="Field Path", 498 ) 499 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
507class ExponentialBackoffStrategy(BaseModel): 508 type: Literal["ExponentialBackoffStrategy"] 509 factor: Optional[Union[float, str]] = Field( 510 5, 511 description="Multiplicative constant applied on each retry.", 512 examples=[5, 5.5, "10"], 513 title="Factor", 514 ) 515 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
518class GroupByKeyMergeStrategy(BaseModel): 519 type: Literal["GroupByKeyMergeStrategy"] 520 key: Union[str, List[str]] = Field( 521 ..., 522 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 523 examples=["id", ["parent_id", "end_date"]], 524 title="Key", 525 ) 526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
538class Action(Enum): 539 SUCCESS = "SUCCESS" 540 FAIL = "FAIL" 541 RETRY = "RETRY" 542 IGNORE = "IGNORE" 543 RESET_PAGINATION = "RESET_PAGINATION" 544 RATE_LIMITED = "RATE_LIMITED"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
547class FailureType(Enum): 548 system_error = "system_error" 549 config_error = "config_error" 550 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
553class HttpResponseFilter(BaseModel): 554 type: Literal["HttpResponseFilter"] 555 action: Optional[Action] = Field( 556 None, 557 description="Action to execute if a response matches the filter.", 558 examples=[ 559 "SUCCESS", 560 "FAIL", 561 "RETRY", 562 "IGNORE", 563 "RESET_PAGINATION", 564 "RATE_LIMITED", 565 ], 566 title="Action", 567 ) 568 failure_type: Optional[FailureType] = Field( 569 None, 570 description="Failure type of traced exception if a response matches the filter.", 571 examples=["system_error", "config_error", "transient_error"], 572 title="Failure Type", 573 ) 574 error_message: Optional[str] = Field( 575 None, 576 description="Error Message to display if the response matches the filter.", 577 title="Error Message", 578 ) 579 error_message_contains: Optional[str] = Field( 580 None, 581 description="Match the response if its error message contains the substring.", 582 example=["This API operation is not enabled for this site"], 583 title="Error Message Substring", 584 ) 585 http_codes: Optional[List[int]] = Field( 586 None, 587 description="Match the response if its HTTP code is included in this list.", 588 examples=[[420, 429], [500]], 589 title="HTTP Codes", 590 unique_items=True, 591 ) 592 predicate: Optional[str] = Field( 593 None, 594 description="Match the response if the predicate evaluates to true.", 595 examples=[ 596 "{{ 'Too much requests' in response }}", 597 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 598 ], 599 title="Predicate", 600 ) 601 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
604class ComplexFieldType(BaseModel): 605 field_type: str 606 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
609class TypesMap(BaseModel): 610 target_type: Union[str, List[str], ComplexFieldType] 611 current_type: Union[str, List[str]] 612 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
615class SchemaTypeIdentifier(BaseModel): 616 type: Optional[Literal["SchemaTypeIdentifier"]] = None 617 schema_pointer: Optional[List[str]] = Field( 618 [], 619 description="List of nested fields defining the schema field path to extract. Defaults to [].", 620 title="Schema Path", 621 ) 622 key_pointer: List[str] = Field( 623 ..., 624 description="List of potentially nested fields describing the full path of the field key to extract.", 625 title="Key Path", 626 ) 627 type_pointer: Optional[List[str]] = Field( 628 None, 629 description="List of potentially nested fields describing the full path of the field type to extract.", 630 title="Type Path", 631 ) 632 types_mapping: Optional[List[TypesMap]] = None 633 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
636class InlineSchemaLoader(BaseModel): 637 type: Literal["InlineSchemaLoader"] 638 schema_: Optional[Dict[str, Any]] = Field( 639 None, 640 alias="schema", 641 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 642 title="Schema", 643 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
646class JsonFileSchemaLoader(BaseModel): 647 type: Literal["JsonFileSchemaLoader"] 648 file_path: Optional[str] = Field( 649 None, 650 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 651 example=["./schemas/users.json"], 652 title="File Path", 653 ) 654 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
675class FlattenFields(BaseModel): 676 type: Literal["FlattenFields"] 677 flatten_lists: Optional[bool] = Field( 678 True, 679 description="Whether to flatten lists or leave it as is. Default is True.", 680 title="Flatten Lists", 681 ) 682 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
685class KeyTransformation(BaseModel): 686 type: Literal["KeyTransformation"] 687 prefix: Optional[str] = Field( 688 None, 689 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 690 examples=["flattened_"], 691 title="Key Prefix", 692 ) 693 suffix: Optional[str] = Field( 694 None, 695 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 696 examples=["_flattened"], 697 title="Key Suffix", 698 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
701class DpathFlattenFields(BaseModel): 702 type: Literal["DpathFlattenFields"] 703 field_path: List[str] = Field( 704 ..., 705 description="A path to field that needs to be flattened.", 706 examples=[["data"], ["data", "*", "field"]], 707 title="Field Path", 708 ) 709 delete_origin_value: Optional[bool] = Field( 710 None, 711 description="Whether to delete the origin value or keep it. Default is False.", 712 title="Delete Origin Value", 713 ) 714 replace_record: Optional[bool] = Field( 715 None, 716 description="Whether to replace the origin record or not. Default is False.", 717 title="Replace Origin Record", 718 ) 719 key_transformation: Optional[KeyTransformation] = Field( 720 None, 721 description="Transformation for object keys. If not provided, original key will be used.", 722 title="Key transformation", 723 ) 724 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
727class KeysReplace(BaseModel): 728 type: Literal["KeysReplace"] 729 old: str = Field( 730 ..., 731 description="Old value to replace.", 732 examples=[ 733 " ", 734 "{{ record.id }}", 735 "{{ config['id'] }}", 736 "{{ stream_slice['id'] }}", 737 ], 738 title="Old value", 739 ) 740 new: str = Field( 741 ..., 742 description="New value to set.", 743 examples=[ 744 "_", 745 "{{ record.id }}", 746 "{{ config['id'] }}", 747 "{{ stream_slice['id'] }}", 748 ], 749 title="New value", 750 ) 751 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
762class CustomDecoder(BaseModel): 763 class Config: 764 extra = Extra.allow 765 766 type: Literal["CustomDecoder"] 767 class_name: str = Field( 768 ..., 769 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 770 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 771 title="Class Name", 772 ) 773 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
776class MinMaxDatetime(BaseModel): 777 type: Literal["MinMaxDatetime"] 778 datetime: str = Field( 779 ..., 780 description="Datetime value.", 781 examples=[ 782 "2021-01-01", 783 "2021-01-01T00:00:00Z", 784 "{{ config['start_time'] }}", 785 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 786 ], 787 title="Datetime", 788 ) 789 datetime_format: Optional[str] = Field( 790 "", 791 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 792 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 793 title="Datetime Format", 794 ) 795 max_datetime: Optional[str] = Field( 796 None, 797 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 798 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 799 title="Max Datetime", 800 ) 801 min_datetime: Optional[str] = Field( 802 None, 803 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 804 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 805 title="Min Datetime", 806 ) 807 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
827class OauthConnectorInputSpecification(BaseModel): 828 class Config: 829 extra = Extra.allow 830 831 consent_url: str = Field( 832 ..., 833 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 834 examples=[ 835 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 836 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 837 ], 838 title="Consent URL", 839 ) 840 scope: Optional[str] = Field( 841 None, 842 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 843 examples=["user:read user:read_orders workspaces:read"], 844 title="Scopes", 845 ) 846 access_token_url: str = Field( 847 ..., 848 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 849 examples=[ 850 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 851 ], 852 title="Access Token URL", 853 ) 854 access_token_headers: Optional[Dict[str, Any]] = Field( 855 None, 856 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 857 examples=[ 858 { 859 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 860 } 861 ], 862 title="Access Token Headers", 863 ) 864 access_token_params: Optional[Dict[str, Any]] = Field( 865 None, 866 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 867 examples=[ 868 { 869 "{{ auth_code_key }}": "{{ auth_code_value }}", 870 "{{ client_id_key }}": "{{ client_id_value }}", 871 "{{ client_secret_key }}": "{{ client_secret_value }}", 872 } 873 ], 874 title="Access Token Query Params (Json Encoded)", 875 ) 876 extract_output: Optional[List[str]] = Field( 877 None, 878 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 879 examples=[["access_token", "refresh_token", "other_field"]], 880 title="Extract Output", 881 ) 882 state: Optional[State] = Field( 883 None, 884 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 885 examples=[{"min": 7, "max": 128}], 886 title="Configurable State Query Param", 887 ) 888 client_id_key: Optional[str] = Field( 889 None, 890 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 891 examples=["my_custom_client_id_key_name"], 892 title="Client ID Key Override", 893 ) 894 client_secret_key: Optional[str] = Field( 895 None, 896 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 897 examples=["my_custom_client_secret_key_name"], 898 title="Client Secret Key Override", 899 ) 900 scope_key: Optional[str] = Field( 901 None, 902 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 903 examples=["my_custom_scope_key_key_name"], 904 title="Scopes Key Override", 905 ) 906 state_key: Optional[str] = Field( 907 None, 908 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 909 examples=["my_custom_state_key_key_name"], 910 title="State Key Override", 911 ) 912 auth_code_key: Optional[str] = Field( 913 None, 914 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 915 examples=["my_custom_auth_code_key_name"], 916 title="Auth Code Key Override", 917 ) 918 redirect_uri_key: Optional[str] = Field( 919 None, 920 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 921 examples=["my_custom_redirect_uri_key_name"], 922 title="Redirect URI Key Override", 923 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
926class OAuthConfigSpecification(BaseModel): 927 class Config: 928 extra = Extra.allow 929 930 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 931 None, 932 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 933 examples=[ 934 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 935 { 936 "app_id": { 937 "type": "string", 938 "path_in_connector_config": ["info", "app_id"], 939 } 940 }, 941 ], 942 title="OAuth user input", 943 ) 944 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 945 None, 946 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 947 title="DeclarativeOAuth Connector Specification", 948 ) 949 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 950 None, 951 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 952 examples=[ 953 { 954 "refresh_token": { 955 "type": "string,", 956 "path_in_connector_config": ["credentials", "refresh_token"], 957 } 958 } 959 ], 960 title="OAuth output specification", 961 ) 962 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 963 None, 964 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 965 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 966 title="OAuth input specification", 967 ) 968 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 969 None, 970 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 971 examples=[ 972 { 973 "client_id": { 974 "type": "string,", 975 "path_in_connector_config": ["credentials", "client_id"], 976 }, 977 "client_secret": { 978 "type": "string,", 979 "path_in_connector_config": ["credentials", "client_secret"], 980 }, 981 } 982 ], 983 title="OAuth server output specification", 984 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
987class OffsetIncrement(BaseModel): 988 type: Literal["OffsetIncrement"] 989 page_size: Optional[Union[int, str]] = Field( 990 None, 991 description="The number of records to include in each pages.", 992 examples=[100, "{{ config['page_size'] }}"], 993 title="Limit", 994 ) 995 inject_on_first_request: Optional[bool] = Field( 996 False, 997 description="Using the `offset` with value `0` during the first request", 998 title="Inject Offset on First Request", 999 ) 1000 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1003class PageIncrement(BaseModel): 1004 type: Literal["PageIncrement"] 1005 page_size: Optional[Union[int, str]] = Field( 1006 None, 1007 description="The number of records to include in each pages.", 1008 examples=[100, "100", "{{ config['page_size'] }}"], 1009 title="Page Size", 1010 ) 1011 start_from_page: Optional[int] = Field( 1012 0, 1013 description="Index of the first page to request.", 1014 examples=[0, 1], 1015 title="Start From Page", 1016 ) 1017 inject_on_first_request: Optional[bool] = Field( 1018 False, 1019 description="Using the `page number` with value defined by `start_from_page` during the first request", 1020 title="Inject Page Number on First Request", 1021 ) 1022 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1025class PrimaryKey(BaseModel): 1026 __root__: Union[str, List[str], List[List[str]]] = Field( 1027 ..., 1028 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1029 examples=["id", ["code", "type"]], 1030 title="Primary Key", 1031 )
class
PropertyLimitType(enum.Enum):
1034class PropertyLimitType(Enum): 1035 characters = "characters" 1036 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1039class PropertyChunking(BaseModel): 1040 type: Literal["PropertyChunking"] 1041 property_limit_type: PropertyLimitType = Field( 1042 ..., 1043 description="The type used to determine the maximum number of properties per chunk", 1044 title="Property Limit Type", 1045 ) 1046 property_limit: Optional[int] = Field( 1047 None, 1048 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1049 title="Property Limit", 1050 ) 1051 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1052 None, 1053 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1054 title="Record Merge Strategy", 1055 ) 1056 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1059class RecordFilter(BaseModel): 1060 type: Literal["RecordFilter"] 1061 condition: Optional[str] = Field( 1062 "", 1063 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1064 examples=[ 1065 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1066 "{{ record.status in ['active', 'expired'] }}", 1067 ], 1068 title="Condition", 1069 ) 1070 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1078class RemoveFields(BaseModel): 1079 type: Literal["RemoveFields"] 1080 condition: Optional[str] = Field( 1081 "", 1082 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1083 examples=[ 1084 "{{ property|string == '' }}", 1085 "{{ property is integer }}", 1086 "{{ property|length > 5 }}", 1087 "{{ property == 'some_string_to_match' }}", 1088 ], 1089 ) 1090 field_pointers: List[List[str]] = Field( 1091 ..., 1092 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1093 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1094 title="Field Paths", 1095 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1102class InjectInto(Enum): 1103 request_parameter = "request_parameter" 1104 header = "header" 1105 body_data = "body_data" 1106 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1109class RequestOption(BaseModel): 1110 type: Literal["RequestOption"] 1111 inject_into: InjectInto = Field( 1112 ..., 1113 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1114 examples=["request_parameter", "header", "body_data", "body_json"], 1115 title="Inject Into", 1116 ) 1117 field_name: Optional[str] = Field( 1118 None, 1119 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1120 examples=["segment_id"], 1121 title="Field Name", 1122 ) 1123 field_path: Optional[List[str]] = Field( 1124 None, 1125 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1126 examples=[["data", "viewer", "id"]], 1127 title="Field Path", 1128 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1138class LegacySessionTokenAuthenticator(BaseModel): 1139 type: Literal["LegacySessionTokenAuthenticator"] 1140 header: str = Field( 1141 ..., 1142 description="The name of the session token header that will be injected in the request", 1143 examples=["X-Session"], 1144 title="Session Request Header", 1145 ) 1146 login_url: str = Field( 1147 ..., 1148 description="Path of the login URL (do not include the base URL)", 1149 examples=["session"], 1150 title="Login Path", 1151 ) 1152 session_token: Optional[str] = Field( 1153 None, 1154 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1155 example=["{{ config['session_token'] }}"], 1156 title="Session Token", 1157 ) 1158 session_token_response_key: str = Field( 1159 ..., 1160 description="Name of the key of the session token to be extracted from the response", 1161 examples=["id"], 1162 title="Response Token Response Key", 1163 ) 1164 username: Optional[str] = Field( 1165 None, 1166 description="Username used to authenticate and obtain a session token", 1167 examples=[" {{ config['username'] }}"], 1168 title="Username", 1169 ) 1170 password: Optional[str] = Field( 1171 "", 1172 description="Password used to authenticate and obtain a session token", 1173 examples=["{{ config['password'] }}", ""], 1174 title="Password", 1175 ) 1176 validate_session_url: str = Field( 1177 ..., 1178 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1179 examples=["user/current"], 1180 title="Validate Session Path", 1181 ) 1182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1210class ValueType(Enum): 1211 string = "string" 1212 number = "number" 1213 integer = "integer" 1214 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1217class WaitTimeFromHeader(BaseModel): 1218 type: Literal["WaitTimeFromHeader"] 1219 header: str = Field( 1220 ..., 1221 description="The name of the response header defining how long to wait before retrying.", 1222 examples=["Retry-After"], 1223 title="Response Header Name", 1224 ) 1225 regex: Optional[str] = Field( 1226 None, 1227 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1228 examples=["([-+]?\\d+)"], 1229 title="Extraction Regex", 1230 ) 1231 max_waiting_time_in_seconds: Optional[float] = Field( 1232 None, 1233 description="Given the value extracted from the header is greater than this value, stop the stream.", 1234 examples=[3600], 1235 title="Max Waiting Time in Seconds", 1236 ) 1237 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1240class WaitUntilTimeFromHeader(BaseModel): 1241 type: Literal["WaitUntilTimeFromHeader"] 1242 header: str = Field( 1243 ..., 1244 description="The name of the response header defining how long to wait before retrying.", 1245 examples=["wait_time"], 1246 title="Response Header", 1247 ) 1248 min_wait: Optional[Union[float, str]] = Field( 1249 None, 1250 description="Minimum time to wait before retrying.", 1251 examples=[10, "60"], 1252 title="Minimum Wait Time", 1253 ) 1254 regex: Optional[str] = Field( 1255 None, 1256 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1257 examples=["([-+]?\\d+)"], 1258 title="Extraction Regex", 1259 ) 1260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1263class ComponentMappingDefinition(BaseModel): 1264 type: Literal["ComponentMappingDefinition"] 1265 field_path: List[str] = Field( 1266 ..., 1267 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1268 examples=[ 1269 ["name"], 1270 ["retriever", "requester", "url"], 1271 ["retriever", "requester", "{{ components_values.field }}"], 1272 ["*", "**", "name"], 1273 ], 1274 title="Field Path", 1275 ) 1276 value: str = Field( 1277 ..., 1278 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1279 examples=[ 1280 "{{ components_values['updates'] }}", 1281 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1282 "{{ config['segment_id'] }}", 1283 "{{ stream_slice['parent_id'] }}", 1284 "{{ stream_slice['extra_fields']['name'] }}", 1285 ], 1286 title="Value", 1287 ) 1288 value_type: Optional[ValueType] = Field( 1289 None, 1290 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1291 title="Value Type", 1292 ) 1293 create_or_update: Optional[bool] = Field( 1294 False, 1295 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1296 title="Create or Update", 1297 ) 1298 condition: Optional[str] = Field( 1299 None, 1300 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1301 examples=[ 1302 "{{ components_values.get('cursor_field', None) }}", 1303 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1304 ], 1305 title="Condition", 1306 ) 1307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1310class StreamConfig(BaseModel): 1311 type: Literal["StreamConfig"] 1312 configs_pointer: List[str] = Field( 1313 ..., 1314 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1315 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1316 title="Configs Pointer", 1317 ) 1318 default_values: Optional[List[Dict[str, Any]]] = Field( 1319 None, 1320 description="A list of default values, each matching the structure expected from the parsed component value.", 1321 title="Default Values", 1322 ) 1323 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1326class ConfigComponentsResolver(BaseModel): 1327 type: Literal["ConfigComponentsResolver"] 1328 stream_config: Union[List[StreamConfig], StreamConfig] 1329 components_mapping: List[ComponentMappingDefinition] 1330 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1333class StreamParametersDefinition(BaseModel): 1334 type: Literal["StreamParametersDefinition"] 1335 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1336 ..., 1337 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1338 examples=[ 1339 [ 1340 { 1341 "name": "test stream", 1342 "$parameters": {"entity": "test entity"}, 1343 "primary_key": "test key", 1344 } 1345 ] 1346 ], 1347 title="Stream Parameters", 1348 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1351class ParametrizedComponentsResolver(BaseModel): 1352 type: Literal["ParametrizedComponentsResolver"] 1353 stream_parameters: StreamParametersDefinition 1354 components_mapping: List[ComponentMappingDefinition] 1355 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1380class ValidateAdheresToSchema(BaseModel): 1381 type: Literal["ValidateAdheresToSchema"] 1382 base_schema: Union[str, Dict[str, Any]] = Field( 1383 ..., 1384 description="The base JSON schema against which the user-provided schema will be validated.", 1385 examples=[ 1386 "{{ config['report_validation_schema'] }}", 1387 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1388 { 1389 "$schema": "http://json-schema.org/draft-07/schema#", 1390 "title": "Person", 1391 "type": "object", 1392 "properties": { 1393 "name": {"type": "string", "description": "The person's name"}, 1394 "age": { 1395 "type": "integer", 1396 "minimum": 0, 1397 "description": "The person's age", 1398 }, 1399 }, 1400 "required": ["name", "age"], 1401 }, 1402 ], 1403 title="Base JSON Schema", 1404 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1407class CustomValidationStrategy(BaseModel): 1408 class Config: 1409 extra = Extra.allow 1410 1411 type: Literal["CustomValidationStrategy"] 1412 class_name: str = Field( 1413 ..., 1414 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1415 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1416 title="Class Name", 1417 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1420class ConfigRemapField(BaseModel): 1421 type: Literal["ConfigRemapField"] 1422 map: Union[Dict[str, Any], str] = Field( 1423 ..., 1424 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1425 examples=[ 1426 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1427 "{{ config['status_mapping'] }}", 1428 ], 1429 title="Value Mapping", 1430 ) 1431 field_path: List[str] = Field( 1432 ..., 1433 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1434 examples=[ 1435 ["status"], 1436 ["data", "status"], 1437 ["data", "{{ config.name }}", "status"], 1438 ["data", "*", "status"], 1439 ], 1440 title="Field Path", 1441 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1444class ConfigRemoveFields(BaseModel): 1445 type: Literal["ConfigRemoveFields"] 1446 field_pointers: List[List[str]] = Field( 1447 ..., 1448 description="A list of field pointers to be removed from the config.", 1449 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1450 title="Field Pointers", 1451 ) 1452 condition: Optional[str] = Field( 1453 "", 1454 description="Fields will be removed if expression is evaluated to True.", 1455 examples=[ 1456 "{{ config['environemnt'] == 'sandbox' }}", 1457 "{{ property is integer }}", 1458 "{{ property|length > 5 }}", 1459 "{{ property == 'some_string_to_match' }}", 1460 ], 1461 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1464class CustomConfigTransformation(BaseModel): 1465 type: Literal["CustomConfigTransformation"] 1466 class_name: str = Field( 1467 ..., 1468 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1469 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1470 ) 1471 parameters: Optional[Dict[str, Any]] = Field( 1472 None, 1473 alias="$parameters", 1474 description="Additional parameters to be passed to the custom config transformation.", 1475 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1478class AddedFieldDefinition(BaseModel): 1479 type: Literal["AddedFieldDefinition"] 1480 path: List[str] = Field( 1481 ..., 1482 description="List of strings defining the path where to add the value on the record.", 1483 examples=[["segment_id"], ["metadata", "segment_id"]], 1484 title="Path", 1485 ) 1486 value: str = Field( 1487 ..., 1488 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1489 examples=[ 1490 "{{ record['updates'] }}", 1491 "{{ record['MetaData']['LastUpdatedTime'] }}", 1492 "{{ stream_partition['segment_id'] }}", 1493 ], 1494 title="Value", 1495 ) 1496 value_type: Optional[ValueType] = Field( 1497 None, 1498 description="Type of the value. If not specified, the type will be inferred from the value.", 1499 title="Value Type", 1500 ) 1501 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1504class AddFields(BaseModel): 1505 type: Literal["AddFields"] 1506 fields: List[AddedFieldDefinition] = Field( 1507 ..., 1508 description="List of transformations (path and corresponding value) that will be added to the record.", 1509 title="Fields", 1510 ) 1511 condition: Optional[str] = Field( 1512 "", 1513 description="Fields will be added if expression is evaluated to True.", 1514 examples=[ 1515 "{{ property|string == '' }}", 1516 "{{ property is integer }}", 1517 "{{ property|length > 5 }}", 1518 "{{ property == 'some_string_to_match' }}", 1519 ], 1520 ) 1521 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1524class ApiKeyAuthenticator(BaseModel): 1525 type: Literal["ApiKeyAuthenticator"] 1526 api_token: Optional[str] = Field( 1527 None, 1528 description="The API key to inject in the request. Fill it in the user inputs.", 1529 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1530 title="API Key", 1531 ) 1532 header: Optional[str] = Field( 1533 None, 1534 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1535 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1536 title="Header Name", 1537 ) 1538 inject_into: Optional[RequestOption] = Field( 1539 None, 1540 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1541 examples=[ 1542 {"inject_into": "header", "field_name": "Authorization"}, 1543 {"inject_into": "request_parameter", "field_name": "authKey"}, 1544 ], 1545 title="Inject API Key Into Outgoing HTTP Request", 1546 ) 1547 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1550class AuthFlow(BaseModel): 1551 auth_flow_type: Optional[AuthFlowType] = Field( 1552 None, description="The type of auth to use", title="Auth flow type" 1553 ) 1554 predicate_key: Optional[List[str]] = Field( 1555 None, 1556 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1557 examples=[["credentials", "auth_type"]], 1558 title="Predicate key", 1559 ) 1560 predicate_value: Optional[str] = Field( 1561 None, 1562 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1563 examples=["Oauth"], 1564 title="Predicate value", 1565 ) 1566 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1569class CheckStream(BaseModel): 1570 type: Literal["CheckStream"] 1571 stream_names: Optional[List[str]] = Field( 1572 None, 1573 description="Names of the streams to try reading from when running a check operation.", 1574 examples=[["users"], ["users", "contacts"]], 1575 title="Stream Names", 1576 ) 1577 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1580class IncrementingCountCursor(BaseModel): 1581 type: Literal["IncrementingCountCursor"] 1582 cursor_field: str = Field( 1583 ..., 1584 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1585 examples=["created_at", "{{ config['record_cursor'] }}"], 1586 title="Cursor Field", 1587 ) 1588 start_value: Optional[Union[str, int]] = Field( 1589 None, 1590 description="The value that determines the earliest record that should be synced.", 1591 examples=[0, "{{ config['start_value'] }}"], 1592 title="Start Value", 1593 ) 1594 start_value_option: Optional[RequestOption] = Field( 1595 None, 1596 description="Optionally configures how the start value will be sent in requests to the source API.", 1597 title="Inject Start Value Into Outgoing HTTP Request", 1598 ) 1599 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1602class DatetimeBasedCursor(BaseModel): 1603 type: Literal["DatetimeBasedCursor"] 1604 clamping: Optional[Clamping] = Field( 1605 None, 1606 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1607 title="Date Range Clamping", 1608 ) 1609 cursor_field: str = Field( 1610 ..., 1611 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1612 examples=["created_at", "{{ config['record_cursor'] }}"], 1613 title="Cursor Field", 1614 ) 1615 cursor_datetime_formats: Optional[List[str]] = Field( 1616 None, 1617 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1618 examples=[ 1619 "%Y-%m-%d", 1620 "%Y-%m-%d %H:%M:%S", 1621 "%Y-%m-%dT%H:%M:%S", 1622 "%Y-%m-%dT%H:%M:%SZ", 1623 "%Y-%m-%dT%H:%M:%S%z", 1624 "%Y-%m-%dT%H:%M:%S.%fZ", 1625 "%Y-%m-%dT%H:%M:%S.%f%z", 1626 "%Y-%m-%d %H:%M:%S.%f+00:00", 1627 "%s", 1628 "%ms", 1629 ], 1630 title="Cursor Datetime Formats", 1631 ) 1632 start_datetime: Union[MinMaxDatetime, str] = Field( 1633 ..., 1634 description="The datetime that determines the earliest record that should be synced.", 1635 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1636 title="Start Datetime", 1637 ) 1638 start_time_option: Optional[RequestOption] = Field( 1639 None, 1640 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1641 title="Inject Start Time Into Outgoing HTTP Request", 1642 ) 1643 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1644 None, 1645 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1646 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1647 title="End Datetime", 1648 ) 1649 end_time_option: Optional[RequestOption] = Field( 1650 None, 1651 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1652 title="Inject End Time Into Outgoing HTTP Request", 1653 ) 1654 datetime_format: str = Field( 1655 ..., 1656 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1657 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1658 title="Outgoing Datetime Format", 1659 ) 1660 cursor_granularity: Optional[str] = Field( 1661 None, 1662 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1663 examples=["PT1S"], 1664 title="Cursor Granularity", 1665 ) 1666 is_data_feed: Optional[bool] = Field( 1667 None, 1668 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1669 title="Data Feed API", 1670 ) 1671 is_client_side_incremental: Optional[bool] = Field( 1672 None, 1673 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1674 title="Client-side Incremental Filtering", 1675 ) 1676 is_compare_strictly: Optional[bool] = Field( 1677 False, 1678 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1679 title="Strict Start-End Time Comparison", 1680 ) 1681 global_substream_cursor: Optional[bool] = Field( 1682 False, 1683 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1684 title="Global Substream Cursor", 1685 ) 1686 lookback_window: Optional[str] = Field( 1687 None, 1688 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1689 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1690 title="Lookback Window", 1691 ) 1692 partition_field_end: Optional[str] = Field( 1693 None, 1694 description="Name of the partition start time field.", 1695 examples=["ending_time"], 1696 title="Partition Field End", 1697 ) 1698 partition_field_start: Optional[str] = Field( 1699 None, 1700 description="Name of the partition end time field.", 1701 examples=["starting_time"], 1702 title="Partition Field Start", 1703 ) 1704 step: Optional[str] = Field( 1705 None, 1706 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1707 examples=["P1W", "{{ config['step_increment'] }}"], 1708 title="Step", 1709 ) 1710 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1713class JwtAuthenticator(BaseModel): 1714 type: Literal["JwtAuthenticator"] 1715 secret_key: str = Field( 1716 ..., 1717 description="Secret used to sign the JSON web token.", 1718 examples=["{{ config['secret_key'] }}"], 1719 title="Secret Key", 1720 ) 1721 base64_encode_secret_key: Optional[bool] = Field( 1722 False, 1723 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1724 title="Base64-encode Secret Key", 1725 ) 1726 algorithm: Algorithm = Field( 1727 ..., 1728 description="Algorithm used to sign the JSON web token.", 1729 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1730 title="Algorithm", 1731 ) 1732 token_duration: Optional[int] = Field( 1733 1200, 1734 description="The amount of time in seconds a JWT token can be valid after being issued.", 1735 examples=[1200, 3600], 1736 title="Token Duration", 1737 ) 1738 header_prefix: Optional[str] = Field( 1739 None, 1740 description="The prefix to be used within the Authentication header.", 1741 examples=["Bearer", "Basic"], 1742 title="Header Prefix", 1743 ) 1744 jwt_headers: Optional[JwtHeaders] = Field( 1745 None, 1746 description="JWT headers used when signing JSON web token.", 1747 title="JWT Headers", 1748 ) 1749 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1750 None, 1751 description="Additional headers to be included with the JWT headers object.", 1752 title="Additional JWT Headers", 1753 ) 1754 jwt_payload: Optional[JwtPayload] = Field( 1755 None, 1756 description="JWT Payload used when signing JSON web token.", 1757 title="JWT Payload", 1758 ) 1759 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1760 None, 1761 description="Additional properties to be added to the JWT payload.", 1762 title="Additional JWT Payload Properties", 1763 ) 1764 passphrase: Optional[str] = Field( 1765 None, 1766 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1767 examples=["{{ config['passphrase'] }}"], 1768 title="Passphrase", 1769 ) 1770 request_option: Optional[RequestOption] = Field( 1771 None, 1772 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1773 title="Request Option", 1774 ) 1775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1778class OAuthAuthenticator(BaseModel): 1779 type: Literal["OAuthAuthenticator"] 1780 client_id_name: Optional[str] = Field( 1781 "client_id", 1782 description="The name of the property to use to refresh the `access_token`.", 1783 examples=["custom_app_id"], 1784 title="Client ID Property Name", 1785 ) 1786 client_id: Optional[str] = Field( 1787 None, 1788 description="The OAuth client ID. Fill it in the user inputs.", 1789 examples=[ 1790 "{{ config['client_id'] }}", 1791 "{{ config['credentials']['client_id }}", 1792 ], 1793 title="Client ID", 1794 ) 1795 client_secret_name: Optional[str] = Field( 1796 "client_secret", 1797 description="The name of the property to use to refresh the `access_token`.", 1798 examples=["custom_app_secret"], 1799 title="Client Secret Property Name", 1800 ) 1801 client_secret: Optional[str] = Field( 1802 None, 1803 description="The OAuth client secret. Fill it in the user inputs.", 1804 examples=[ 1805 "{{ config['client_secret'] }}", 1806 "{{ config['credentials']['client_secret }}", 1807 ], 1808 title="Client Secret", 1809 ) 1810 refresh_token_name: Optional[str] = Field( 1811 "refresh_token", 1812 description="The name of the property to use to refresh the `access_token`.", 1813 examples=["custom_app_refresh_value"], 1814 title="Refresh Token Property Name", 1815 ) 1816 refresh_token: Optional[str] = Field( 1817 None, 1818 description="Credential artifact used to get a new access token.", 1819 examples=[ 1820 "{{ config['refresh_token'] }}", 1821 "{{ config['credentials]['refresh_token'] }}", 1822 ], 1823 title="Refresh Token", 1824 ) 1825 token_refresh_endpoint: Optional[str] = Field( 1826 None, 1827 description="The full URL to call to obtain a new access token.", 1828 examples=["https://connect.squareup.com/oauth2/token"], 1829 title="Token Refresh Endpoint", 1830 ) 1831 access_token_name: Optional[str] = Field( 1832 "access_token", 1833 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1834 examples=["access_token"], 1835 title="Access Token Property Name", 1836 ) 1837 access_token_value: Optional[str] = Field( 1838 None, 1839 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1840 examples=["secret_access_token_value"], 1841 title="Access Token Value", 1842 ) 1843 expires_in_name: Optional[str] = Field( 1844 "expires_in", 1845 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1846 examples=["expires_in"], 1847 title="Token Expiry Property Name", 1848 ) 1849 grant_type_name: Optional[str] = Field( 1850 "grant_type", 1851 description="The name of the property to use to refresh the `access_token`.", 1852 examples=["custom_grant_type"], 1853 title="Grant Type Property Name", 1854 ) 1855 grant_type: Optional[str] = Field( 1856 "refresh_token", 1857 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1858 examples=["refresh_token", "client_credentials"], 1859 title="Grant Type", 1860 ) 1861 refresh_request_body: Optional[Dict[str, Any]] = Field( 1862 None, 1863 description="Body of the request sent to get a new access token.", 1864 examples=[ 1865 { 1866 "applicationId": "{{ config['application_id'] }}", 1867 "applicationSecret": "{{ config['application_secret'] }}", 1868 "token": "{{ config['token'] }}", 1869 } 1870 ], 1871 title="Refresh Request Body", 1872 ) 1873 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1874 None, 1875 description="Headers of the request sent to get a new access token.", 1876 examples=[ 1877 { 1878 "Authorization": "<AUTH_TOKEN>", 1879 "Content-Type": "application/x-www-form-urlencoded", 1880 } 1881 ], 1882 title="Refresh Request Headers", 1883 ) 1884 scopes: Optional[List[str]] = Field( 1885 None, 1886 description="List of scopes that should be granted to the access token.", 1887 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1888 title="Scopes", 1889 ) 1890 token_expiry_date: Optional[str] = Field( 1891 None, 1892 description="The access token expiry date.", 1893 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1894 title="Token Expiry Date", 1895 ) 1896 token_expiry_date_format: Optional[str] = Field( 1897 None, 1898 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1899 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1900 title="Token Expiry Date Format", 1901 ) 1902 refresh_token_error_status_codes: Optional[List[int]] = Field( 1903 None, 1904 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1905 examples=[[400, 500]], 1906 title="Refresh Token Error Status Codes", 1907 ) 1908 refresh_token_error_key: Optional[str] = Field( 1909 None, 1910 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1911 examples=["error"], 1912 title="Refresh Token Error Key", 1913 ) 1914 refresh_token_error_values: Optional[List[str]] = Field( 1915 None, 1916 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1917 examples=[["invalid_grant", "invalid_permissions"]], 1918 title="Refresh Token Error Values", 1919 ) 1920 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1921 None, 1922 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1923 title="Refresh Token Updater", 1924 ) 1925 profile_assertion: Optional[JwtAuthenticator] = Field( 1926 None, 1927 description="The authenticator being used to authenticate the client authenticator.", 1928 title="Profile Assertion", 1929 ) 1930 use_profile_assertion: Optional[bool] = Field( 1931 False, 1932 description="Enable using profile assertion as a flow for OAuth authorization.", 1933 title="Use Profile Assertion", 1934 ) 1935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1938class FixedWindowCallRatePolicy(BaseModel): 1939 class Config: 1940 extra = Extra.allow 1941 1942 type: Literal["FixedWindowCallRatePolicy"] 1943 period: str = Field( 1944 ..., description="The time interval for the rate limit window.", title="Period" 1945 ) 1946 call_limit: int = Field( 1947 ..., 1948 description="The maximum number of calls allowed within the period.", 1949 title="Call Limit", 1950 ) 1951 matchers: List[HttpRequestRegexMatcher] = Field( 1952 ..., 1953 description="List of matchers that define which requests this policy applies to.", 1954 title="Matchers", 1955 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1958class MovingWindowCallRatePolicy(BaseModel): 1959 class Config: 1960 extra = Extra.allow 1961 1962 type: Literal["MovingWindowCallRatePolicy"] 1963 rates: List[Rate] = Field( 1964 ..., 1965 description="List of rates that define the call limits for different time intervals.", 1966 title="Rates", 1967 ) 1968 matchers: List[HttpRequestRegexMatcher] = Field( 1969 ..., 1970 description="List of matchers that define which requests this policy applies to.", 1971 title="Matchers", 1972 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1975class UnlimitedCallRatePolicy(BaseModel): 1976 class Config: 1977 extra = Extra.allow 1978 1979 type: Literal["UnlimitedCallRatePolicy"] 1980 matchers: List[HttpRequestRegexMatcher] = Field( 1981 ..., 1982 description="List of matchers that define which requests this policy applies to.", 1983 title="Matchers", 1984 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1987class DefaultErrorHandler(BaseModel): 1988 type: Literal["DefaultErrorHandler"] 1989 backoff_strategies: Optional[ 1990 List[ 1991 Union[ 1992 ConstantBackoffStrategy, 1993 ExponentialBackoffStrategy, 1994 WaitTimeFromHeader, 1995 WaitUntilTimeFromHeader, 1996 CustomBackoffStrategy, 1997 ] 1998 ] 1999 ] = Field( 2000 None, 2001 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2002 title="Backoff Strategies", 2003 ) 2004 max_retries: Optional[int] = Field( 2005 5, 2006 description="The maximum number of time to retry a retryable request before giving up and failing.", 2007 examples=[5, 0, 10], 2008 title="Max Retry Count", 2009 ) 2010 response_filters: Optional[List[HttpResponseFilter]] = Field( 2011 None, 2012 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2013 title="Response Filters", 2014 ) 2015 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2018class DefaultPaginator(BaseModel): 2019 type: Literal["DefaultPaginator"] 2020 pagination_strategy: Union[ 2021 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2022 ] = Field( 2023 ..., 2024 description="Strategy defining how records are paginated.", 2025 title="Pagination Strategy", 2026 ) 2027 page_size_option: Optional[RequestOption] = Field( 2028 None, title="Inject Page Size Into Outgoing HTTP Request" 2029 ) 2030 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2031 None, 2032 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2033 title="Inject Page Token Into Outgoing HTTP Request", 2034 ) 2035 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2038class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2039 type: Literal["ApiKey"] 2040 inject_into: RequestOption = Field( 2041 ..., 2042 description="Configure how the API Key will be sent in requests to the source API.", 2043 examples=[ 2044 {"inject_into": "header", "field_name": "Authorization"}, 2045 {"inject_into": "request_parameter", "field_name": "authKey"}, 2046 ], 2047 title="Inject API Key Into Outgoing HTTP Request", 2048 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2051class JsonSchemaPropertySelector(BaseModel): 2052 type: Literal["JsonSchemaPropertySelector"] 2053 transformations: Optional[ 2054 List[ 2055 Union[ 2056 AddFields, 2057 RemoveFields, 2058 KeysToLower, 2059 KeysToSnakeCase, 2060 FlattenFields, 2061 DpathFlattenFields, 2062 KeysReplace, 2063 CustomTransformation, 2064 ] 2065 ] 2066 ] = Field( 2067 None, 2068 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2069 title="Transformations", 2070 ) 2071 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2074class ListPartitionRouter(BaseModel): 2075 type: Literal["ListPartitionRouter"] 2076 cursor_field: str = Field( 2077 ..., 2078 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2079 examples=["section", "{{ config['section_key'] }}"], 2080 title="Current Partition Value Identifier", 2081 ) 2082 values: Union[str, List[str]] = Field( 2083 ..., 2084 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2085 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2086 title="Partition Values", 2087 ) 2088 request_option: Optional[RequestOption] = Field( 2089 None, 2090 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2091 title="Inject Partition Value Into Outgoing HTTP Request", 2092 ) 2093 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2096class RecordSelector(BaseModel): 2097 type: Literal["RecordSelector"] 2098 extractor: Union[DpathExtractor, CustomRecordExtractor] 2099 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2100 None, 2101 description="Responsible for filtering records to be emitted by the Source.", 2102 title="Record Filter", 2103 ) 2104 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2105 None, 2106 description="Responsible for normalization according to the schema.", 2107 title="Schema Normalization", 2108 ) 2109 transform_before_filtering: Optional[bool] = Field( 2110 None, 2111 description="If true, transformation will be applied before record filtering.", 2112 title="Transform Before Filtering", 2113 ) 2114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
PaginationReset(pydantic.v1.main.BaseModel):
2117class PaginationReset(BaseModel): 2118 type: Literal["PaginationReset"] 2119 action: Action1 2120 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2123class GzipDecoder(BaseModel): 2124 type: Literal["GzipDecoder"] 2125 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2128class RequestBodyGraphQL(BaseModel): 2129 type: Literal["RequestBodyGraphQL"] 2130 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2133class DpathValidator(BaseModel): 2134 type: Literal["DpathValidator"] 2135 field_path: List[str] = Field( 2136 ..., 2137 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2138 examples=[ 2139 ["data"], 2140 ["data", "records"], 2141 ["data", "{{ parameters.name }}"], 2142 ["data", "*", "record"], 2143 ], 2144 title="Field Path", 2145 ) 2146 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2147 ..., 2148 description="The condition that the specified config value will be evaluated against", 2149 title="Validation Strategy", 2150 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2153class PredicateValidator(BaseModel): 2154 type: Literal["PredicateValidator"] 2155 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2156 ..., 2157 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2158 examples=[ 2159 "test-value", 2160 "{{ config['api_version'] }}", 2161 "{{ config['tenant_id'] }}", 2162 123, 2163 ], 2164 title="Value", 2165 ) 2166 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2167 ..., 2168 description="The validation strategy to apply to the value.", 2169 title="Validation Strategy", 2170 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2173class ConfigAddFields(BaseModel): 2174 type: Literal["ConfigAddFields"] 2175 fields: List[AddedFieldDefinition] = Field( 2176 ..., 2177 description="A list of transformations (path and corresponding value) that will be added to the config.", 2178 title="Fields", 2179 ) 2180 condition: Optional[str] = Field( 2181 "", 2182 description="Fields will be added if expression is evaluated to True.", 2183 examples=[ 2184 "{{ config['environemnt'] == 'sandbox' }}", 2185 "{{ property is integer }}", 2186 "{{ property|length > 5 }}", 2187 "{{ property == 'some_string_to_match' }}", 2188 ], 2189 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2192class CompositeErrorHandler(BaseModel): 2193 type: Literal["CompositeErrorHandler"] 2194 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2195 Field( 2196 ..., 2197 description="List of error handlers to iterate on to determine how to handle a failed response.", 2198 title="Error Handlers", 2199 ) 2200 ) 2201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2204class HTTPAPIBudget(BaseModel): 2205 class Config: 2206 extra = Extra.allow 2207 2208 type: Literal["HTTPAPIBudget"] 2209 policies: List[ 2210 Union[ 2211 FixedWindowCallRatePolicy, 2212 MovingWindowCallRatePolicy, 2213 UnlimitedCallRatePolicy, 2214 ] 2215 ] = Field( 2216 ..., 2217 description="List of call rate policies that define how many calls are allowed.", 2218 title="Policies", 2219 ) 2220 ratelimit_reset_header: Optional[str] = Field( 2221 "ratelimit-reset", 2222 description="The HTTP response header name that indicates when the rate limit resets.", 2223 title="Rate Limit Reset Header", 2224 ) 2225 ratelimit_remaining_header: Optional[str] = Field( 2226 "ratelimit-remaining", 2227 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2228 title="Rate Limit Remaining Header", 2229 ) 2230 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2231 [429], 2232 description="List of HTTP status codes that indicate a rate limit has been hit.", 2233 title="Status Codes for Rate Limit Hit", 2234 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2237class ZipfileDecoder(BaseModel): 2238 class Config: 2239 extra = Extra.allow 2240 2241 type: Literal["ZipfileDecoder"] 2242 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2243 ..., 2244 description="Parser to parse the decompressed data from the zipfile(s).", 2245 title="Parser", 2246 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2249class ConfigMigration(BaseModel): 2250 type: Literal["ConfigMigration"] 2251 description: Optional[str] = Field( 2252 None, description="The description/purpose of the config migration." 2253 ) 2254 transformations: List[ 2255 Union[ 2256 ConfigRemapField, 2257 ConfigAddFields, 2258 ConfigRemoveFields, 2259 CustomConfigTransformation, 2260 ] 2261 ] = Field( 2262 ..., 2263 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2264 title="Transformations", 2265 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2268class ConfigNormalizationRules(BaseModel): 2269 class Config: 2270 extra = Extra.forbid 2271 2272 type: Literal["ConfigNormalizationRules"] 2273 config_migrations: Optional[List[ConfigMigration]] = Field( 2274 [], 2275 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2276 title="Config Migrations", 2277 ) 2278 transformations: Optional[ 2279 List[ 2280 Union[ 2281 ConfigRemapField, 2282 ConfigAddFields, 2283 ConfigRemoveFields, 2284 CustomConfigTransformation, 2285 ] 2286 ] 2287 ] = Field( 2288 [], 2289 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2290 title="Transformations", 2291 ) 2292 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2293 [], 2294 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2295 title="Validations", 2296 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2299class Spec(BaseModel): 2300 type: Literal["Spec"] 2301 connection_specification: Dict[str, Any] = Field( 2302 ..., 2303 description="A connection specification describing how a the connector can be configured.", 2304 title="Connection Specification", 2305 ) 2306 documentation_url: Optional[str] = Field( 2307 None, 2308 description="URL of the connector's documentation page.", 2309 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2310 title="Documentation URL", 2311 ) 2312 advanced_auth: Optional[AuthFlow] = Field( 2313 None, 2314 description="Advanced specification for configuring the authentication flow.", 2315 title="Advanced Auth", 2316 ) 2317 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2318 None, title="Config Normalization Rules" 2319 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2322class DeclarativeSource1(BaseModel): 2323 class Config: 2324 extra = Extra.forbid 2325 2326 type: Literal["DeclarativeSource"] 2327 check: Union[CheckStream, CheckDynamicStream] 2328 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2329 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2330 version: str = Field( 2331 ..., 2332 description="The version of the Airbyte CDK used to build and test the source.", 2333 ) 2334 schemas: Optional[Schemas] = None 2335 definitions: Optional[Dict[str, Any]] = None 2336 spec: Optional[Spec] = None 2337 concurrency_level: Optional[ConcurrencyLevel] = None 2338 api_budget: Optional[HTTPAPIBudget] = None 2339 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2340 None, 2341 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2342 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2343 title="Maximum Concurrent Asynchronous Jobs", 2344 ) 2345 metadata: Optional[Dict[str, Any]] = Field( 2346 None, 2347 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2348 ) 2349 description: Optional[str] = Field( 2350 None, 2351 description="A description of the connector. It will be presented on the Source documentation page.", 2352 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2355class DeclarativeSource2(BaseModel): 2356 class Config: 2357 extra = Extra.forbid 2358 2359 type: Literal["DeclarativeSource"] 2360 check: Union[CheckStream, CheckDynamicStream] 2361 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2362 None 2363 ) 2364 dynamic_streams: List[DynamicDeclarativeStream] 2365 version: str = Field( 2366 ..., 2367 description="The version of the Airbyte CDK used to build and test the source.", 2368 ) 2369 schemas: Optional[Schemas] = None 2370 definitions: Optional[Dict[str, Any]] = None 2371 spec: Optional[Spec] = None 2372 concurrency_level: Optional[ConcurrencyLevel] = None 2373 api_budget: Optional[HTTPAPIBudget] = None 2374 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2375 None, 2376 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2377 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2378 title="Maximum Concurrent Asynchronous Jobs", 2379 ) 2380 metadata: Optional[Dict[str, Any]] = Field( 2381 None, 2382 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2383 ) 2384 description: Optional[str] = Field( 2385 None, 2386 description="A description of the connector. It will be presented on the Source documentation page.", 2387 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2390class DeclarativeSource(BaseModel): 2391 class Config: 2392 extra = Extra.forbid 2393 2394 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2395 ..., 2396 description="An API source that extracts data according to its declarative components.", 2397 title="DeclarativeSource", 2398 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2401class SelectiveAuthenticator(BaseModel): 2402 class Config: 2403 extra = Extra.allow 2404 2405 type: Literal["SelectiveAuthenticator"] 2406 authenticator_selection_path: List[str] = Field( 2407 ..., 2408 description="Path of the field in config with selected authenticator name", 2409 examples=[["auth"], ["auth", "type"]], 2410 title="Authenticator Selection Path", 2411 ) 2412 authenticators: Dict[ 2413 str, 2414 Union[ 2415 ApiKeyAuthenticator, 2416 BasicHttpAuthenticator, 2417 BearerAuthenticator, 2418 OAuthAuthenticator, 2419 JwtAuthenticator, 2420 SessionTokenAuthenticator, 2421 LegacySessionTokenAuthenticator, 2422 CustomAuthenticator, 2423 NoAuth, 2424 ], 2425 ] = Field( 2426 ..., 2427 description="Authenticators to select from.", 2428 examples=[ 2429 { 2430 "authenticators": { 2431 "token": "#/definitions/ApiKeyAuthenticator", 2432 "oauth": "#/definitions/OAuthAuthenticator", 2433 "jwt": "#/definitions/JwtAuthenticator", 2434 } 2435 } 2436 ], 2437 title="Authenticators", 2438 ) 2439 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2442class ConditionalStreams(BaseModel): 2443 type: Literal["ConditionalStreams"] 2444 condition: str = Field( 2445 ..., 2446 description="Condition that will be evaluated to determine if a set of streams should be available.", 2447 examples=["{{ config['is_sandbox'] }}"], 2448 title="Condition", 2449 ) 2450 streams: List[DeclarativeStream] = Field( 2451 ..., 2452 description="Streams that will be used during an operation based on the condition.", 2453 title="Streams", 2454 ) 2455 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2458class FileUploader(BaseModel): 2459 type: Literal["FileUploader"] 2460 requester: Union[HttpRequester, CustomRequester] = Field( 2461 ..., 2462 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2463 ) 2464 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2465 ..., 2466 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2467 ) 2468 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2469 None, 2470 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2471 ) 2472 filename_extractor: Optional[str] = Field( 2473 None, 2474 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2475 examples=[ 2476 "{{ record.id }}/{{ record.file_name }}/", 2477 "{{ record.id }}_{{ record.file_name }}/", 2478 ], 2479 ) 2480 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2483class DeclarativeStream(BaseModel): 2484 class Config: 2485 extra = Extra.allow 2486 2487 type: Literal["DeclarativeStream"] 2488 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2489 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2490 ..., 2491 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2492 title="Retriever", 2493 ) 2494 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2495 None, 2496 description="Component used to fetch data incrementally based on a time field in the data.", 2497 title="Incremental Sync", 2498 ) 2499 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2500 schema_loader: Optional[ 2501 Union[ 2502 InlineSchemaLoader, 2503 DynamicSchemaLoader, 2504 JsonFileSchemaLoader, 2505 List[ 2506 Union[ 2507 InlineSchemaLoader, 2508 DynamicSchemaLoader, 2509 JsonFileSchemaLoader, 2510 CustomSchemaLoader, 2511 ] 2512 ], 2513 CustomSchemaLoader, 2514 ] 2515 ] = Field( 2516 None, 2517 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2518 title="Schema Loader", 2519 ) 2520 transformations: Optional[ 2521 List[ 2522 Union[ 2523 AddFields, 2524 RemoveFields, 2525 KeysToLower, 2526 KeysToSnakeCase, 2527 FlattenFields, 2528 DpathFlattenFields, 2529 KeysReplace, 2530 CustomTransformation, 2531 ] 2532 ] 2533 ] = Field( 2534 None, 2535 description="A list of transformations to be applied to each output record.", 2536 title="Transformations", 2537 ) 2538 state_migrations: Optional[ 2539 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2540 ] = Field( 2541 [], 2542 description="Array of state migrations to be applied on the input state", 2543 title="State Migrations", 2544 ) 2545 file_uploader: Optional[FileUploader] = Field( 2546 None, 2547 description="(experimental) Describes how to fetch a file", 2548 title="File Uploader", 2549 ) 2550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2553class SessionTokenAuthenticator(BaseModel): 2554 type: Literal["SessionTokenAuthenticator"] 2555 login_requester: HttpRequester = Field( 2556 ..., 2557 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2558 examples=[ 2559 { 2560 "type": "HttpRequester", 2561 "url_base": "https://my_api.com", 2562 "path": "/login", 2563 "authenticator": { 2564 "type": "BasicHttpAuthenticator", 2565 "username": "{{ config.username }}", 2566 "password": "{{ config.password }}", 2567 }, 2568 } 2569 ], 2570 title="Login Requester", 2571 ) 2572 session_token_path: List[str] = Field( 2573 ..., 2574 description="The path in the response body returned from the login requester to the session token.", 2575 examples=[["access_token"], ["result", "token"]], 2576 title="Session Token Path", 2577 ) 2578 expiration_duration: Optional[str] = Field( 2579 None, 2580 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2581 examples=["PT1H", "P1D"], 2582 title="Expiration Duration", 2583 ) 2584 request_authentication: Union[ 2585 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2586 ] = Field( 2587 ..., 2588 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2589 title="Data Request Authentication", 2590 ) 2591 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2592 None, description="Component used to decode the response.", title="Decoder" 2593 ) 2594 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2597class HttpRequester(BaseModelWithDeprecations): 2598 type: Literal["HttpRequester"] 2599 url_base: Optional[str] = Field( 2600 None, 2601 deprecated=True, 2602 deprecation_message="Use `url` field instead.", 2603 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2604 examples=[ 2605 "https://connect.squareup.com/v2", 2606 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2607 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2608 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2609 ], 2610 title="API Base URL", 2611 ) 2612 url: Optional[str] = Field( 2613 None, 2614 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2615 examples=[ 2616 "https://connect.squareup.com/v2", 2617 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2618 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2619 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2620 ], 2621 title="API Endpoint URL", 2622 ) 2623 path: Optional[str] = Field( 2624 None, 2625 deprecated=True, 2626 deprecation_message="Use `url` field instead.", 2627 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2628 examples=[ 2629 "/products", 2630 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2631 "/trades/{{ config['symbol_id'] }}/history", 2632 ], 2633 title="URL Path", 2634 ) 2635 http_method: Optional[HttpMethod] = Field( 2636 HttpMethod.GET, 2637 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2638 examples=["GET", "POST"], 2639 title="HTTP Method", 2640 ) 2641 authenticator: Optional[ 2642 Union[ 2643 ApiKeyAuthenticator, 2644 BasicHttpAuthenticator, 2645 BearerAuthenticator, 2646 OAuthAuthenticator, 2647 JwtAuthenticator, 2648 SessionTokenAuthenticator, 2649 SelectiveAuthenticator, 2650 CustomAuthenticator, 2651 NoAuth, 2652 LegacySessionTokenAuthenticator, 2653 ] 2654 ] = Field( 2655 None, 2656 description="Authentication method to use for requests sent to the API.", 2657 title="Authenticator", 2658 ) 2659 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2660 None, 2661 deprecated=True, 2662 deprecation_message="Use `query_properties` field instead.", 2663 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2664 title="Fetch Properties from Endpoint", 2665 ) 2666 query_properties: Optional[QueryProperties] = Field( 2667 None, 2668 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2669 title="Query Properties", 2670 ) 2671 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2672 None, 2673 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2674 examples=[ 2675 {"unit": "day"}, 2676 { 2677 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2678 }, 2679 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2680 {"sort_by[asc]": "updated_at"}, 2681 ], 2682 title="Query Parameters", 2683 ) 2684 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2685 None, 2686 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2687 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2688 title="Request Headers", 2689 ) 2690 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2691 None, 2692 deprecated=True, 2693 deprecation_message="Use `request_body` field instead.", 2694 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2695 examples=[ 2696 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2697 ], 2698 title="Request Body Payload (Non-JSON)", 2699 ) 2700 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2701 None, 2702 deprecated=True, 2703 deprecation_message="Use `request_body` field instead.", 2704 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2705 examples=[ 2706 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2707 {"key": "{{ config['value'] }}"}, 2708 {"sort": {"field": "updated_at", "order": "ascending"}}, 2709 ], 2710 title="Request Body JSON Payload", 2711 ) 2712 request_body: Optional[ 2713 Union[ 2714 RequestBodyPlainText, 2715 RequestBodyUrlEncodedForm, 2716 RequestBodyJsonObject, 2717 RequestBodyGraphQL, 2718 ] 2719 ] = Field( 2720 None, 2721 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2722 title="Request Body", 2723 ) 2724 error_handler: Optional[ 2725 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2726 ] = Field( 2727 None, 2728 description="Error handler component that defines how to handle errors.", 2729 title="Error Handler", 2730 ) 2731 use_cache: Optional[bool] = Field( 2732 False, 2733 description="Enables stream requests caching. This field is automatically set by the CDK.", 2734 title="Use Cache", 2735 ) 2736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2739class DynamicSchemaLoader(BaseModel): 2740 type: Literal["DynamicSchemaLoader"] 2741 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2742 ..., 2743 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2744 title="Retriever", 2745 ) 2746 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2747 None, 2748 description="Responsible for filtering fields to be added to json schema.", 2749 title="Schema Filter", 2750 ) 2751 schema_transformations: Optional[ 2752 List[ 2753 Union[ 2754 AddFields, 2755 RemoveFields, 2756 KeysToLower, 2757 KeysToSnakeCase, 2758 FlattenFields, 2759 DpathFlattenFields, 2760 KeysReplace, 2761 CustomTransformation, 2762 ] 2763 ] 2764 ] = Field( 2765 None, 2766 description="A list of transformations to be applied to the schema.", 2767 title="Schema Transformations", 2768 ) 2769 schema_type_identifier: SchemaTypeIdentifier 2770 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2773class ParentStreamConfig(BaseModel): 2774 type: Literal["ParentStreamConfig"] 2775 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2776 ..., description="Reference to the parent stream.", title="Parent Stream" 2777 ) 2778 parent_key: str = Field( 2779 ..., 2780 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2781 examples=["id", "{{ config['parent_record_id'] }}"], 2782 title="Parent Key", 2783 ) 2784 partition_field: str = Field( 2785 ..., 2786 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2787 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2788 title="Current Parent Key Value Identifier", 2789 ) 2790 request_option: Optional[RequestOption] = Field( 2791 None, 2792 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2793 title="Request Option", 2794 ) 2795 incremental_dependency: Optional[bool] = Field( 2796 False, 2797 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2798 title="Incremental Dependency", 2799 ) 2800 lazy_read_pointer: Optional[List[str]] = Field( 2801 [], 2802 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2803 title="Lazy Read Pointer", 2804 ) 2805 extra_fields: Optional[List[List[str]]] = Field( 2806 None, 2807 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2808 title="Extra Fields", 2809 ) 2810 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2813class PropertiesFromEndpoint(BaseModel): 2814 type: Literal["PropertiesFromEndpoint"] 2815 property_field_path: List[str] = Field( 2816 ..., 2817 description="Describes the path to the field that should be extracted", 2818 examples=[["name"]], 2819 ) 2820 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2821 ..., 2822 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2823 ) 2824 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2827class QueryProperties(BaseModel): 2828 type: Literal["QueryProperties"] 2829 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2830 ..., 2831 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2832 title="Property List", 2833 ) 2834 always_include_properties: Optional[List[str]] = Field( 2835 None, 2836 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2837 title="Always Include Properties", 2838 ) 2839 property_chunking: Optional[PropertyChunking] = Field( 2840 None, 2841 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2842 title="Property Chunking", 2843 ) 2844 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2845 None, 2846 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2847 title="Property Selector", 2848 ) 2849 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2852class StateDelegatingStream(BaseModel): 2853 type: Literal["StateDelegatingStream"] 2854 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2855 full_refresh_stream: DeclarativeStream = Field( 2856 ..., 2857 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2858 title="Full Refresh Stream", 2859 ) 2860 incremental_stream: DeclarativeStream = Field( 2861 ..., 2862 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2863 title="Incremental Stream", 2864 ) 2865 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2868class SimpleRetriever(BaseModel): 2869 type: Literal["SimpleRetriever"] 2870 requester: Union[HttpRequester, CustomRequester] = Field( 2871 ..., 2872 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2873 ) 2874 decoder: Optional[ 2875 Union[ 2876 JsonDecoder, 2877 XmlDecoder, 2878 CsvDecoder, 2879 JsonlDecoder, 2880 GzipDecoder, 2881 IterableDecoder, 2882 ZipfileDecoder, 2883 CustomDecoder, 2884 ] 2885 ] = Field( 2886 None, 2887 description="Component decoding the response so records can be extracted.", 2888 title="HTTP Response Format", 2889 ) 2890 record_selector: RecordSelector = Field( 2891 ..., 2892 description="Component that describes how to extract records from a HTTP response.", 2893 ) 2894 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2895 None, 2896 description="Paginator component that describes how to navigate through the API's pages.", 2897 ) 2898 pagination_reset: Optional[PaginationReset] = Field( 2899 None, 2900 description="Describes what triggers pagination reset and how to handle it.", 2901 ) 2902 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2903 False, 2904 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2905 ) 2906 partition_router: Optional[ 2907 Union[ 2908 SubstreamPartitionRouter, 2909 ListPartitionRouter, 2910 GroupingPartitionRouter, 2911 CustomPartitionRouter, 2912 List[ 2913 Union[ 2914 SubstreamPartitionRouter, 2915 ListPartitionRouter, 2916 GroupingPartitionRouter, 2917 CustomPartitionRouter, 2918 ] 2919 ], 2920 ] 2921 ] = Field( 2922 None, 2923 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2924 title="Partition Router", 2925 ) 2926 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2929class AsyncRetriever(BaseModel): 2930 type: Literal["AsyncRetriever"] 2931 record_selector: RecordSelector = Field( 2932 ..., 2933 description="Component that describes how to extract records from a HTTP response.", 2934 ) 2935 status_mapping: AsyncJobStatusMap = Field( 2936 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2937 ) 2938 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2939 ..., description="Responsible for fetching the actual status of the async job." 2940 ) 2941 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2942 None, 2943 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2944 ) 2945 download_extractor: Optional[ 2946 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2947 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2948 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2949 ..., 2950 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2951 ) 2952 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2953 ..., 2954 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2955 ) 2956 polling_job_timeout: Optional[Union[int, str]] = Field( 2957 None, 2958 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2959 ) 2960 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2961 None, 2962 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2963 ) 2964 download_requester: Union[HttpRequester, CustomRequester] = Field( 2965 ..., 2966 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2967 ) 2968 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2969 None, 2970 description="Paginator component that describes how to navigate through the API's pages during download.", 2971 ) 2972 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2973 None, 2974 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2975 ) 2976 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2977 None, 2978 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2979 ) 2980 partition_router: Optional[ 2981 Union[ 2982 ListPartitionRouter, 2983 SubstreamPartitionRouter, 2984 GroupingPartitionRouter, 2985 CustomPartitionRouter, 2986 List[ 2987 Union[ 2988 ListPartitionRouter, 2989 SubstreamPartitionRouter, 2990 GroupingPartitionRouter, 2991 CustomPartitionRouter, 2992 ] 2993 ], 2994 ] 2995 ] = Field( 2996 [], 2997 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2998 title="Partition Router", 2999 ) 3000 decoder: Optional[ 3001 Union[ 3002 CsvDecoder, 3003 GzipDecoder, 3004 JsonDecoder, 3005 JsonlDecoder, 3006 IterableDecoder, 3007 XmlDecoder, 3008 ZipfileDecoder, 3009 CustomDecoder, 3010 ] 3011 ] = Field( 3012 None, 3013 description="Component decoding the response so records can be extracted.", 3014 title="HTTP Response Format", 3015 ) 3016 download_decoder: Optional[ 3017 Union[ 3018 CsvDecoder, 3019 GzipDecoder, 3020 JsonDecoder, 3021 JsonlDecoder, 3022 IterableDecoder, 3023 XmlDecoder, 3024 ZipfileDecoder, 3025 CustomDecoder, 3026 ] 3027 ] = Field( 3028 None, 3029 description="Component decoding the download response so records can be extracted.", 3030 title="Download HTTP Response Format", 3031 ) 3032 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3035class SubstreamPartitionRouter(BaseModel): 3036 type: Literal["SubstreamPartitionRouter"] 3037 parent_stream_configs: List[ParentStreamConfig] = Field( 3038 ..., 3039 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3040 title="Parent Stream Configs", 3041 ) 3042 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3045class GroupingPartitionRouter(BaseModel): 3046 type: Literal["GroupingPartitionRouter"] 3047 group_size: int = Field( 3048 ..., 3049 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3050 examples=[10, 50], 3051 title="Group Size", 3052 ) 3053 underlying_partition_router: Union[ 3054 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3055 ] = Field( 3056 ..., 3057 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3058 title="Underlying Partition Router", 3059 ) 3060 deduplicate: Optional[bool] = Field( 3061 True, 3062 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3063 title="Deduplicate Partitions", 3064 ) 3065 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3068class HttpComponentsResolver(BaseModel): 3069 type: Literal["HttpComponentsResolver"] 3070 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3071 ..., 3072 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3073 title="Retriever", 3074 ) 3075 components_mapping: List[ComponentMappingDefinition] 3076 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3079class DynamicDeclarativeStream(BaseModel): 3080 type: Literal["DynamicDeclarativeStream"] 3081 name: Optional[str] = Field( 3082 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3083 ) 3084 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3085 ..., description="Reference to the stream template.", title="Stream Template" 3086 ) 3087 components_resolver: Union[ 3088 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3089 ] = Field( 3090 ..., 3091 description="Component resolve and populates stream templates with components values.", 3092 title="Components Resolver", 3093 ) 3094 use_parent_parameters: Optional[bool] = Field( 3095 True, 3096 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3097 title="Use Parent Parameters", 3098 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]