airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 108 109 110class CursorPagination(BaseModel): 111 type: Literal["CursorPagination"] 112 cursor_value: str = Field( 113 ..., 114 description="Value of the cursor defining the next page to fetch.", 115 examples=[ 116 "{{ headers.link.next.cursor }}", 117 "{{ last_record['key'] }}", 118 "{{ response['nextPage'] }}", 119 ], 120 title="Cursor Value", 121 ) 122 page_size: Optional[Union[int, str]] = Field( 123 None, 124 description="The number of records to include in each pages.", 125 examples=[100, "{{ config['page_size'] }}"], 126 title="Page Size", 127 ) 128 stop_condition: Optional[str] = Field( 129 None, 130 description="Template string evaluating when to stop paginating.", 131 examples=[ 132 "{{ response.data.has_more is false }}", 133 "{{ 'next' not in headers['link'] }}", 134 ], 135 title="Stop Condition", 136 ) 137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 138 139 140class CustomAuthenticator(BaseModel): 141 class Config: 142 extra = Extra.allow 143 144 type: Literal["CustomAuthenticator"] 145 class_name: str = Field( 146 ..., 147 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 148 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 149 title="Class Name", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 152 153 154class CustomBackoffStrategy(BaseModel): 155 class Config: 156 extra = Extra.allow 157 158 type: Literal["CustomBackoffStrategy"] 159 class_name: str = Field( 160 ..., 161 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 162 examples=["source_railz.components.MyCustomBackoffStrategy"], 163 title="Class Name", 164 ) 165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 166 167 168class CustomErrorHandler(BaseModel): 169 class Config: 170 extra = Extra.allow 171 172 type: Literal["CustomErrorHandler"] 173 class_name: str = Field( 174 ..., 175 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 176 examples=["source_railz.components.MyCustomErrorHandler"], 177 title="Class Name", 178 ) 179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 180 181 182class CustomPaginationStrategy(BaseModel): 183 class Config: 184 extra = Extra.allow 185 186 type: Literal["CustomPaginationStrategy"] 187 class_name: str = Field( 188 ..., 189 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 190 examples=["source_railz.components.MyCustomPaginationStrategy"], 191 title="Class Name", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 194 195 196class CustomRecordExtractor(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomRecordExtractor"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomRecordExtractor"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 208 209 210class CustomRecordFilter(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordFilter"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomCustomRecordFilter"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 222 223 224class CustomRequester(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRequester"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomRecordExtractor"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 236 237 238class CustomRetriever(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRetriever"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRetriever"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 250 251 252class CustomPartitionRouter(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomPartitionRouter"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomPartitionRouter"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 264 265 266class CustomSchemaLoader(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomSchemaLoader"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomSchemaLoader"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 278 279 280class CustomSchemaNormalization(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaNormalization"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 288 examples=[ 289 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 290 ], 291 title="Class Name", 292 ) 293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 294 295 296class CustomStateMigration(BaseModel): 297 class Config: 298 extra = Extra.allow 299 300 type: Literal["CustomStateMigration"] 301 class_name: str = Field( 302 ..., 303 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 304 examples=["source_railz.components.MyCustomStateMigration"], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 308 309 310class CustomTransformation(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomTransformation"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomTransformation"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 322 323 324class LegacyToPerPartitionStateMigration(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 329 330 331class Clamping(BaseModel): 332 target: str = Field( 333 ..., 334 description="The period of time that datetime windows will be clamped by", 335 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 336 title="Target", 337 ) 338 target_details: Optional[Dict[str, Any]] = None 339 340 341class Algorithm(Enum): 342 HS256 = "HS256" 343 HS384 = "HS384" 344 HS512 = "HS512" 345 ES256 = "ES256" 346 ES256K = "ES256K" 347 ES384 = "ES384" 348 ES512 = "ES512" 349 RS256 = "RS256" 350 RS384 = "RS384" 351 RS512 = "RS512" 352 PS256 = "PS256" 353 PS384 = "PS384" 354 PS512 = "PS512" 355 EdDSA = "EdDSA" 356 357 358class JwtHeaders(BaseModel): 359 class Config: 360 extra = Extra.forbid 361 362 kid: Optional[str] = Field( 363 None, 364 description="Private key ID for user account.", 365 examples=["{{ config['kid'] }}"], 366 title="Key Identifier", 367 ) 368 typ: Optional[str] = Field( 369 "JWT", 370 description="The media type of the complete JWT.", 371 examples=["JWT"], 372 title="Type", 373 ) 374 cty: Optional[str] = Field( 375 None, 376 description="Content type of JWT header.", 377 examples=["JWT"], 378 title="Content Type", 379 ) 380 381 382class JwtPayload(BaseModel): 383 class Config: 384 extra = Extra.forbid 385 386 iss: Optional[str] = Field( 387 None, 388 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 389 examples=["{{ config['iss'] }}"], 390 title="Issuer", 391 ) 392 sub: Optional[str] = Field( 393 None, 394 description="The subject of the JWT. Commonly defined by the API.", 395 title="Subject", 396 ) 397 aud: Optional[str] = Field( 398 None, 399 description="The recipient that the JWT is intended for. Commonly defined by the API.", 400 examples=["appstoreconnect-v1"], 401 title="Audience", 402 ) 403 404 405class RefreshTokenUpdater(BaseModel): 406 refresh_token_name: Optional[str] = Field( 407 "refresh_token", 408 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 409 examples=["refresh_token"], 410 title="Refresh Token Property Name", 411 ) 412 access_token_config_path: Optional[List[str]] = Field( 413 ["credentials", "access_token"], 414 description="Config path to the access token. Make sure the field actually exists in the config.", 415 examples=[["credentials", "access_token"], ["access_token"]], 416 title="Config Path To Access Token", 417 ) 418 refresh_token_config_path: Optional[List[str]] = Field( 419 ["credentials", "refresh_token"], 420 description="Config path to the access token. Make sure the field actually exists in the config.", 421 examples=[["credentials", "refresh_token"], ["refresh_token"]], 422 title="Config Path To Refresh Token", 423 ) 424 token_expiry_date_config_path: Optional[List[str]] = Field( 425 ["credentials", "token_expiry_date"], 426 description="Config path to the expiry date. Make sure actually exists in the config.", 427 examples=[["credentials", "token_expiry_date"]], 428 title="Config Path To Expiry Date", 429 ) 430 refresh_token_error_status_codes: Optional[List[int]] = Field( 431 [], 432 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 433 examples=[[400, 500]], 434 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 435 ) 436 refresh_token_error_key: Optional[str] = Field( 437 "", 438 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 439 examples=["error"], 440 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 441 ) 442 refresh_token_error_values: Optional[List[str]] = Field( 443 [], 444 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 445 examples=[["invalid_grant", "invalid_permissions"]], 446 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 447 ) 448 449 450class Rate(BaseModel): 451 class Config: 452 extra = Extra.allow 453 454 limit: Union[int, str] = Field( 455 ..., 456 description="The maximum number of calls allowed within the interval.", 457 title="Limit", 458 ) 459 interval: str = Field( 460 ..., 461 description="The time interval for the rate limit.", 462 examples=["PT1H", "P1D"], 463 title="Interval", 464 ) 465 466 467class HttpRequestRegexMatcher(BaseModel): 468 class Config: 469 extra = Extra.allow 470 471 method: Optional[str] = Field( 472 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 473 ) 474 url_base: Optional[str] = Field( 475 None, 476 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 477 title="URL Base", 478 ) 479 url_path_pattern: Optional[str] = Field( 480 None, 481 description="A regular expression pattern to match the URL path.", 482 title="URL Path Pattern", 483 ) 484 params: Optional[Dict[str, Any]] = Field( 485 None, description="The query parameters to match.", title="Parameters" 486 ) 487 headers: Optional[Dict[str, Any]] = Field( 488 None, description="The headers to match.", title="Headers" 489 ) 490 weight: Optional[Union[int, str]] = Field( 491 None, 492 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 493 title="Weight", 494 ) 495 496 497class ResponseToFileExtractor(BaseModel): 498 type: Literal["ResponseToFileExtractor"] 499 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 500 501 502class OnNoRecords(Enum): 503 skip = "skip" 504 emit_parent = "emit_parent" 505 506 507class ExponentialBackoffStrategy(BaseModel): 508 type: Literal["ExponentialBackoffStrategy"] 509 factor: Optional[Union[float, str]] = Field( 510 5, 511 description="Multiplicative constant applied on each retry.", 512 examples=[5, 5.5, "10"], 513 title="Factor", 514 ) 515 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 516 517 518class GroupByKeyMergeStrategy(BaseModel): 519 type: Literal["GroupByKeyMergeStrategy"] 520 key: Union[str, List[str]] = Field( 521 ..., 522 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 523 examples=["id", ["parent_id", "end_date"]], 524 title="Key", 525 ) 526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 527 528 529class SessionTokenRequestBearerAuthenticator(BaseModel): 530 type: Literal["Bearer"] 531 532 533class HttpMethod(Enum): 534 GET = "GET" 535 POST = "POST" 536 537 538class Action(Enum): 539 SUCCESS = "SUCCESS" 540 FAIL = "FAIL" 541 RETRY = "RETRY" 542 IGNORE = "IGNORE" 543 RESET_PAGINATION = "RESET_PAGINATION" 544 RATE_LIMITED = "RATE_LIMITED" 545 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 546 547 548class FailureType(Enum): 549 system_error = "system_error" 550 config_error = "config_error" 551 transient_error = "transient_error" 552 553 554class HttpResponseFilter(BaseModel): 555 type: Literal["HttpResponseFilter"] 556 action: Optional[Action] = Field( 557 None, 558 description="Action to execute if a response matches the filter.", 559 examples=[ 560 "SUCCESS", 561 "FAIL", 562 "RETRY", 563 "IGNORE", 564 "RESET_PAGINATION", 565 "RATE_LIMITED", 566 "REFRESH_TOKEN_THEN_RETRY", 567 ], 568 title="Action", 569 ) 570 failure_type: Optional[FailureType] = Field( 571 None, 572 description="Failure type of traced exception if a response matches the filter.", 573 examples=["system_error", "config_error", "transient_error"], 574 title="Failure Type", 575 ) 576 error_message: Optional[str] = Field( 577 None, 578 description="Error Message to display if the response matches the filter.", 579 title="Error Message", 580 ) 581 error_message_contains: Optional[str] = Field( 582 None, 583 description="Match the response if its error message contains the substring.", 584 example=["This API operation is not enabled for this site"], 585 title="Error Message Substring", 586 ) 587 http_codes: Optional[List[int]] = Field( 588 None, 589 description="Match the response if its HTTP code is included in this list.", 590 examples=[[420, 429], [500]], 591 title="HTTP Codes", 592 unique_items=True, 593 ) 594 predicate: Optional[str] = Field( 595 None, 596 description="Match the response if the predicate evaluates to true.", 597 examples=[ 598 "{{ 'Too much requests' in response }}", 599 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 600 ], 601 title="Predicate", 602 ) 603 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 604 605 606class ComplexFieldType(BaseModel): 607 field_type: str 608 items: Optional[Union[str, ComplexFieldType]] = None 609 610 611class TypesMap(BaseModel): 612 target_type: Union[str, List[str], ComplexFieldType] 613 current_type: Union[str, List[str]] 614 condition: Optional[str] = None 615 616 617class SchemaTypeIdentifier(BaseModel): 618 type: Optional[Literal["SchemaTypeIdentifier"]] = None 619 schema_pointer: Optional[List[str]] = Field( 620 [], 621 description="List of nested fields defining the schema field path to extract. Defaults to [].", 622 title="Schema Path", 623 ) 624 key_pointer: List[str] = Field( 625 ..., 626 description="List of potentially nested fields describing the full path of the field key to extract.", 627 title="Key Path", 628 ) 629 type_pointer: Optional[List[str]] = Field( 630 None, 631 description="List of potentially nested fields describing the full path of the field type to extract.", 632 title="Type Path", 633 ) 634 types_mapping: Optional[List[TypesMap]] = None 635 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 636 637 638class InlineSchemaLoader(BaseModel): 639 type: Literal["InlineSchemaLoader"] 640 schema_: Optional[Dict[str, Any]] = Field( 641 None, 642 alias="schema", 643 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 644 title="Schema", 645 ) 646 647 648class JsonFileSchemaLoader(BaseModel): 649 type: Literal["JsonFileSchemaLoader"] 650 file_path: Optional[str] = Field( 651 None, 652 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 653 example=["./schemas/users.json"], 654 title="File Path", 655 ) 656 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 657 658 659class JsonDecoder(BaseModel): 660 type: Literal["JsonDecoder"] 661 662 663class JsonlDecoder(BaseModel): 664 type: Literal["JsonlDecoder"] 665 666 667class KeysToLower(BaseModel): 668 type: Literal["KeysToLower"] 669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 670 671 672class KeysToSnakeCase(BaseModel): 673 type: Literal["KeysToSnakeCase"] 674 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 675 676 677class FlattenFields(BaseModel): 678 type: Literal["FlattenFields"] 679 flatten_lists: Optional[bool] = Field( 680 True, 681 description="Whether to flatten lists or leave it as is. Default is True.", 682 title="Flatten Lists", 683 ) 684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 685 686 687class KeyTransformation(BaseModel): 688 type: Literal["KeyTransformation"] 689 prefix: Optional[str] = Field( 690 None, 691 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 692 examples=["flattened_"], 693 title="Key Prefix", 694 ) 695 suffix: Optional[str] = Field( 696 None, 697 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 698 examples=["_flattened"], 699 title="Key Suffix", 700 ) 701 702 703class DpathFlattenFields(BaseModel): 704 type: Literal["DpathFlattenFields"] 705 field_path: List[str] = Field( 706 ..., 707 description="A path to field that needs to be flattened.", 708 examples=[["data"], ["data", "*", "field"]], 709 title="Field Path", 710 ) 711 delete_origin_value: Optional[bool] = Field( 712 None, 713 description="Whether to delete the origin value or keep it. Default is False.", 714 title="Delete Origin Value", 715 ) 716 replace_record: Optional[bool] = Field( 717 None, 718 description="Whether to replace the origin record or not. Default is False.", 719 title="Replace Origin Record", 720 ) 721 key_transformation: Optional[KeyTransformation] = Field( 722 None, 723 description="Transformation for object keys. If not provided, original key will be used.", 724 title="Key transformation", 725 ) 726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 727 728 729class KeysReplace(BaseModel): 730 type: Literal["KeysReplace"] 731 old: str = Field( 732 ..., 733 description="Old value to replace.", 734 examples=[ 735 " ", 736 "{{ record.id }}", 737 "{{ config['id'] }}", 738 "{{ stream_slice['id'] }}", 739 ], 740 title="Old value", 741 ) 742 new: str = Field( 743 ..., 744 description="New value to set.", 745 examples=[ 746 "_", 747 "{{ record.id }}", 748 "{{ config['id'] }}", 749 "{{ stream_slice['id'] }}", 750 ], 751 title="New value", 752 ) 753 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 754 755 756class IterableDecoder(BaseModel): 757 type: Literal["IterableDecoder"] 758 759 760class XmlDecoder(BaseModel): 761 type: Literal["XmlDecoder"] 762 763 764class CustomDecoder(BaseModel): 765 class Config: 766 extra = Extra.allow 767 768 type: Literal["CustomDecoder"] 769 class_name: str = Field( 770 ..., 771 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 772 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 773 title="Class Name", 774 ) 775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 776 777 778class MinMaxDatetime(BaseModel): 779 type: Literal["MinMaxDatetime"] 780 datetime: str = Field( 781 ..., 782 description="Datetime value.", 783 examples=[ 784 "2021-01-01", 785 "2021-01-01T00:00:00Z", 786 "{{ config['start_time'] }}", 787 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 788 ], 789 title="Datetime", 790 ) 791 datetime_format: Optional[str] = Field( 792 "", 793 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 794 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 795 title="Datetime Format", 796 ) 797 max_datetime: Optional[str] = Field( 798 None, 799 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 800 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 801 title="Max Datetime", 802 ) 803 min_datetime: Optional[str] = Field( 804 None, 805 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 806 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 807 title="Min Datetime", 808 ) 809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 810 811 812class NoAuth(BaseModel): 813 type: Literal["NoAuth"] 814 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 815 816 817class NoPagination(BaseModel): 818 type: Literal["NoPagination"] 819 820 821class State(BaseModel): 822 class Config: 823 extra = Extra.allow 824 825 min: int 826 max: int 827 828 829class OAuthScope(BaseModel): 830 class Config: 831 extra = Extra.allow 832 833 scope: str = Field( 834 ..., 835 description="The OAuth scope string to request from the provider.", 836 ) 837 838 839class OauthConnectorInputSpecification(BaseModel): 840 class Config: 841 extra = Extra.allow 842 843 consent_url: str = Field( 844 ..., 845 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 846 examples=[ 847 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 848 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 849 ], 850 title="Consent URL", 851 ) 852 scope: Optional[str] = Field( 853 None, 854 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 855 examples=["user:read user:read_orders workspaces:read"], 856 title="Scopes", 857 ) 858 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 859 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 860 # The CDK schema defines the manifest contract; the platform reads these fields 861 # during the OAuth consent flow to build the authorization URL. 862 scopes: Optional[List[OAuthScope]] = Field( 863 None, 864 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 865 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 866 title="Scopes", 867 ) 868 optional_scopes: Optional[List[OAuthScope]] = Field( 869 None, 870 description="Optional OAuth scope objects that may or may not be granted.", 871 examples=[[{"scope": "admin:read"}]], 872 title="Optional Scopes", 873 ) 874 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 875 ScopesJoinStrategy.space, 876 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 877 title="Scopes Join Strategy", 878 ) 879 access_token_url: str = Field( 880 ..., 881 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 882 examples=[ 883 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 884 ], 885 title="Access Token URL", 886 ) 887 access_token_headers: Optional[Dict[str, Any]] = Field( 888 None, 889 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 890 examples=[ 891 { 892 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 893 } 894 ], 895 title="Access Token Headers", 896 ) 897 access_token_params: Optional[Dict[str, Any]] = Field( 898 None, 899 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 900 examples=[ 901 { 902 "{{ auth_code_key }}": "{{ auth_code_value }}", 903 "{{ client_id_key }}": "{{ client_id_value }}", 904 "{{ client_secret_key }}": "{{ client_secret_value }}", 905 } 906 ], 907 title="Access Token Query Params (Json Encoded)", 908 ) 909 extract_output: Optional[List[str]] = Field( 910 None, 911 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 912 examples=[["access_token", "refresh_token", "other_field"]], 913 title="Extract Output", 914 ) 915 state: Optional[State] = Field( 916 None, 917 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 918 examples=[{"min": 7, "max": 128}], 919 title="Configurable State Query Param", 920 ) 921 client_id_key: Optional[str] = Field( 922 None, 923 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 924 examples=["my_custom_client_id_key_name"], 925 title="Client ID Key Override", 926 ) 927 client_secret_key: Optional[str] = Field( 928 None, 929 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 930 examples=["my_custom_client_secret_key_name"], 931 title="Client Secret Key Override", 932 ) 933 scope_key: Optional[str] = Field( 934 None, 935 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 936 examples=["my_custom_scope_key_key_name"], 937 title="Scopes Key Override", 938 ) 939 state_key: Optional[str] = Field( 940 None, 941 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 942 examples=["my_custom_state_key_key_name"], 943 title="State Key Override", 944 ) 945 auth_code_key: Optional[str] = Field( 946 None, 947 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 948 examples=["my_custom_auth_code_key_name"], 949 title="Auth Code Key Override", 950 ) 951 redirect_uri_key: Optional[str] = Field( 952 None, 953 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 954 examples=["my_custom_redirect_uri_key_name"], 955 title="Redirect URI Key Override", 956 ) 957 958 959class OAuthConfigSpecification(BaseModel): 960 class Config: 961 extra = Extra.allow 962 963 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 964 None, 965 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 966 examples=[ 967 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 968 { 969 "app_id": { 970 "type": "string", 971 "path_in_connector_config": ["info", "app_id"], 972 } 973 }, 974 ], 975 title="OAuth user input", 976 ) 977 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 978 None, 979 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 980 title="DeclarativeOAuth Connector Specification", 981 ) 982 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 983 None, 984 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 985 examples=[ 986 { 987 "refresh_token": { 988 "type": "string,", 989 "path_in_connector_config": ["credentials", "refresh_token"], 990 } 991 } 992 ], 993 title="OAuth output specification", 994 ) 995 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 996 None, 997 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 998 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 999 title="OAuth input specification", 1000 ) 1001 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1002 None, 1003 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1004 examples=[ 1005 { 1006 "client_id": { 1007 "type": "string,", 1008 "path_in_connector_config": ["credentials", "client_id"], 1009 }, 1010 "client_secret": { 1011 "type": "string,", 1012 "path_in_connector_config": ["credentials", "client_secret"], 1013 }, 1014 } 1015 ], 1016 title="OAuth server output specification", 1017 ) 1018 1019 1020class OffsetIncrement(BaseModel): 1021 type: Literal["OffsetIncrement"] 1022 page_size: Optional[Union[int, str]] = Field( 1023 None, 1024 description="The number of records to include in each pages.", 1025 examples=[100, "{{ config['page_size'] }}"], 1026 title="Limit", 1027 ) 1028 inject_on_first_request: Optional[bool] = Field( 1029 False, 1030 description="Using the `offset` with value `0` during the first request", 1031 title="Inject Offset on First Request", 1032 ) 1033 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1034 1035 1036class PageIncrement(BaseModel): 1037 type: Literal["PageIncrement"] 1038 page_size: Optional[Union[int, str]] = Field( 1039 None, 1040 description="The number of records to include in each pages.", 1041 examples=[100, "100", "{{ config['page_size'] }}"], 1042 title="Page Size", 1043 ) 1044 start_from_page: Optional[int] = Field( 1045 0, 1046 description="Index of the first page to request.", 1047 examples=[0, 1], 1048 title="Start From Page", 1049 ) 1050 inject_on_first_request: Optional[bool] = Field( 1051 False, 1052 description="Using the `page number` with value defined by `start_from_page` during the first request", 1053 title="Inject Page Number on First Request", 1054 ) 1055 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1056 1057 1058class PrimaryKey(BaseModel): 1059 __root__: Union[str, List[str], List[List[str]]] = Field( 1060 ..., 1061 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1062 examples=["id", ["code", "type"]], 1063 title="Primary Key", 1064 ) 1065 1066 1067class PropertyLimitType(Enum): 1068 characters = "characters" 1069 property_count = "property_count" 1070 1071 1072class PropertyChunking(BaseModel): 1073 type: Literal["PropertyChunking"] 1074 property_limit_type: PropertyLimitType = Field( 1075 ..., 1076 description="The type used to determine the maximum number of properties per chunk", 1077 title="Property Limit Type", 1078 ) 1079 property_limit: Optional[int] = Field( 1080 None, 1081 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1082 title="Property Limit", 1083 ) 1084 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1085 None, 1086 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1087 title="Record Merge Strategy", 1088 ) 1089 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1090 1091 1092class RecordFilter(BaseModel): 1093 type: Literal["RecordFilter"] 1094 condition: Optional[str] = Field( 1095 "", 1096 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1097 examples=[ 1098 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1099 "{{ record.status in ['active', 'expired'] }}", 1100 ], 1101 title="Condition", 1102 ) 1103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1104 1105 1106class SchemaNormalization(Enum): 1107 Default = "Default" 1108 None_ = "None" 1109 1110 1111class RemoveFields(BaseModel): 1112 type: Literal["RemoveFields"] 1113 condition: Optional[str] = Field( 1114 "", 1115 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1116 examples=[ 1117 "{{ property|string == '' }}", 1118 "{{ property is integer }}", 1119 "{{ property|length > 5 }}", 1120 "{{ property == 'some_string_to_match' }}", 1121 ], 1122 ) 1123 field_pointers: List[List[str]] = Field( 1124 ..., 1125 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1126 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1127 title="Field Paths", 1128 ) 1129 1130 1131class RequestPath(BaseModel): 1132 type: Literal["RequestPath"] 1133 1134 1135class InjectInto(Enum): 1136 request_parameter = "request_parameter" 1137 header = "header" 1138 body_data = "body_data" 1139 body_json = "body_json" 1140 1141 1142class RequestOption(BaseModel): 1143 type: Literal["RequestOption"] 1144 inject_into: InjectInto = Field( 1145 ..., 1146 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1147 examples=["request_parameter", "header", "body_data", "body_json"], 1148 title="Inject Into", 1149 ) 1150 field_name: Optional[str] = Field( 1151 None, 1152 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1153 examples=["segment_id"], 1154 title="Field Name", 1155 ) 1156 field_path: Optional[List[str]] = Field( 1157 None, 1158 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1159 examples=[["data", "viewer", "id"]], 1160 title="Field Path", 1161 ) 1162 1163 1164class Schemas(BaseModel): 1165 pass 1166 1167 class Config: 1168 extra = Extra.allow 1169 1170 1171class LegacySessionTokenAuthenticator(BaseModel): 1172 type: Literal["LegacySessionTokenAuthenticator"] 1173 header: str = Field( 1174 ..., 1175 description="The name of the session token header that will be injected in the request", 1176 examples=["X-Session"], 1177 title="Session Request Header", 1178 ) 1179 login_url: str = Field( 1180 ..., 1181 description="Path of the login URL (do not include the base URL)", 1182 examples=["session"], 1183 title="Login Path", 1184 ) 1185 session_token: Optional[str] = Field( 1186 None, 1187 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1188 example=["{{ config['session_token'] }}"], 1189 title="Session Token", 1190 ) 1191 session_token_response_key: str = Field( 1192 ..., 1193 description="Name of the key of the session token to be extracted from the response", 1194 examples=["id"], 1195 title="Response Token Response Key", 1196 ) 1197 username: Optional[str] = Field( 1198 None, 1199 description="Username used to authenticate and obtain a session token", 1200 examples=[" {{ config['username'] }}"], 1201 title="Username", 1202 ) 1203 password: Optional[str] = Field( 1204 "", 1205 description="Password used to authenticate and obtain a session token", 1206 examples=["{{ config['password'] }}", ""], 1207 title="Password", 1208 ) 1209 validate_session_url: str = Field( 1210 ..., 1211 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1212 examples=["user/current"], 1213 title="Validate Session Path", 1214 ) 1215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1216 1217 1218class Action1(Enum): 1219 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1220 RESET = "RESET" 1221 1222 1223class PaginationResetLimits(BaseModel): 1224 type: Literal["PaginationResetLimits"] 1225 number_of_records: Optional[int] = None 1226 1227 1228class CsvDecoder(BaseModel): 1229 type: Literal["CsvDecoder"] 1230 encoding: Optional[str] = "utf-8" 1231 delimiter: Optional[str] = "," 1232 set_values_to_none: Optional[List[str]] = None 1233 1234 1235class AsyncJobStatusMap(BaseModel): 1236 type: Optional[Literal["AsyncJobStatusMap"]] = None 1237 running: List[str] 1238 completed: List[str] 1239 failed: List[str] 1240 timeout: List[str] 1241 skipped: Optional[List[str]] = None 1242 1243 1244class ValueType(Enum): 1245 string = "string" 1246 number = "number" 1247 integer = "integer" 1248 boolean = "boolean" 1249 1250 1251class WaitTimeFromHeader(BaseModel): 1252 type: Literal["WaitTimeFromHeader"] 1253 header: str = Field( 1254 ..., 1255 description="The name of the response header defining how long to wait before retrying.", 1256 examples=["Retry-After"], 1257 title="Response Header Name", 1258 ) 1259 regex: Optional[str] = Field( 1260 None, 1261 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1262 examples=["([-+]?\\d+)"], 1263 title="Extraction Regex", 1264 ) 1265 max_waiting_time_in_seconds: Optional[float] = Field( 1266 None, 1267 description="Given the value extracted from the header is greater than this value, stop the stream.", 1268 examples=[3600], 1269 title="Max Waiting Time in Seconds", 1270 ) 1271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1272 1273 1274class WaitUntilTimeFromHeader(BaseModel): 1275 type: Literal["WaitUntilTimeFromHeader"] 1276 header: str = Field( 1277 ..., 1278 description="The name of the response header defining how long to wait before retrying.", 1279 examples=["wait_time"], 1280 title="Response Header", 1281 ) 1282 min_wait: Optional[Union[float, str]] = Field( 1283 None, 1284 description="Minimum time to wait before retrying.", 1285 examples=[10, "60"], 1286 title="Minimum Wait Time", 1287 ) 1288 regex: Optional[str] = Field( 1289 None, 1290 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1291 examples=["([-+]?\\d+)"], 1292 title="Extraction Regex", 1293 ) 1294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1295 1296 1297class ComponentMappingDefinition(BaseModel): 1298 type: Literal["ComponentMappingDefinition"] 1299 field_path: List[str] = Field( 1300 ..., 1301 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1302 examples=[ 1303 ["name"], 1304 ["retriever", "requester", "url"], 1305 ["retriever", "requester", "{{ components_values.field }}"], 1306 ["*", "**", "name"], 1307 ], 1308 title="Field Path", 1309 ) 1310 value: str = Field( 1311 ..., 1312 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1313 examples=[ 1314 "{{ components_values['updates'] }}", 1315 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1316 "{{ config['segment_id'] }}", 1317 "{{ stream_slice['parent_id'] }}", 1318 "{{ stream_slice['extra_fields']['name'] }}", 1319 ], 1320 title="Value", 1321 ) 1322 value_type: Optional[ValueType] = Field( 1323 None, 1324 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1325 title="Value Type", 1326 ) 1327 create_or_update: Optional[bool] = Field( 1328 False, 1329 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1330 title="Create or Update", 1331 ) 1332 condition: Optional[str] = Field( 1333 None, 1334 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1335 examples=[ 1336 "{{ components_values.get('cursor_field', None) }}", 1337 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1338 ], 1339 title="Condition", 1340 ) 1341 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1342 1343 1344class StreamConfig(BaseModel): 1345 type: Literal["StreamConfig"] 1346 configs_pointer: List[str] = Field( 1347 ..., 1348 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1349 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1350 title="Configs Pointer", 1351 ) 1352 default_values: Optional[List[Dict[str, Any]]] = Field( 1353 None, 1354 description="A list of default values, each matching the structure expected from the parsed component value.", 1355 title="Default Values", 1356 ) 1357 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1358 1359 1360class ConfigComponentsResolver(BaseModel): 1361 type: Literal["ConfigComponentsResolver"] 1362 stream_config: Union[List[StreamConfig], StreamConfig] 1363 components_mapping: List[ComponentMappingDefinition] 1364 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1365 1366 1367class StreamParametersDefinition(BaseModel): 1368 type: Literal["StreamParametersDefinition"] 1369 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1370 ..., 1371 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1372 examples=[ 1373 [ 1374 { 1375 "name": "test stream", 1376 "$parameters": {"entity": "test entity"}, 1377 "primary_key": "test key", 1378 } 1379 ] 1380 ], 1381 title="Stream Parameters", 1382 ) 1383 1384 1385class ParametrizedComponentsResolver(BaseModel): 1386 type: Literal["ParametrizedComponentsResolver"] 1387 stream_parameters: StreamParametersDefinition 1388 components_mapping: List[ComponentMappingDefinition] 1389 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1390 1391 1392class RequestBodyPlainText(BaseModel): 1393 type: Literal["RequestBodyPlainText"] 1394 value: str 1395 1396 1397class RequestBodyUrlEncodedForm(BaseModel): 1398 type: Literal["RequestBodyUrlEncodedForm"] 1399 value: Dict[str, str] 1400 1401 1402class RequestBodyJsonObject(BaseModel): 1403 type: Literal["RequestBodyJsonObject"] 1404 value: Dict[str, Any] 1405 1406 1407class RequestBodyGraphQlQuery(BaseModel): 1408 class Config: 1409 extra = Extra.allow 1410 1411 query: str = Field(..., description="The GraphQL query to be executed") 1412 1413 1414class ValidateAdheresToSchema(BaseModel): 1415 type: Literal["ValidateAdheresToSchema"] 1416 base_schema: Union[str, Dict[str, Any]] = Field( 1417 ..., 1418 description="The base JSON schema against which the user-provided schema will be validated.", 1419 examples=[ 1420 "{{ config['report_validation_schema'] }}", 1421 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1422 { 1423 "$schema": "http://json-schema.org/draft-07/schema#", 1424 "title": "Person", 1425 "type": "object", 1426 "properties": { 1427 "name": {"type": "string", "description": "The person's name"}, 1428 "age": { 1429 "type": "integer", 1430 "minimum": 0, 1431 "description": "The person's age", 1432 }, 1433 }, 1434 "required": ["name", "age"], 1435 }, 1436 ], 1437 title="Base JSON Schema", 1438 ) 1439 1440 1441class CustomValidationStrategy(BaseModel): 1442 class Config: 1443 extra = Extra.allow 1444 1445 type: Literal["CustomValidationStrategy"] 1446 class_name: str = Field( 1447 ..., 1448 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1449 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1450 title="Class Name", 1451 ) 1452 1453 1454class ConfigRemapField(BaseModel): 1455 type: Literal["ConfigRemapField"] 1456 map: Union[Dict[str, Any], str] = Field( 1457 ..., 1458 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1459 examples=[ 1460 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1461 "{{ config['status_mapping'] }}", 1462 ], 1463 title="Value Mapping", 1464 ) 1465 field_path: List[str] = Field( 1466 ..., 1467 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1468 examples=[ 1469 ["status"], 1470 ["data", "status"], 1471 ["data", "{{ config.name }}", "status"], 1472 ["data", "*", "status"], 1473 ], 1474 title="Field Path", 1475 ) 1476 1477 1478class ConfigRemoveFields(BaseModel): 1479 type: Literal["ConfigRemoveFields"] 1480 field_pointers: List[List[str]] = Field( 1481 ..., 1482 description="A list of field pointers to be removed from the config.", 1483 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1484 title="Field Pointers", 1485 ) 1486 condition: Optional[str] = Field( 1487 "", 1488 description="Fields will be removed if expression is evaluated to True.", 1489 examples=[ 1490 "{{ config['environemnt'] == 'sandbox' }}", 1491 "{{ property is integer }}", 1492 "{{ property|length > 5 }}", 1493 "{{ property == 'some_string_to_match' }}", 1494 ], 1495 ) 1496 1497 1498class CustomConfigTransformation(BaseModel): 1499 type: Literal["CustomConfigTransformation"] 1500 class_name: str = Field( 1501 ..., 1502 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1503 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field( 1506 None, 1507 alias="$parameters", 1508 description="Additional parameters to be passed to the custom config transformation.", 1509 ) 1510 1511 1512class AddedFieldDefinition(BaseModel): 1513 type: Literal["AddedFieldDefinition"] 1514 path: List[str] = Field( 1515 ..., 1516 description="List of strings defining the path where to add the value on the record.", 1517 examples=[["segment_id"], ["metadata", "segment_id"]], 1518 title="Path", 1519 ) 1520 value: str = Field( 1521 ..., 1522 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1523 examples=[ 1524 "{{ record['updates'] }}", 1525 "{{ record['MetaData']['LastUpdatedTime'] }}", 1526 "{{ stream_partition['segment_id'] }}", 1527 ], 1528 title="Value", 1529 ) 1530 value_type: Optional[ValueType] = Field( 1531 None, 1532 description="Type of the value. If not specified, the type will be inferred from the value.", 1533 title="Value Type", 1534 ) 1535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1536 1537 1538class AddFields(BaseModel): 1539 type: Literal["AddFields"] 1540 fields: List[AddedFieldDefinition] = Field( 1541 ..., 1542 description="List of transformations (path and corresponding value) that will be added to the record.", 1543 title="Fields", 1544 ) 1545 condition: Optional[str] = Field( 1546 "", 1547 description="Fields will be added if expression is evaluated to True.", 1548 examples=[ 1549 "{{ property|string == '' }}", 1550 "{{ property is integer }}", 1551 "{{ property|length > 5 }}", 1552 "{{ property == 'some_string_to_match' }}", 1553 ], 1554 ) 1555 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1556 1557 1558class ApiKeyAuthenticator(BaseModel): 1559 type: Literal["ApiKeyAuthenticator"] 1560 api_token: Optional[str] = Field( 1561 None, 1562 description="The API key to inject in the request. Fill it in the user inputs.", 1563 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1564 title="API Key", 1565 ) 1566 header: Optional[str] = Field( 1567 None, 1568 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1569 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1570 title="Header Name", 1571 ) 1572 inject_into: Optional[RequestOption] = Field( 1573 None, 1574 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1575 examples=[ 1576 {"inject_into": "header", "field_name": "Authorization"}, 1577 {"inject_into": "request_parameter", "field_name": "authKey"}, 1578 ], 1579 title="Inject API Key Into Outgoing HTTP Request", 1580 ) 1581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1582 1583 1584class AuthFlow(BaseModel): 1585 auth_flow_type: Optional[AuthFlowType] = Field( 1586 None, description="The type of auth to use", title="Auth flow type" 1587 ) 1588 predicate_key: Optional[List[str]] = Field( 1589 None, 1590 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1591 examples=[["credentials", "auth_type"]], 1592 title="Predicate key", 1593 ) 1594 predicate_value: Optional[str] = Field( 1595 None, 1596 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1597 examples=["Oauth"], 1598 title="Predicate value", 1599 ) 1600 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1601 1602 1603class CheckStream(BaseModel): 1604 type: Literal["CheckStream"] 1605 stream_names: Optional[List[str]] = Field( 1606 None, 1607 description="Names of the streams to try reading from when running a check operation.", 1608 examples=[["users"], ["users", "contacts"]], 1609 title="Stream Names", 1610 ) 1611 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1612 1613 1614class IncrementingCountCursor(BaseModel): 1615 type: Literal["IncrementingCountCursor"] 1616 cursor_field: str = Field( 1617 ..., 1618 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1619 examples=["created_at", "{{ config['record_cursor'] }}"], 1620 title="Cursor Field", 1621 ) 1622 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1623 None, 1624 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1625 title="Allow Catalog Defined Cursor Field", 1626 ) 1627 start_value: Optional[Union[str, int]] = Field( 1628 None, 1629 description="The value that determines the earliest record that should be synced.", 1630 examples=[0, "{{ config['start_value'] }}"], 1631 title="Start Value", 1632 ) 1633 start_value_option: Optional[RequestOption] = Field( 1634 None, 1635 description="Optionally configures how the start value will be sent in requests to the source API.", 1636 title="Inject Start Value Into Outgoing HTTP Request", 1637 ) 1638 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1639 1640 1641class DatetimeBasedCursor(BaseModel): 1642 type: Literal["DatetimeBasedCursor"] 1643 clamping: Optional[Clamping] = Field( 1644 None, 1645 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1646 title="Date Range Clamping", 1647 ) 1648 cursor_field: str = Field( 1649 ..., 1650 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1651 examples=["created_at", "{{ config['record_cursor'] }}"], 1652 title="Cursor Field", 1653 ) 1654 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1655 None, 1656 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1657 title="Allow Catalog Defined Cursor Field", 1658 ) 1659 cursor_datetime_formats: Optional[List[str]] = Field( 1660 None, 1661 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1662 examples=[ 1663 "%Y-%m-%d", 1664 "%Y-%m-%d %H:%M:%S", 1665 "%Y-%m-%dT%H:%M:%S", 1666 "%Y-%m-%dT%H:%M:%SZ", 1667 "%Y-%m-%dT%H:%M:%S%z", 1668 "%Y-%m-%dT%H:%M:%S.%fZ", 1669 "%Y-%m-%dT%H:%M:%S.%f%z", 1670 "%Y-%m-%d %H:%M:%S.%f+00:00", 1671 "%s", 1672 "%ms", 1673 ], 1674 title="Cursor Datetime Formats", 1675 ) 1676 start_datetime: Union[MinMaxDatetime, str] = Field( 1677 ..., 1678 description="The datetime that determines the earliest record that should be synced.", 1679 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1680 title="Start Datetime", 1681 ) 1682 start_time_option: Optional[RequestOption] = Field( 1683 None, 1684 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1685 title="Inject Start Time Into Outgoing HTTP Request", 1686 ) 1687 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1688 None, 1689 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1690 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1691 title="End Datetime", 1692 ) 1693 end_time_option: Optional[RequestOption] = Field( 1694 None, 1695 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1696 title="Inject End Time Into Outgoing HTTP Request", 1697 ) 1698 datetime_format: str = Field( 1699 ..., 1700 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1701 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1702 title="Outgoing Datetime Format", 1703 ) 1704 cursor_granularity: Optional[str] = Field( 1705 None, 1706 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1707 examples=["PT1S"], 1708 title="Cursor Granularity", 1709 ) 1710 is_data_feed: Optional[bool] = Field( 1711 None, 1712 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1713 title="Data Feed API", 1714 ) 1715 is_client_side_incremental: Optional[bool] = Field( 1716 None, 1717 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1718 title="Client-side Incremental Filtering", 1719 ) 1720 is_compare_strictly: Optional[bool] = Field( 1721 False, 1722 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1723 title="Strict Start-End Time Comparison", 1724 ) 1725 global_substream_cursor: Optional[bool] = Field( 1726 False, 1727 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1728 title="Global Substream Cursor", 1729 ) 1730 lookback_window: Optional[str] = Field( 1731 None, 1732 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1733 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1734 title="Lookback Window", 1735 ) 1736 partition_field_end: Optional[str] = Field( 1737 None, 1738 description="Name of the partition start time field.", 1739 examples=["ending_time"], 1740 title="Partition Field End", 1741 ) 1742 partition_field_start: Optional[str] = Field( 1743 None, 1744 description="Name of the partition end time field.", 1745 examples=["starting_time"], 1746 title="Partition Field Start", 1747 ) 1748 step: Optional[str] = Field( 1749 None, 1750 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1751 examples=["P1W", "{{ config['step_increment'] }}"], 1752 title="Step", 1753 ) 1754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1755 1756 1757class JwtAuthenticator(BaseModel): 1758 type: Literal["JwtAuthenticator"] 1759 secret_key: str = Field( 1760 ..., 1761 description="Secret used to sign the JSON web token.", 1762 examples=["{{ config['secret_key'] }}"], 1763 title="Secret Key", 1764 ) 1765 base64_encode_secret_key: Optional[bool] = Field( 1766 False, 1767 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1768 title="Base64-encode Secret Key", 1769 ) 1770 algorithm: Algorithm = Field( 1771 ..., 1772 description="Algorithm used to sign the JSON web token.", 1773 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1774 title="Algorithm", 1775 ) 1776 token_duration: Optional[int] = Field( 1777 1200, 1778 description="The amount of time in seconds a JWT token can be valid after being issued.", 1779 examples=[1200, 3600], 1780 title="Token Duration", 1781 ) 1782 header_prefix: Optional[str] = Field( 1783 None, 1784 description="The prefix to be used within the Authentication header.", 1785 examples=["Bearer", "Basic"], 1786 title="Header Prefix", 1787 ) 1788 jwt_headers: Optional[JwtHeaders] = Field( 1789 None, 1790 description="JWT headers used when signing JSON web token.", 1791 title="JWT Headers", 1792 ) 1793 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1794 None, 1795 description="Additional headers to be included with the JWT headers object.", 1796 title="Additional JWT Headers", 1797 ) 1798 jwt_payload: Optional[JwtPayload] = Field( 1799 None, 1800 description="JWT Payload used when signing JSON web token.", 1801 title="JWT Payload", 1802 ) 1803 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1804 None, 1805 description="Additional properties to be added to the JWT payload.", 1806 title="Additional JWT Payload Properties", 1807 ) 1808 passphrase: Optional[str] = Field( 1809 None, 1810 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1811 examples=["{{ config['passphrase'] }}"], 1812 title="Passphrase", 1813 ) 1814 request_option: Optional[RequestOption] = Field( 1815 None, 1816 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1817 title="Request Option", 1818 ) 1819 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1820 1821 1822class OAuthAuthenticator(BaseModel): 1823 type: Literal["OAuthAuthenticator"] 1824 client_id_name: Optional[str] = Field( 1825 "client_id", 1826 description="The name of the property to use to refresh the `access_token`.", 1827 examples=["custom_app_id"], 1828 title="Client ID Property Name", 1829 ) 1830 client_id: Optional[str] = Field( 1831 None, 1832 description="The OAuth client ID. Fill it in the user inputs.", 1833 examples=[ 1834 "{{ config['client_id'] }}", 1835 "{{ config['credentials']['client_id }}", 1836 ], 1837 title="Client ID", 1838 ) 1839 client_secret_name: Optional[str] = Field( 1840 "client_secret", 1841 description="The name of the property to use to refresh the `access_token`.", 1842 examples=["custom_app_secret"], 1843 title="Client Secret Property Name", 1844 ) 1845 client_secret: Optional[str] = Field( 1846 None, 1847 description="The OAuth client secret. Fill it in the user inputs.", 1848 examples=[ 1849 "{{ config['client_secret'] }}", 1850 "{{ config['credentials']['client_secret }}", 1851 ], 1852 title="Client Secret", 1853 ) 1854 refresh_token_name: Optional[str] = Field( 1855 "refresh_token", 1856 description="The name of the property to use to refresh the `access_token`.", 1857 examples=["custom_app_refresh_value"], 1858 title="Refresh Token Property Name", 1859 ) 1860 refresh_token: Optional[str] = Field( 1861 None, 1862 description="Credential artifact used to get a new access token.", 1863 examples=[ 1864 "{{ config['refresh_token'] }}", 1865 "{{ config['credentials]['refresh_token'] }}", 1866 ], 1867 title="Refresh Token", 1868 ) 1869 token_refresh_endpoint: Optional[str] = Field( 1870 None, 1871 description="The full URL to call to obtain a new access token.", 1872 examples=["https://connect.squareup.com/oauth2/token"], 1873 title="Token Refresh Endpoint", 1874 ) 1875 access_token_name: Optional[str] = Field( 1876 "access_token", 1877 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1878 examples=["access_token"], 1879 title="Access Token Property Name", 1880 ) 1881 access_token_value: Optional[str] = Field( 1882 None, 1883 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1884 examples=["secret_access_token_value"], 1885 title="Access Token Value", 1886 ) 1887 expires_in_name: Optional[str] = Field( 1888 "expires_in", 1889 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1890 examples=["expires_in"], 1891 title="Token Expiry Property Name", 1892 ) 1893 grant_type_name: Optional[str] = Field( 1894 "grant_type", 1895 description="The name of the property to use to refresh the `access_token`.", 1896 examples=["custom_grant_type"], 1897 title="Grant Type Property Name", 1898 ) 1899 grant_type: Optional[str] = Field( 1900 "refresh_token", 1901 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1902 examples=["refresh_token", "client_credentials"], 1903 title="Grant Type", 1904 ) 1905 refresh_request_body: Optional[Dict[str, Any]] = Field( 1906 None, 1907 description="Body of the request sent to get a new access token.", 1908 examples=[ 1909 { 1910 "applicationId": "{{ config['application_id'] }}", 1911 "applicationSecret": "{{ config['application_secret'] }}", 1912 "token": "{{ config['token'] }}", 1913 } 1914 ], 1915 title="Refresh Request Body", 1916 ) 1917 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1918 None, 1919 description="Headers of the request sent to get a new access token.", 1920 examples=[ 1921 { 1922 "Authorization": "<AUTH_TOKEN>", 1923 "Content-Type": "application/x-www-form-urlencoded", 1924 } 1925 ], 1926 title="Refresh Request Headers", 1927 ) 1928 scopes: Optional[List[str]] = Field( 1929 None, 1930 description="List of scopes that should be granted to the access token.", 1931 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1932 title="Scopes", 1933 ) 1934 token_expiry_date: Optional[str] = Field( 1935 None, 1936 description="The access token expiry date.", 1937 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1938 title="Token Expiry Date", 1939 ) 1940 token_expiry_date_format: Optional[str] = Field( 1941 None, 1942 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1943 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1944 title="Token Expiry Date Format", 1945 ) 1946 refresh_token_error_status_codes: Optional[List[int]] = Field( 1947 None, 1948 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1949 examples=[[400, 500]], 1950 title="Refresh Token Error Status Codes", 1951 ) 1952 refresh_token_error_key: Optional[str] = Field( 1953 None, 1954 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1955 examples=["error"], 1956 title="Refresh Token Error Key", 1957 ) 1958 refresh_token_error_values: Optional[List[str]] = Field( 1959 None, 1960 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1961 examples=[["invalid_grant", "invalid_permissions"]], 1962 title="Refresh Token Error Values", 1963 ) 1964 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1965 None, 1966 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1967 title="Refresh Token Updater", 1968 ) 1969 profile_assertion: Optional[JwtAuthenticator] = Field( 1970 None, 1971 description="The authenticator being used to authenticate the client authenticator.", 1972 title="Profile Assertion", 1973 ) 1974 use_profile_assertion: Optional[bool] = Field( 1975 False, 1976 description="Enable using profile assertion as a flow for OAuth authorization.", 1977 title="Use Profile Assertion", 1978 ) 1979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1980 1981 1982class FixedWindowCallRatePolicy(BaseModel): 1983 class Config: 1984 extra = Extra.allow 1985 1986 type: Literal["FixedWindowCallRatePolicy"] 1987 period: str = Field( 1988 ..., description="The time interval for the rate limit window.", title="Period" 1989 ) 1990 call_limit: int = Field( 1991 ..., 1992 description="The maximum number of calls allowed within the period.", 1993 title="Call Limit", 1994 ) 1995 matchers: List[HttpRequestRegexMatcher] = Field( 1996 ..., 1997 description="List of matchers that define which requests this policy applies to.", 1998 title="Matchers", 1999 ) 2000 2001 2002class MovingWindowCallRatePolicy(BaseModel): 2003 class Config: 2004 extra = Extra.allow 2005 2006 type: Literal["MovingWindowCallRatePolicy"] 2007 rates: List[Rate] = Field( 2008 ..., 2009 description="List of rates that define the call limits for different time intervals.", 2010 title="Rates", 2011 ) 2012 matchers: List[HttpRequestRegexMatcher] = Field( 2013 ..., 2014 description="List of matchers that define which requests this policy applies to.", 2015 title="Matchers", 2016 ) 2017 2018 2019class UnlimitedCallRatePolicy(BaseModel): 2020 class Config: 2021 extra = Extra.allow 2022 2023 type: Literal["UnlimitedCallRatePolicy"] 2024 matchers: List[HttpRequestRegexMatcher] = Field( 2025 ..., 2026 description="List of matchers that define which requests this policy applies to.", 2027 title="Matchers", 2028 ) 2029 2030 2031class DefaultErrorHandler(BaseModel): 2032 type: Literal["DefaultErrorHandler"] 2033 backoff_strategies: Optional[ 2034 List[ 2035 Union[ 2036 ConstantBackoffStrategy, 2037 ExponentialBackoffStrategy, 2038 WaitTimeFromHeader, 2039 WaitUntilTimeFromHeader, 2040 CustomBackoffStrategy, 2041 ] 2042 ] 2043 ] = Field( 2044 None, 2045 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2046 title="Backoff Strategies", 2047 ) 2048 max_retries: Optional[Union[int, str]] = Field( 2049 5, 2050 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2051 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2052 title="Max Retry Count", 2053 ) 2054 response_filters: Optional[List[HttpResponseFilter]] = Field( 2055 None, 2056 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2057 title="Response Filters", 2058 ) 2059 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2060 2061 2062class DefaultPaginator(BaseModel): 2063 type: Literal["DefaultPaginator"] 2064 pagination_strategy: Union[ 2065 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2066 ] = Field( 2067 ..., 2068 description="Strategy defining how records are paginated.", 2069 title="Pagination Strategy", 2070 ) 2071 page_size_option: Optional[RequestOption] = Field( 2072 None, title="Inject Page Size Into Outgoing HTTP Request" 2073 ) 2074 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2075 None, 2076 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2077 title="Inject Page Token Into Outgoing HTTP Request", 2078 ) 2079 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2080 2081 2082class RecordExpander(BaseModel): 2083 type: Literal["RecordExpander"] 2084 expand_records_from_field: List[str] = Field( 2085 ..., 2086 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2087 examples=[ 2088 ["lines", "data"], 2089 ["items"], 2090 ["nested", "array"], 2091 ["sections", "*", "items"], 2092 ], 2093 title="Expand Records From Field", 2094 ) 2095 remain_original_record: Optional[bool] = Field( 2096 False, 2097 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2098 title="Remain Original Record", 2099 ) 2100 on_no_records: Optional[OnNoRecords] = Field( 2101 OnNoRecords.skip, 2102 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2103 title="On No Records", 2104 ) 2105 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2106 2107 2108class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2109 type: Literal["ApiKey"] 2110 inject_into: RequestOption = Field( 2111 ..., 2112 description="Configure how the API Key will be sent in requests to the source API.", 2113 examples=[ 2114 {"inject_into": "header", "field_name": "Authorization"}, 2115 {"inject_into": "request_parameter", "field_name": "authKey"}, 2116 ], 2117 title="Inject API Key Into Outgoing HTTP Request", 2118 ) 2119 api_token: Optional[str] = Field( 2120 "{{ session_token }}", 2121 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2122 examples=[ 2123 "{{ session_token }}", 2124 "Token {{ session_token }}", 2125 "Bearer {{ session_token }}", 2126 ], 2127 title="API Token Template", 2128 ) 2129 2130 2131class JsonSchemaPropertySelector(BaseModel): 2132 type: Literal["JsonSchemaPropertySelector"] 2133 transformations: Optional[ 2134 List[ 2135 Union[ 2136 AddFields, 2137 RemoveFields, 2138 KeysToLower, 2139 KeysToSnakeCase, 2140 FlattenFields, 2141 DpathFlattenFields, 2142 KeysReplace, 2143 CustomTransformation, 2144 ] 2145 ] 2146 ] = Field( 2147 None, 2148 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2149 title="Transformations", 2150 ) 2151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2152 2153 2154class ListPartitionRouter(BaseModel): 2155 type: Literal["ListPartitionRouter"] 2156 cursor_field: str = Field( 2157 ..., 2158 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2159 examples=["section", "{{ config['section_key'] }}"], 2160 title="Current Partition Value Identifier", 2161 ) 2162 values: Union[str, List[str]] = Field( 2163 ..., 2164 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2165 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2166 title="Partition Values", 2167 ) 2168 request_option: Optional[RequestOption] = Field( 2169 None, 2170 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2171 title="Inject Partition Value Into Outgoing HTTP Request", 2172 ) 2173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2174 2175 2176class PaginationReset(BaseModel): 2177 type: Literal["PaginationReset"] 2178 action: Action1 2179 limits: Optional[PaginationResetLimits] = None 2180 2181 2182class GzipDecoder(BaseModel): 2183 type: Literal["GzipDecoder"] 2184 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2185 2186 2187class RequestBodyGraphQL(BaseModel): 2188 type: Literal["RequestBodyGraphQL"] 2189 value: RequestBodyGraphQlQuery 2190 2191 2192class DpathValidator(BaseModel): 2193 type: Literal["DpathValidator"] 2194 field_path: List[str] = Field( 2195 ..., 2196 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2197 examples=[ 2198 ["data"], 2199 ["data", "records"], 2200 ["data", "{{ parameters.name }}"], 2201 ["data", "*", "record"], 2202 ], 2203 title="Field Path", 2204 ) 2205 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2206 ..., 2207 description="The condition that the specified config value will be evaluated against", 2208 title="Validation Strategy", 2209 ) 2210 2211 2212class PredicateValidator(BaseModel): 2213 type: Literal["PredicateValidator"] 2214 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2215 ..., 2216 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2217 examples=[ 2218 "test-value", 2219 "{{ config['api_version'] }}", 2220 "{{ config['tenant_id'] }}", 2221 123, 2222 ], 2223 title="Value", 2224 ) 2225 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2226 ..., 2227 description="The validation strategy to apply to the value.", 2228 title="Validation Strategy", 2229 ) 2230 2231 2232class ConfigAddFields(BaseModel): 2233 type: Literal["ConfigAddFields"] 2234 fields: List[AddedFieldDefinition] = Field( 2235 ..., 2236 description="A list of transformations (path and corresponding value) that will be added to the config.", 2237 title="Fields", 2238 ) 2239 condition: Optional[str] = Field( 2240 "", 2241 description="Fields will be added if expression is evaluated to True.", 2242 examples=[ 2243 "{{ config['environemnt'] == 'sandbox' }}", 2244 "{{ property is integer }}", 2245 "{{ property|length > 5 }}", 2246 "{{ property == 'some_string_to_match' }}", 2247 ], 2248 ) 2249 2250 2251class CompositeErrorHandler(BaseModel): 2252 type: Literal["CompositeErrorHandler"] 2253 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2254 Field( 2255 ..., 2256 description="List of error handlers to iterate on to determine how to handle a failed response.", 2257 title="Error Handlers", 2258 ) 2259 ) 2260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2261 2262 2263class HTTPAPIBudget(BaseModel): 2264 class Config: 2265 extra = Extra.allow 2266 2267 type: Literal["HTTPAPIBudget"] 2268 policies: List[ 2269 Union[ 2270 FixedWindowCallRatePolicy, 2271 MovingWindowCallRatePolicy, 2272 UnlimitedCallRatePolicy, 2273 ] 2274 ] = Field( 2275 ..., 2276 description="List of call rate policies that define how many calls are allowed.", 2277 title="Policies", 2278 ) 2279 ratelimit_reset_header: Optional[str] = Field( 2280 "ratelimit-reset", 2281 description="The HTTP response header name that indicates when the rate limit resets.", 2282 title="Rate Limit Reset Header", 2283 ) 2284 ratelimit_remaining_header: Optional[str] = Field( 2285 "ratelimit-remaining", 2286 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2287 title="Rate Limit Remaining Header", 2288 ) 2289 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2290 [429], 2291 description="List of HTTP status codes that indicate a rate limit has been hit.", 2292 title="Status Codes for Rate Limit Hit", 2293 ) 2294 2295 2296class DpathExtractor(BaseModel): 2297 type: Literal["DpathExtractor"] 2298 field_path: List[str] = Field( 2299 ..., 2300 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2301 examples=[ 2302 ["data"], 2303 ["data", "records"], 2304 ["data", "{{ parameters.name }}"], 2305 ["data", "*", "record"], 2306 ], 2307 title="Field Path", 2308 ) 2309 record_expander: Optional[RecordExpander] = Field( 2310 None, 2311 description="Optional component to expand records by extracting items from nested array fields.", 2312 title="Record Expander", 2313 ) 2314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2315 2316 2317class ZipfileDecoder(BaseModel): 2318 class Config: 2319 extra = Extra.allow 2320 2321 type: Literal["ZipfileDecoder"] 2322 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2323 ..., 2324 description="Parser to parse the decompressed data from the zipfile(s).", 2325 title="Parser", 2326 ) 2327 2328 2329class RecordSelector(BaseModel): 2330 type: Literal["RecordSelector"] 2331 extractor: Union[DpathExtractor, CustomRecordExtractor] 2332 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2333 None, 2334 description="Responsible for filtering records to be emitted by the Source.", 2335 title="Record Filter", 2336 ) 2337 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2338 None, 2339 description="Responsible for normalization according to the schema.", 2340 title="Schema Normalization", 2341 ) 2342 transform_before_filtering: Optional[bool] = Field( 2343 None, 2344 description="If true, transformation will be applied before record filtering.", 2345 title="Transform Before Filtering", 2346 ) 2347 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2348 2349 2350class ConfigMigration(BaseModel): 2351 type: Literal["ConfigMigration"] 2352 description: Optional[str] = Field( 2353 None, description="The description/purpose of the config migration." 2354 ) 2355 transformations: List[ 2356 Union[ 2357 ConfigRemapField, 2358 ConfigAddFields, 2359 ConfigRemoveFields, 2360 CustomConfigTransformation, 2361 ] 2362 ] = Field( 2363 ..., 2364 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2365 title="Transformations", 2366 ) 2367 2368 2369class ConfigNormalizationRules(BaseModel): 2370 class Config: 2371 extra = Extra.forbid 2372 2373 type: Literal["ConfigNormalizationRules"] 2374 config_migrations: Optional[List[ConfigMigration]] = Field( 2375 [], 2376 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2377 title="Config Migrations", 2378 ) 2379 transformations: Optional[ 2380 List[ 2381 Union[ 2382 ConfigRemapField, 2383 ConfigAddFields, 2384 ConfigRemoveFields, 2385 CustomConfigTransformation, 2386 ] 2387 ] 2388 ] = Field( 2389 [], 2390 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2391 title="Transformations", 2392 ) 2393 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2394 [], 2395 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2396 title="Validations", 2397 ) 2398 2399 2400class Spec(BaseModel): 2401 type: Literal["Spec"] 2402 connection_specification: Dict[str, Any] = Field( 2403 ..., 2404 description="A connection specification describing how a the connector can be configured.", 2405 title="Connection Specification", 2406 ) 2407 documentation_url: Optional[str] = Field( 2408 None, 2409 description="URL of the connector's documentation page.", 2410 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2411 title="Documentation URL", 2412 ) 2413 advanced_auth: Optional[AuthFlow] = Field( 2414 None, 2415 description="Advanced specification for configuring the authentication flow.", 2416 title="Advanced Auth", 2417 ) 2418 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2419 None, title="Config Normalization Rules" 2420 ) 2421 2422 2423class DeclarativeSource1(BaseModel): 2424 class Config: 2425 extra = Extra.forbid 2426 2427 type: Literal["DeclarativeSource"] 2428 check: Union[CheckStream, CheckDynamicStream] 2429 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2430 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2431 version: str = Field( 2432 ..., 2433 description="The version of the Airbyte CDK used to build and test the source.", 2434 ) 2435 schemas: Optional[Schemas] = None 2436 definitions: Optional[Dict[str, Any]] = None 2437 spec: Optional[Spec] = None 2438 concurrency_level: Optional[ConcurrencyLevel] = None 2439 api_budget: Optional[HTTPAPIBudget] = None 2440 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2441 None, 2442 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2443 title="Stream Groups", 2444 ) 2445 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2446 None, 2447 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2448 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2449 title="Maximum Concurrent Asynchronous Jobs", 2450 ) 2451 metadata: Optional[Dict[str, Any]] = Field( 2452 None, 2453 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2454 ) 2455 description: Optional[str] = Field( 2456 None, 2457 description="A description of the connector. It will be presented on the Source documentation page.", 2458 ) 2459 2460 2461class DeclarativeSource2(BaseModel): 2462 class Config: 2463 extra = Extra.forbid 2464 2465 type: Literal["DeclarativeSource"] 2466 check: Union[CheckStream, CheckDynamicStream] 2467 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2468 None 2469 ) 2470 dynamic_streams: List[DynamicDeclarativeStream] 2471 version: str = Field( 2472 ..., 2473 description="The version of the Airbyte CDK used to build and test the source.", 2474 ) 2475 schemas: Optional[Schemas] = None 2476 definitions: Optional[Dict[str, Any]] = None 2477 spec: Optional[Spec] = None 2478 concurrency_level: Optional[ConcurrencyLevel] = None 2479 api_budget: Optional[HTTPAPIBudget] = None 2480 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2481 None, 2482 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2483 title="Stream Groups", 2484 ) 2485 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2486 None, 2487 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2488 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2489 title="Maximum Concurrent Asynchronous Jobs", 2490 ) 2491 metadata: Optional[Dict[str, Any]] = Field( 2492 None, 2493 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2494 ) 2495 description: Optional[str] = Field( 2496 None, 2497 description="A description of the connector. It will be presented on the Source documentation page.", 2498 ) 2499 2500 2501class DeclarativeSource(BaseModel): 2502 class Config: 2503 extra = Extra.forbid 2504 2505 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2506 ..., 2507 description="An API source that extracts data according to its declarative components.", 2508 title="DeclarativeSource", 2509 ) 2510 2511 2512class SelectiveAuthenticator(BaseModel): 2513 class Config: 2514 extra = Extra.allow 2515 2516 type: Literal["SelectiveAuthenticator"] 2517 authenticator_selection_path: List[str] = Field( 2518 ..., 2519 description="Path of the field in config with selected authenticator name", 2520 examples=[["auth"], ["auth", "type"]], 2521 title="Authenticator Selection Path", 2522 ) 2523 authenticators: Dict[ 2524 str, 2525 Union[ 2526 ApiKeyAuthenticator, 2527 BasicHttpAuthenticator, 2528 BearerAuthenticator, 2529 OAuthAuthenticator, 2530 JwtAuthenticator, 2531 SessionTokenAuthenticator, 2532 LegacySessionTokenAuthenticator, 2533 CustomAuthenticator, 2534 NoAuth, 2535 ], 2536 ] = Field( 2537 ..., 2538 description="Authenticators to select from.", 2539 examples=[ 2540 { 2541 "authenticators": { 2542 "token": "#/definitions/ApiKeyAuthenticator", 2543 "oauth": "#/definitions/OAuthAuthenticator", 2544 "jwt": "#/definitions/JwtAuthenticator", 2545 } 2546 } 2547 ], 2548 title="Authenticators", 2549 ) 2550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2551 2552 2553class ConditionalStreams(BaseModel): 2554 type: Literal["ConditionalStreams"] 2555 condition: str = Field( 2556 ..., 2557 description="Condition that will be evaluated to determine if a set of streams should be available.", 2558 examples=["{{ config['is_sandbox'] }}"], 2559 title="Condition", 2560 ) 2561 streams: List[DeclarativeStream] = Field( 2562 ..., 2563 description="Streams that will be used during an operation based on the condition.", 2564 title="Streams", 2565 ) 2566 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2567 2568 2569class FileUploader(BaseModel): 2570 type: Literal["FileUploader"] 2571 requester: Union[HttpRequester, CustomRequester] = Field( 2572 ..., 2573 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2574 ) 2575 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2576 ..., 2577 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2578 ) 2579 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2580 None, 2581 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2582 ) 2583 filename_extractor: Optional[str] = Field( 2584 None, 2585 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2586 examples=[ 2587 "{{ record.id }}/{{ record.file_name }}/", 2588 "{{ record.id }}_{{ record.file_name }}/", 2589 ], 2590 ) 2591 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2592 2593 2594class DeclarativeStream(BaseModel): 2595 class Config: 2596 extra = Extra.allow 2597 2598 type: Literal["DeclarativeStream"] 2599 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2600 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2601 ..., 2602 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2603 title="Retriever", 2604 ) 2605 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2606 None, 2607 description="Component used to fetch data incrementally based on a time field in the data.", 2608 title="Incremental Sync", 2609 ) 2610 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2611 schema_loader: Optional[ 2612 Union[ 2613 InlineSchemaLoader, 2614 DynamicSchemaLoader, 2615 JsonFileSchemaLoader, 2616 List[ 2617 Union[ 2618 InlineSchemaLoader, 2619 DynamicSchemaLoader, 2620 JsonFileSchemaLoader, 2621 CustomSchemaLoader, 2622 ] 2623 ], 2624 CustomSchemaLoader, 2625 ] 2626 ] = Field( 2627 None, 2628 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2629 title="Schema Loader", 2630 ) 2631 transformations: Optional[ 2632 List[ 2633 Union[ 2634 AddFields, 2635 RemoveFields, 2636 KeysToLower, 2637 KeysToSnakeCase, 2638 FlattenFields, 2639 DpathFlattenFields, 2640 KeysReplace, 2641 CustomTransformation, 2642 ] 2643 ] 2644 ] = Field( 2645 None, 2646 description="A list of transformations to be applied to each output record.", 2647 title="Transformations", 2648 ) 2649 state_migrations: Optional[ 2650 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2651 ] = Field( 2652 [], 2653 description="Array of state migrations to be applied on the input state", 2654 title="State Migrations", 2655 ) 2656 file_uploader: Optional[FileUploader] = Field( 2657 None, 2658 description="(experimental) Describes how to fetch a file", 2659 title="File Uploader", 2660 ) 2661 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2662 2663 2664class SessionTokenAuthenticator(BaseModel): 2665 type: Literal["SessionTokenAuthenticator"] 2666 login_requester: HttpRequester = Field( 2667 ..., 2668 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2669 examples=[ 2670 { 2671 "type": "HttpRequester", 2672 "url_base": "https://my_api.com", 2673 "path": "/login", 2674 "authenticator": { 2675 "type": "BasicHttpAuthenticator", 2676 "username": "{{ config.username }}", 2677 "password": "{{ config.password }}", 2678 }, 2679 } 2680 ], 2681 title="Login Requester", 2682 ) 2683 session_token_path: List[str] = Field( 2684 ..., 2685 description="The path in the response body returned from the login requester to the session token.", 2686 examples=[["access_token"], ["result", "token"]], 2687 title="Session Token Path", 2688 ) 2689 expiration_duration: Optional[str] = Field( 2690 None, 2691 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2692 examples=["PT1H", "P1D"], 2693 title="Expiration Duration", 2694 ) 2695 request_authentication: Union[ 2696 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2697 ] = Field( 2698 ..., 2699 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2700 title="Data Request Authentication", 2701 ) 2702 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2703 None, description="Component used to decode the response.", title="Decoder" 2704 ) 2705 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2706 2707 2708class HttpRequester(BaseModelWithDeprecations): 2709 type: Literal["HttpRequester"] 2710 url_base: Optional[str] = Field( 2711 None, 2712 deprecated=True, 2713 deprecation_message="Use `url` field instead.", 2714 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2715 examples=[ 2716 "https://connect.squareup.com/v2", 2717 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2718 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2719 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2720 ], 2721 title="API Base URL", 2722 ) 2723 url: Optional[str] = Field( 2724 None, 2725 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2726 examples=[ 2727 "https://connect.squareup.com/v2", 2728 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2729 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2730 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2731 ], 2732 title="API Endpoint URL", 2733 ) 2734 path: Optional[str] = Field( 2735 None, 2736 deprecated=True, 2737 deprecation_message="Use `url` field instead.", 2738 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2739 examples=[ 2740 "/products", 2741 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2742 "/trades/{{ config['symbol_id'] }}/history", 2743 ], 2744 title="URL Path", 2745 ) 2746 http_method: Optional[HttpMethod] = Field( 2747 HttpMethod.GET, 2748 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2749 examples=["GET", "POST"], 2750 title="HTTP Method", 2751 ) 2752 authenticator: Optional[ 2753 Union[ 2754 ApiKeyAuthenticator, 2755 BasicHttpAuthenticator, 2756 BearerAuthenticator, 2757 OAuthAuthenticator, 2758 JwtAuthenticator, 2759 SessionTokenAuthenticator, 2760 SelectiveAuthenticator, 2761 CustomAuthenticator, 2762 NoAuth, 2763 LegacySessionTokenAuthenticator, 2764 ] 2765 ] = Field( 2766 None, 2767 description="Authentication method to use for requests sent to the API.", 2768 title="Authenticator", 2769 ) 2770 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2771 None, 2772 deprecated=True, 2773 deprecation_message="Use `query_properties` field instead.", 2774 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2775 title="Fetch Properties from Endpoint", 2776 ) 2777 query_properties: Optional[QueryProperties] = Field( 2778 None, 2779 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2780 title="Query Properties", 2781 ) 2782 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2783 None, 2784 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2785 examples=[ 2786 {"unit": "day"}, 2787 { 2788 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2789 }, 2790 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2791 {"sort_by[asc]": "updated_at"}, 2792 ], 2793 title="Query Parameters", 2794 ) 2795 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2796 None, 2797 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2798 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2799 title="Request Headers", 2800 ) 2801 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2802 None, 2803 deprecated=True, 2804 deprecation_message="Use `request_body` field instead.", 2805 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2806 examples=[ 2807 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2808 ], 2809 title="Request Body Payload (Non-JSON)", 2810 ) 2811 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2812 None, 2813 deprecated=True, 2814 deprecation_message="Use `request_body` field instead.", 2815 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2816 examples=[ 2817 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2818 {"key": "{{ config['value'] }}"}, 2819 {"sort": {"field": "updated_at", "order": "ascending"}}, 2820 ], 2821 title="Request Body JSON Payload", 2822 ) 2823 request_body: Optional[ 2824 Union[ 2825 RequestBodyPlainText, 2826 RequestBodyUrlEncodedForm, 2827 RequestBodyJsonObject, 2828 RequestBodyGraphQL, 2829 ] 2830 ] = Field( 2831 None, 2832 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2833 title="Request Body", 2834 ) 2835 error_handler: Optional[ 2836 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2837 ] = Field( 2838 None, 2839 description="Error handler component that defines how to handle errors.", 2840 title="Error Handler", 2841 ) 2842 use_cache: Optional[bool] = Field( 2843 False, 2844 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2845 title="Use Cache", 2846 ) 2847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2848 2849 2850class DynamicSchemaLoader(BaseModel): 2851 type: Literal["DynamicSchemaLoader"] 2852 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2853 ..., 2854 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2855 title="Retriever", 2856 ) 2857 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2858 None, 2859 description="Responsible for filtering fields to be added to json schema.", 2860 title="Schema Filter", 2861 ) 2862 schema_transformations: Optional[ 2863 List[ 2864 Union[ 2865 AddFields, 2866 RemoveFields, 2867 KeysToLower, 2868 KeysToSnakeCase, 2869 FlattenFields, 2870 DpathFlattenFields, 2871 KeysReplace, 2872 CustomTransformation, 2873 ] 2874 ] 2875 ] = Field( 2876 None, 2877 description="A list of transformations to be applied to the schema.", 2878 title="Schema Transformations", 2879 ) 2880 schema_type_identifier: SchemaTypeIdentifier 2881 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2882 2883 2884class ParentStreamConfig(BaseModel): 2885 type: Literal["ParentStreamConfig"] 2886 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2887 ..., description="Reference to the parent stream.", title="Parent Stream" 2888 ) 2889 parent_key: str = Field( 2890 ..., 2891 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2892 examples=["id", "{{ config['parent_record_id'] }}"], 2893 title="Parent Key", 2894 ) 2895 partition_field: str = Field( 2896 ..., 2897 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2898 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2899 title="Current Parent Key Value Identifier", 2900 ) 2901 request_option: Optional[RequestOption] = Field( 2902 None, 2903 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2904 title="Request Option", 2905 ) 2906 incremental_dependency: Optional[bool] = Field( 2907 False, 2908 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2909 title="Incremental Dependency", 2910 ) 2911 lazy_read_pointer: Optional[List[str]] = Field( 2912 [], 2913 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2914 title="Lazy Read Pointer", 2915 ) 2916 extra_fields: Optional[List[List[str]]] = Field( 2917 None, 2918 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2919 title="Extra Fields", 2920 ) 2921 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2922 2923 2924class PropertiesFromEndpoint(BaseModel): 2925 type: Literal["PropertiesFromEndpoint"] 2926 property_field_path: List[str] = Field( 2927 ..., 2928 description="Describes the path to the field that should be extracted", 2929 examples=[["name"]], 2930 ) 2931 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2932 ..., 2933 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2934 ) 2935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2936 2937 2938class QueryProperties(BaseModel): 2939 type: Literal["QueryProperties"] 2940 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2941 ..., 2942 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2943 title="Property List", 2944 ) 2945 always_include_properties: Optional[List[str]] = Field( 2946 None, 2947 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2948 title="Always Include Properties", 2949 ) 2950 property_chunking: Optional[PropertyChunking] = Field( 2951 None, 2952 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2953 title="Property Chunking", 2954 ) 2955 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2956 None, 2957 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2958 title="Property Selector", 2959 ) 2960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2961 2962 2963class StateDelegatingStream(BaseModel): 2964 type: Literal["StateDelegatingStream"] 2965 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2966 full_refresh_stream: DeclarativeStream = Field( 2967 ..., 2968 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2969 title="Full Refresh Stream", 2970 ) 2971 incremental_stream: DeclarativeStream = Field( 2972 ..., 2973 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2974 title="Incremental Stream", 2975 ) 2976 api_retention_period: Optional[str] = Field( 2977 None, 2978 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2979 examples=["P30D", "P90D", "P1Y"], 2980 title="API Retention Period", 2981 ) 2982 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2983 2984 2985class SimpleRetriever(BaseModel): 2986 type: Literal["SimpleRetriever"] 2987 requester: Union[HttpRequester, CustomRequester] = Field( 2988 ..., 2989 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2990 ) 2991 decoder: Optional[ 2992 Union[ 2993 JsonDecoder, 2994 XmlDecoder, 2995 CsvDecoder, 2996 JsonlDecoder, 2997 GzipDecoder, 2998 IterableDecoder, 2999 ZipfileDecoder, 3000 CustomDecoder, 3001 ] 3002 ] = Field( 3003 None, 3004 description="Component decoding the response so records can be extracted.", 3005 title="HTTP Response Format", 3006 ) 3007 record_selector: RecordSelector = Field( 3008 ..., 3009 description="Component that describes how to extract records from a HTTP response.", 3010 ) 3011 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3012 None, 3013 description="Paginator component that describes how to navigate through the API's pages.", 3014 ) 3015 pagination_reset: Optional[PaginationReset] = Field( 3016 None, 3017 description="Describes what triggers pagination reset and how to handle it.", 3018 ) 3019 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3020 False, 3021 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3022 ) 3023 partition_router: Optional[ 3024 Union[ 3025 SubstreamPartitionRouter, 3026 ListPartitionRouter, 3027 GroupingPartitionRouter, 3028 CustomPartitionRouter, 3029 List[ 3030 Union[ 3031 SubstreamPartitionRouter, 3032 ListPartitionRouter, 3033 GroupingPartitionRouter, 3034 CustomPartitionRouter, 3035 ] 3036 ], 3037 ] 3038 ] = Field( 3039 None, 3040 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3041 title="Partition Router", 3042 ) 3043 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3044 3045 3046class AsyncRetriever(BaseModel): 3047 type: Literal["AsyncRetriever"] 3048 record_selector: RecordSelector = Field( 3049 ..., 3050 description="Component that describes how to extract records from a HTTP response.", 3051 ) 3052 status_mapping: AsyncJobStatusMap = Field( 3053 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3054 ) 3055 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3056 ..., description="Responsible for fetching the actual status of the async job." 3057 ) 3058 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3059 None, 3060 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3061 ) 3062 download_extractor: Optional[ 3063 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3064 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3065 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3066 ..., 3067 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3068 ) 3069 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3070 ..., 3071 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3072 ) 3073 polling_job_timeout: Optional[Union[int, str]] = Field( 3074 None, 3075 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3076 ) 3077 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3078 None, 3079 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3080 ) 3081 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3082 None, 3083 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3084 ) 3085 download_requester: Union[HttpRequester, CustomRequester] = Field( 3086 ..., 3087 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3088 ) 3089 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3090 None, 3091 description="Paginator component that describes how to navigate through the API's pages during download.", 3092 ) 3093 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3094 None, 3095 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3096 ) 3097 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3098 None, 3099 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3100 ) 3101 partition_router: Optional[ 3102 Union[ 3103 ListPartitionRouter, 3104 SubstreamPartitionRouter, 3105 GroupingPartitionRouter, 3106 CustomPartitionRouter, 3107 List[ 3108 Union[ 3109 ListPartitionRouter, 3110 SubstreamPartitionRouter, 3111 GroupingPartitionRouter, 3112 CustomPartitionRouter, 3113 ] 3114 ], 3115 ] 3116 ] = Field( 3117 [], 3118 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3119 title="Partition Router", 3120 ) 3121 decoder: Optional[ 3122 Union[ 3123 CsvDecoder, 3124 GzipDecoder, 3125 JsonDecoder, 3126 JsonlDecoder, 3127 IterableDecoder, 3128 XmlDecoder, 3129 ZipfileDecoder, 3130 CustomDecoder, 3131 ] 3132 ] = Field( 3133 None, 3134 description="Component decoding the response so records can be extracted.", 3135 title="HTTP Response Format", 3136 ) 3137 download_decoder: Optional[ 3138 Union[ 3139 CsvDecoder, 3140 GzipDecoder, 3141 JsonDecoder, 3142 JsonlDecoder, 3143 IterableDecoder, 3144 XmlDecoder, 3145 ZipfileDecoder, 3146 CustomDecoder, 3147 ] 3148 ] = Field( 3149 None, 3150 description="Component decoding the download response so records can be extracted.", 3151 title="Download HTTP Response Format", 3152 ) 3153 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3154 3155 3156class BlockSimultaneousSyncsAction(BaseModel): 3157 type: Literal["BlockSimultaneousSyncsAction"] 3158 3159 3160class StreamGroup(BaseModel): 3161 streams: List[str] = Field( 3162 ..., 3163 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3164 title="Streams", 3165 ) 3166 action: BlockSimultaneousSyncsAction = Field( 3167 ..., 3168 description="The action to apply to streams in this group.", 3169 title="Action", 3170 ) 3171 3172 3173class SubstreamPartitionRouter(BaseModel): 3174 type: Literal["SubstreamPartitionRouter"] 3175 parent_stream_configs: List[ParentStreamConfig] = Field( 3176 ..., 3177 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3178 title="Parent Stream Configs", 3179 ) 3180 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3181 3182 3183class GroupingPartitionRouter(BaseModel): 3184 type: Literal["GroupingPartitionRouter"] 3185 group_size: int = Field( 3186 ..., 3187 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3188 examples=[10, 50], 3189 title="Group Size", 3190 ) 3191 underlying_partition_router: Union[ 3192 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3193 ] = Field( 3194 ..., 3195 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3196 title="Underlying Partition Router", 3197 ) 3198 deduplicate: Optional[bool] = Field( 3199 True, 3200 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3201 title="Deduplicate Partitions", 3202 ) 3203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3204 3205 3206class HttpComponentsResolver(BaseModel): 3207 type: Literal["HttpComponentsResolver"] 3208 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3209 ..., 3210 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3211 title="Retriever", 3212 ) 3213 components_mapping: List[ComponentMappingDefinition] 3214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3215 3216 3217class DynamicDeclarativeStream(BaseModel): 3218 type: Literal["DynamicDeclarativeStream"] 3219 name: Optional[str] = Field( 3220 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3221 ) 3222 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3223 ..., description="Reference to the stream template.", title="Stream Template" 3224 ) 3225 components_resolver: Union[ 3226 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3227 ] = Field( 3228 ..., 3229 description="Component resolve and populates stream templates with components values.", 3230 title="Components Resolver", 3231 ) 3232 use_parent_parameters: Optional[bool] = Field( 3233 True, 3234 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3235 title="Use Parent Parameters", 3236 ) 3237 3238 3239ComplexFieldType.update_forward_refs() 3240GzipDecoder.update_forward_refs() 3241CompositeErrorHandler.update_forward_refs() 3242DeclarativeSource1.update_forward_refs() 3243DeclarativeSource2.update_forward_refs() 3244SelectiveAuthenticator.update_forward_refs() 3245ConditionalStreams.update_forward_refs() 3246FileUploader.update_forward_refs() 3247DeclarativeStream.update_forward_refs() 3248SessionTokenAuthenticator.update_forward_refs() 3249HttpRequester.update_forward_refs() 3250DynamicSchemaLoader.update_forward_refs() 3251ParentStreamConfig.update_forward_refs() 3252PropertiesFromEndpoint.update_forward_refs() 3253SimpleRetriever.update_forward_refs() 3254AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
111class CursorPagination(BaseModel): 112 type: Literal["CursorPagination"] 113 cursor_value: str = Field( 114 ..., 115 description="Value of the cursor defining the next page to fetch.", 116 examples=[ 117 "{{ headers.link.next.cursor }}", 118 "{{ last_record['key'] }}", 119 "{{ response['nextPage'] }}", 120 ], 121 title="Cursor Value", 122 ) 123 page_size: Optional[Union[int, str]] = Field( 124 None, 125 description="The number of records to include in each pages.", 126 examples=[100, "{{ config['page_size'] }}"], 127 title="Page Size", 128 ) 129 stop_condition: Optional[str] = Field( 130 None, 131 description="Template string evaluating when to stop paginating.", 132 examples=[ 133 "{{ response.data.has_more is false }}", 134 "{{ 'next' not in headers['link'] }}", 135 ], 136 title="Stop Condition", 137 ) 138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
141class CustomAuthenticator(BaseModel): 142 class Config: 143 extra = Extra.allow 144 145 type: Literal["CustomAuthenticator"] 146 class_name: str = Field( 147 ..., 148 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 149 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 150 title="Class Name", 151 ) 152 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
155class CustomBackoffStrategy(BaseModel): 156 class Config: 157 extra = Extra.allow 158 159 type: Literal["CustomBackoffStrategy"] 160 class_name: str = Field( 161 ..., 162 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 163 examples=["source_railz.components.MyCustomBackoffStrategy"], 164 title="Class Name", 165 ) 166 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
169class CustomErrorHandler(BaseModel): 170 class Config: 171 extra = Extra.allow 172 173 type: Literal["CustomErrorHandler"] 174 class_name: str = Field( 175 ..., 176 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 177 examples=["source_railz.components.MyCustomErrorHandler"], 178 title="Class Name", 179 ) 180 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
183class CustomPaginationStrategy(BaseModel): 184 class Config: 185 extra = Extra.allow 186 187 type: Literal["CustomPaginationStrategy"] 188 class_name: str = Field( 189 ..., 190 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 191 examples=["source_railz.components.MyCustomPaginationStrategy"], 192 title="Class Name", 193 ) 194 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
197class CustomRecordExtractor(BaseModel): 198 class Config: 199 extra = Extra.allow 200 201 type: Literal["CustomRecordExtractor"] 202 class_name: str = Field( 203 ..., 204 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 205 examples=["source_railz.components.MyCustomRecordExtractor"], 206 title="Class Name", 207 ) 208 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
211class CustomRecordFilter(BaseModel): 212 class Config: 213 extra = Extra.allow 214 215 type: Literal["CustomRecordFilter"] 216 class_name: str = Field( 217 ..., 218 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 219 examples=["source_railz.components.MyCustomCustomRecordFilter"], 220 title="Class Name", 221 ) 222 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
225class CustomRequester(BaseModel): 226 class Config: 227 extra = Extra.allow 228 229 type: Literal["CustomRequester"] 230 class_name: str = Field( 231 ..., 232 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 233 examples=["source_railz.components.MyCustomRecordExtractor"], 234 title="Class Name", 235 ) 236 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
239class CustomRetriever(BaseModel): 240 class Config: 241 extra = Extra.allow 242 243 type: Literal["CustomRetriever"] 244 class_name: str = Field( 245 ..., 246 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 247 examples=["source_railz.components.MyCustomRetriever"], 248 title="Class Name", 249 ) 250 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
253class CustomPartitionRouter(BaseModel): 254 class Config: 255 extra = Extra.allow 256 257 type: Literal["CustomPartitionRouter"] 258 class_name: str = Field( 259 ..., 260 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 261 examples=["source_railz.components.MyCustomPartitionRouter"], 262 title="Class Name", 263 ) 264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
267class CustomSchemaLoader(BaseModel): 268 class Config: 269 extra = Extra.allow 270 271 type: Literal["CustomSchemaLoader"] 272 class_name: str = Field( 273 ..., 274 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 275 examples=["source_railz.components.MyCustomSchemaLoader"], 276 title="Class Name", 277 ) 278 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
281class CustomSchemaNormalization(BaseModel): 282 class Config: 283 extra = Extra.allow 284 285 type: Literal["CustomSchemaNormalization"] 286 class_name: str = Field( 287 ..., 288 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 289 examples=[ 290 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 291 ], 292 title="Class Name", 293 ) 294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
297class CustomStateMigration(BaseModel): 298 class Config: 299 extra = Extra.allow 300 301 type: Literal["CustomStateMigration"] 302 class_name: str = Field( 303 ..., 304 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 305 examples=["source_railz.components.MyCustomStateMigration"], 306 title="Class Name", 307 ) 308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
311class CustomTransformation(BaseModel): 312 class Config: 313 extra = Extra.allow 314 315 type: Literal["CustomTransformation"] 316 class_name: str = Field( 317 ..., 318 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 319 examples=["source_railz.components.MyCustomTransformation"], 320 title="Class Name", 321 ) 322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
342class Algorithm(Enum): 343 HS256 = "HS256" 344 HS384 = "HS384" 345 HS512 = "HS512" 346 ES256 = "ES256" 347 ES256K = "ES256K" 348 ES384 = "ES384" 349 ES512 = "ES512" 350 RS256 = "RS256" 351 RS384 = "RS384" 352 RS512 = "RS512" 353 PS256 = "PS256" 354 PS384 = "PS384" 355 PS512 = "PS512" 356 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
359class JwtHeaders(BaseModel): 360 class Config: 361 extra = Extra.forbid 362 363 kid: Optional[str] = Field( 364 None, 365 description="Private key ID for user account.", 366 examples=["{{ config['kid'] }}"], 367 title="Key Identifier", 368 ) 369 typ: Optional[str] = Field( 370 "JWT", 371 description="The media type of the complete JWT.", 372 examples=["JWT"], 373 title="Type", 374 ) 375 cty: Optional[str] = Field( 376 None, 377 description="Content type of JWT header.", 378 examples=["JWT"], 379 title="Content Type", 380 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
383class JwtPayload(BaseModel): 384 class Config: 385 extra = Extra.forbid 386 387 iss: Optional[str] = Field( 388 None, 389 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 390 examples=["{{ config['iss'] }}"], 391 title="Issuer", 392 ) 393 sub: Optional[str] = Field( 394 None, 395 description="The subject of the JWT. Commonly defined by the API.", 396 title="Subject", 397 ) 398 aud: Optional[str] = Field( 399 None, 400 description="The recipient that the JWT is intended for. Commonly defined by the API.", 401 examples=["appstoreconnect-v1"], 402 title="Audience", 403 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
406class RefreshTokenUpdater(BaseModel): 407 refresh_token_name: Optional[str] = Field( 408 "refresh_token", 409 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 410 examples=["refresh_token"], 411 title="Refresh Token Property Name", 412 ) 413 access_token_config_path: Optional[List[str]] = Field( 414 ["credentials", "access_token"], 415 description="Config path to the access token. Make sure the field actually exists in the config.", 416 examples=[["credentials", "access_token"], ["access_token"]], 417 title="Config Path To Access Token", 418 ) 419 refresh_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "refresh_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "refresh_token"], ["refresh_token"]], 423 title="Config Path To Refresh Token", 424 ) 425 token_expiry_date_config_path: Optional[List[str]] = Field( 426 ["credentials", "token_expiry_date"], 427 description="Config path to the expiry date. Make sure actually exists in the config.", 428 examples=[["credentials", "token_expiry_date"]], 429 title="Config Path To Expiry Date", 430 ) 431 refresh_token_error_status_codes: Optional[List[int]] = Field( 432 [], 433 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 434 examples=[[400, 500]], 435 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 436 ) 437 refresh_token_error_key: Optional[str] = Field( 438 "", 439 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 440 examples=["error"], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 442 ) 443 refresh_token_error_values: Optional[List[str]] = Field( 444 [], 445 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 446 examples=[["invalid_grant", "invalid_permissions"]], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 448 )
class
Rate(pydantic.v1.main.BaseModel):
451class Rate(BaseModel): 452 class Config: 453 extra = Extra.allow 454 455 limit: Union[int, str] = Field( 456 ..., 457 description="The maximum number of calls allowed within the interval.", 458 title="Limit", 459 ) 460 interval: str = Field( 461 ..., 462 description="The time interval for the rate limit.", 463 examples=["PT1H", "P1D"], 464 title="Interval", 465 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
468class HttpRequestRegexMatcher(BaseModel): 469 class Config: 470 extra = Extra.allow 471 472 method: Optional[str] = Field( 473 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 474 ) 475 url_base: Optional[str] = Field( 476 None, 477 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 478 title="URL Base", 479 ) 480 url_path_pattern: Optional[str] = Field( 481 None, 482 description="A regular expression pattern to match the URL path.", 483 title="URL Path Pattern", 484 ) 485 params: Optional[Dict[str, Any]] = Field( 486 None, description="The query parameters to match.", title="Parameters" 487 ) 488 headers: Optional[Dict[str, Any]] = Field( 489 None, description="The headers to match.", title="Headers" 490 ) 491 weight: Optional[Union[int, str]] = Field( 492 None, 493 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 494 title="Weight", 495 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
OnNoRecords(enum.Enum):
An enumeration.
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
508class ExponentialBackoffStrategy(BaseModel): 509 type: Literal["ExponentialBackoffStrategy"] 510 factor: Optional[Union[float, str]] = Field( 511 5, 512 description="Multiplicative constant applied on each retry.", 513 examples=[5, 5.5, "10"], 514 title="Factor", 515 ) 516 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
519class GroupByKeyMergeStrategy(BaseModel): 520 type: Literal["GroupByKeyMergeStrategy"] 521 key: Union[str, List[str]] = Field( 522 ..., 523 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 524 examples=["id", ["parent_id", "end_date"]], 525 title="Key", 526 ) 527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
539class Action(Enum): 540 SUCCESS = "SUCCESS" 541 FAIL = "FAIL" 542 RETRY = "RETRY" 543 IGNORE = "IGNORE" 544 RESET_PAGINATION = "RESET_PAGINATION" 545 RATE_LIMITED = "RATE_LIMITED" 546 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
549class FailureType(Enum): 550 system_error = "system_error" 551 config_error = "config_error" 552 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
555class HttpResponseFilter(BaseModel): 556 type: Literal["HttpResponseFilter"] 557 action: Optional[Action] = Field( 558 None, 559 description="Action to execute if a response matches the filter.", 560 examples=[ 561 "SUCCESS", 562 "FAIL", 563 "RETRY", 564 "IGNORE", 565 "RESET_PAGINATION", 566 "RATE_LIMITED", 567 "REFRESH_TOKEN_THEN_RETRY", 568 ], 569 title="Action", 570 ) 571 failure_type: Optional[FailureType] = Field( 572 None, 573 description="Failure type of traced exception if a response matches the filter.", 574 examples=["system_error", "config_error", "transient_error"], 575 title="Failure Type", 576 ) 577 error_message: Optional[str] = Field( 578 None, 579 description="Error Message to display if the response matches the filter.", 580 title="Error Message", 581 ) 582 error_message_contains: Optional[str] = Field( 583 None, 584 description="Match the response if its error message contains the substring.", 585 example=["This API operation is not enabled for this site"], 586 title="Error Message Substring", 587 ) 588 http_codes: Optional[List[int]] = Field( 589 None, 590 description="Match the response if its HTTP code is included in this list.", 591 examples=[[420, 429], [500]], 592 title="HTTP Codes", 593 unique_items=True, 594 ) 595 predicate: Optional[str] = Field( 596 None, 597 description="Match the response if the predicate evaluates to true.", 598 examples=[ 599 "{{ 'Too much requests' in response }}", 600 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 601 ], 602 title="Predicate", 603 ) 604 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
607class ComplexFieldType(BaseModel): 608 field_type: str 609 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
612class TypesMap(BaseModel): 613 target_type: Union[str, List[str], ComplexFieldType] 614 current_type: Union[str, List[str]] 615 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
618class SchemaTypeIdentifier(BaseModel): 619 type: Optional[Literal["SchemaTypeIdentifier"]] = None 620 schema_pointer: Optional[List[str]] = Field( 621 [], 622 description="List of nested fields defining the schema field path to extract. Defaults to [].", 623 title="Schema Path", 624 ) 625 key_pointer: List[str] = Field( 626 ..., 627 description="List of potentially nested fields describing the full path of the field key to extract.", 628 title="Key Path", 629 ) 630 type_pointer: Optional[List[str]] = Field( 631 None, 632 description="List of potentially nested fields describing the full path of the field type to extract.", 633 title="Type Path", 634 ) 635 types_mapping: Optional[List[TypesMap]] = None 636 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
639class InlineSchemaLoader(BaseModel): 640 type: Literal["InlineSchemaLoader"] 641 schema_: Optional[Dict[str, Any]] = Field( 642 None, 643 alias="schema", 644 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 645 title="Schema", 646 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
649class JsonFileSchemaLoader(BaseModel): 650 type: Literal["JsonFileSchemaLoader"] 651 file_path: Optional[str] = Field( 652 None, 653 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 654 example=["./schemas/users.json"], 655 title="File Path", 656 ) 657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
678class FlattenFields(BaseModel): 679 type: Literal["FlattenFields"] 680 flatten_lists: Optional[bool] = Field( 681 True, 682 description="Whether to flatten lists or leave it as is. Default is True.", 683 title="Flatten Lists", 684 ) 685 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
688class KeyTransformation(BaseModel): 689 type: Literal["KeyTransformation"] 690 prefix: Optional[str] = Field( 691 None, 692 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 693 examples=["flattened_"], 694 title="Key Prefix", 695 ) 696 suffix: Optional[str] = Field( 697 None, 698 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 699 examples=["_flattened"], 700 title="Key Suffix", 701 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
704class DpathFlattenFields(BaseModel): 705 type: Literal["DpathFlattenFields"] 706 field_path: List[str] = Field( 707 ..., 708 description="A path to field that needs to be flattened.", 709 examples=[["data"], ["data", "*", "field"]], 710 title="Field Path", 711 ) 712 delete_origin_value: Optional[bool] = Field( 713 None, 714 description="Whether to delete the origin value or keep it. Default is False.", 715 title="Delete Origin Value", 716 ) 717 replace_record: Optional[bool] = Field( 718 None, 719 description="Whether to replace the origin record or not. Default is False.", 720 title="Replace Origin Record", 721 ) 722 key_transformation: Optional[KeyTransformation] = Field( 723 None, 724 description="Transformation for object keys. If not provided, original key will be used.", 725 title="Key transformation", 726 ) 727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
730class KeysReplace(BaseModel): 731 type: Literal["KeysReplace"] 732 old: str = Field( 733 ..., 734 description="Old value to replace.", 735 examples=[ 736 " ", 737 "{{ record.id }}", 738 "{{ config['id'] }}", 739 "{{ stream_slice['id'] }}", 740 ], 741 title="Old value", 742 ) 743 new: str = Field( 744 ..., 745 description="New value to set.", 746 examples=[ 747 "_", 748 "{{ record.id }}", 749 "{{ config['id'] }}", 750 "{{ stream_slice['id'] }}", 751 ], 752 title="New value", 753 ) 754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
765class CustomDecoder(BaseModel): 766 class Config: 767 extra = Extra.allow 768 769 type: Literal["CustomDecoder"] 770 class_name: str = Field( 771 ..., 772 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 773 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 774 title="Class Name", 775 ) 776 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
779class MinMaxDatetime(BaseModel): 780 type: Literal["MinMaxDatetime"] 781 datetime: str = Field( 782 ..., 783 description="Datetime value.", 784 examples=[ 785 "2021-01-01", 786 "2021-01-01T00:00:00Z", 787 "{{ config['start_time'] }}", 788 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 789 ], 790 title="Datetime", 791 ) 792 datetime_format: Optional[str] = Field( 793 "", 794 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 795 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 796 title="Datetime Format", 797 ) 798 max_datetime: Optional[str] = Field( 799 None, 800 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 801 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 802 title="Max Datetime", 803 ) 804 min_datetime: Optional[str] = Field( 805 None, 806 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 807 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 808 title="Min Datetime", 809 ) 810 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
840class OauthConnectorInputSpecification(BaseModel): 841 class Config: 842 extra = Extra.allow 843 844 consent_url: str = Field( 845 ..., 846 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 847 examples=[ 848 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 849 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 850 ], 851 title="Consent URL", 852 ) 853 scope: Optional[str] = Field( 854 None, 855 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 856 examples=["user:read user:read_orders workspaces:read"], 857 title="Scopes", 858 ) 859 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 860 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 861 # The CDK schema defines the manifest contract; the platform reads these fields 862 # during the OAuth consent flow to build the authorization URL. 863 scopes: Optional[List[OAuthScope]] = Field( 864 None, 865 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 866 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 867 title="Scopes", 868 ) 869 optional_scopes: Optional[List[OAuthScope]] = Field( 870 None, 871 description="Optional OAuth scope objects that may or may not be granted.", 872 examples=[[{"scope": "admin:read"}]], 873 title="Optional Scopes", 874 ) 875 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 876 ScopesJoinStrategy.space, 877 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 878 title="Scopes Join Strategy", 879 ) 880 access_token_url: str = Field( 881 ..., 882 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 883 examples=[ 884 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 885 ], 886 title="Access Token URL", 887 ) 888 access_token_headers: Optional[Dict[str, Any]] = Field( 889 None, 890 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 891 examples=[ 892 { 893 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 894 } 895 ], 896 title="Access Token Headers", 897 ) 898 access_token_params: Optional[Dict[str, Any]] = Field( 899 None, 900 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 901 examples=[ 902 { 903 "{{ auth_code_key }}": "{{ auth_code_value }}", 904 "{{ client_id_key }}": "{{ client_id_value }}", 905 "{{ client_secret_key }}": "{{ client_secret_value }}", 906 } 907 ], 908 title="Access Token Query Params (Json Encoded)", 909 ) 910 extract_output: Optional[List[str]] = Field( 911 None, 912 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 913 examples=[["access_token", "refresh_token", "other_field"]], 914 title="Extract Output", 915 ) 916 state: Optional[State] = Field( 917 None, 918 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 919 examples=[{"min": 7, "max": 128}], 920 title="Configurable State Query Param", 921 ) 922 client_id_key: Optional[str] = Field( 923 None, 924 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 925 examples=["my_custom_client_id_key_name"], 926 title="Client ID Key Override", 927 ) 928 client_secret_key: Optional[str] = Field( 929 None, 930 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 931 examples=["my_custom_client_secret_key_name"], 932 title="Client Secret Key Override", 933 ) 934 scope_key: Optional[str] = Field( 935 None, 936 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 937 examples=["my_custom_scope_key_key_name"], 938 title="Scopes Key Override", 939 ) 940 state_key: Optional[str] = Field( 941 None, 942 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 943 examples=["my_custom_state_key_key_name"], 944 title="State Key Override", 945 ) 946 auth_code_key: Optional[str] = Field( 947 None, 948 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 949 examples=["my_custom_auth_code_key_name"], 950 title="Auth Code Key Override", 951 ) 952 redirect_uri_key: Optional[str] = Field( 953 None, 954 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 955 examples=["my_custom_redirect_uri_key_name"], 956 title="Redirect URI Key Override", 957 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
960class OAuthConfigSpecification(BaseModel): 961 class Config: 962 extra = Extra.allow 963 964 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 965 None, 966 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 967 examples=[ 968 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 969 { 970 "app_id": { 971 "type": "string", 972 "path_in_connector_config": ["info", "app_id"], 973 } 974 }, 975 ], 976 title="OAuth user input", 977 ) 978 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 979 None, 980 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 981 title="DeclarativeOAuth Connector Specification", 982 ) 983 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 984 None, 985 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 986 examples=[ 987 { 988 "refresh_token": { 989 "type": "string,", 990 "path_in_connector_config": ["credentials", "refresh_token"], 991 } 992 } 993 ], 994 title="OAuth output specification", 995 ) 996 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 997 None, 998 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 999 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1000 title="OAuth input specification", 1001 ) 1002 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1003 None, 1004 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1005 examples=[ 1006 { 1007 "client_id": { 1008 "type": "string,", 1009 "path_in_connector_config": ["credentials", "client_id"], 1010 }, 1011 "client_secret": { 1012 "type": "string,", 1013 "path_in_connector_config": ["credentials", "client_secret"], 1014 }, 1015 } 1016 ], 1017 title="OAuth server output specification", 1018 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1021class OffsetIncrement(BaseModel): 1022 type: Literal["OffsetIncrement"] 1023 page_size: Optional[Union[int, str]] = Field( 1024 None, 1025 description="The number of records to include in each pages.", 1026 examples=[100, "{{ config['page_size'] }}"], 1027 title="Limit", 1028 ) 1029 inject_on_first_request: Optional[bool] = Field( 1030 False, 1031 description="Using the `offset` with value `0` during the first request", 1032 title="Inject Offset on First Request", 1033 ) 1034 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1037class PageIncrement(BaseModel): 1038 type: Literal["PageIncrement"] 1039 page_size: Optional[Union[int, str]] = Field( 1040 None, 1041 description="The number of records to include in each pages.", 1042 examples=[100, "100", "{{ config['page_size'] }}"], 1043 title="Page Size", 1044 ) 1045 start_from_page: Optional[int] = Field( 1046 0, 1047 description="Index of the first page to request.", 1048 examples=[0, 1], 1049 title="Start From Page", 1050 ) 1051 inject_on_first_request: Optional[bool] = Field( 1052 False, 1053 description="Using the `page number` with value defined by `start_from_page` during the first request", 1054 title="Inject Page Number on First Request", 1055 ) 1056 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1059class PrimaryKey(BaseModel): 1060 __root__: Union[str, List[str], List[List[str]]] = Field( 1061 ..., 1062 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1063 examples=["id", ["code", "type"]], 1064 title="Primary Key", 1065 )
class
PropertyLimitType(enum.Enum):
1068class PropertyLimitType(Enum): 1069 characters = "characters" 1070 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1073class PropertyChunking(BaseModel): 1074 type: Literal["PropertyChunking"] 1075 property_limit_type: PropertyLimitType = Field( 1076 ..., 1077 description="The type used to determine the maximum number of properties per chunk", 1078 title="Property Limit Type", 1079 ) 1080 property_limit: Optional[int] = Field( 1081 None, 1082 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1083 title="Property Limit", 1084 ) 1085 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1086 None, 1087 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1088 title="Record Merge Strategy", 1089 ) 1090 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1093class RecordFilter(BaseModel): 1094 type: Literal["RecordFilter"] 1095 condition: Optional[str] = Field( 1096 "", 1097 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1098 examples=[ 1099 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1100 "{{ record.status in ['active', 'expired'] }}", 1101 ], 1102 title="Condition", 1103 ) 1104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1112class RemoveFields(BaseModel): 1113 type: Literal["RemoveFields"] 1114 condition: Optional[str] = Field( 1115 "", 1116 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1117 examples=[ 1118 "{{ property|string == '' }}", 1119 "{{ property is integer }}", 1120 "{{ property|length > 5 }}", 1121 "{{ property == 'some_string_to_match' }}", 1122 ], 1123 ) 1124 field_pointers: List[List[str]] = Field( 1125 ..., 1126 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1127 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1128 title="Field Paths", 1129 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1136class InjectInto(Enum): 1137 request_parameter = "request_parameter" 1138 header = "header" 1139 body_data = "body_data" 1140 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1143class RequestOption(BaseModel): 1144 type: Literal["RequestOption"] 1145 inject_into: InjectInto = Field( 1146 ..., 1147 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1148 examples=["request_parameter", "header", "body_data", "body_json"], 1149 title="Inject Into", 1150 ) 1151 field_name: Optional[str] = Field( 1152 None, 1153 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1154 examples=["segment_id"], 1155 title="Field Name", 1156 ) 1157 field_path: Optional[List[str]] = Field( 1158 None, 1159 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1160 examples=[["data", "viewer", "id"]], 1161 title="Field Path", 1162 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1172class LegacySessionTokenAuthenticator(BaseModel): 1173 type: Literal["LegacySessionTokenAuthenticator"] 1174 header: str = Field( 1175 ..., 1176 description="The name of the session token header that will be injected in the request", 1177 examples=["X-Session"], 1178 title="Session Request Header", 1179 ) 1180 login_url: str = Field( 1181 ..., 1182 description="Path of the login URL (do not include the base URL)", 1183 examples=["session"], 1184 title="Login Path", 1185 ) 1186 session_token: Optional[str] = Field( 1187 None, 1188 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1189 example=["{{ config['session_token'] }}"], 1190 title="Session Token", 1191 ) 1192 session_token_response_key: str = Field( 1193 ..., 1194 description="Name of the key of the session token to be extracted from the response", 1195 examples=["id"], 1196 title="Response Token Response Key", 1197 ) 1198 username: Optional[str] = Field( 1199 None, 1200 description="Username used to authenticate and obtain a session token", 1201 examples=[" {{ config['username'] }}"], 1202 title="Username", 1203 ) 1204 password: Optional[str] = Field( 1205 "", 1206 description="Password used to authenticate and obtain a session token", 1207 examples=["{{ config['password'] }}", ""], 1208 title="Password", 1209 ) 1210 validate_session_url: str = Field( 1211 ..., 1212 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1213 examples=["user/current"], 1214 title="Validate Session Path", 1215 ) 1216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1245class ValueType(Enum): 1246 string = "string" 1247 number = "number" 1248 integer = "integer" 1249 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1252class WaitTimeFromHeader(BaseModel): 1253 type: Literal["WaitTimeFromHeader"] 1254 header: str = Field( 1255 ..., 1256 description="The name of the response header defining how long to wait before retrying.", 1257 examples=["Retry-After"], 1258 title="Response Header Name", 1259 ) 1260 regex: Optional[str] = Field( 1261 None, 1262 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1263 examples=["([-+]?\\d+)"], 1264 title="Extraction Regex", 1265 ) 1266 max_waiting_time_in_seconds: Optional[float] = Field( 1267 None, 1268 description="Given the value extracted from the header is greater than this value, stop the stream.", 1269 examples=[3600], 1270 title="Max Waiting Time in Seconds", 1271 ) 1272 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1275class WaitUntilTimeFromHeader(BaseModel): 1276 type: Literal["WaitUntilTimeFromHeader"] 1277 header: str = Field( 1278 ..., 1279 description="The name of the response header defining how long to wait before retrying.", 1280 examples=["wait_time"], 1281 title="Response Header", 1282 ) 1283 min_wait: Optional[Union[float, str]] = Field( 1284 None, 1285 description="Minimum time to wait before retrying.", 1286 examples=[10, "60"], 1287 title="Minimum Wait Time", 1288 ) 1289 regex: Optional[str] = Field( 1290 None, 1291 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1292 examples=["([-+]?\\d+)"], 1293 title="Extraction Regex", 1294 ) 1295 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1298class ComponentMappingDefinition(BaseModel): 1299 type: Literal["ComponentMappingDefinition"] 1300 field_path: List[str] = Field( 1301 ..., 1302 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1303 examples=[ 1304 ["name"], 1305 ["retriever", "requester", "url"], 1306 ["retriever", "requester", "{{ components_values.field }}"], 1307 ["*", "**", "name"], 1308 ], 1309 title="Field Path", 1310 ) 1311 value: str = Field( 1312 ..., 1313 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1314 examples=[ 1315 "{{ components_values['updates'] }}", 1316 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1317 "{{ config['segment_id'] }}", 1318 "{{ stream_slice['parent_id'] }}", 1319 "{{ stream_slice['extra_fields']['name'] }}", 1320 ], 1321 title="Value", 1322 ) 1323 value_type: Optional[ValueType] = Field( 1324 None, 1325 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1326 title="Value Type", 1327 ) 1328 create_or_update: Optional[bool] = Field( 1329 False, 1330 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1331 title="Create or Update", 1332 ) 1333 condition: Optional[str] = Field( 1334 None, 1335 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1336 examples=[ 1337 "{{ components_values.get('cursor_field', None) }}", 1338 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1339 ], 1340 title="Condition", 1341 ) 1342 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1345class StreamConfig(BaseModel): 1346 type: Literal["StreamConfig"] 1347 configs_pointer: List[str] = Field( 1348 ..., 1349 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1350 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1351 title="Configs Pointer", 1352 ) 1353 default_values: Optional[List[Dict[str, Any]]] = Field( 1354 None, 1355 description="A list of default values, each matching the structure expected from the parsed component value.", 1356 title="Default Values", 1357 ) 1358 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1361class ConfigComponentsResolver(BaseModel): 1362 type: Literal["ConfigComponentsResolver"] 1363 stream_config: Union[List[StreamConfig], StreamConfig] 1364 components_mapping: List[ComponentMappingDefinition] 1365 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1368class StreamParametersDefinition(BaseModel): 1369 type: Literal["StreamParametersDefinition"] 1370 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1371 ..., 1372 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1373 examples=[ 1374 [ 1375 { 1376 "name": "test stream", 1377 "$parameters": {"entity": "test entity"}, 1378 "primary_key": "test key", 1379 } 1380 ] 1381 ], 1382 title="Stream Parameters", 1383 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1386class ParametrizedComponentsResolver(BaseModel): 1387 type: Literal["ParametrizedComponentsResolver"] 1388 stream_parameters: StreamParametersDefinition 1389 components_mapping: List[ComponentMappingDefinition] 1390 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1415class ValidateAdheresToSchema(BaseModel): 1416 type: Literal["ValidateAdheresToSchema"] 1417 base_schema: Union[str, Dict[str, Any]] = Field( 1418 ..., 1419 description="The base JSON schema against which the user-provided schema will be validated.", 1420 examples=[ 1421 "{{ config['report_validation_schema'] }}", 1422 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1423 { 1424 "$schema": "http://json-schema.org/draft-07/schema#", 1425 "title": "Person", 1426 "type": "object", 1427 "properties": { 1428 "name": {"type": "string", "description": "The person's name"}, 1429 "age": { 1430 "type": "integer", 1431 "minimum": 0, 1432 "description": "The person's age", 1433 }, 1434 }, 1435 "required": ["name", "age"], 1436 }, 1437 ], 1438 title="Base JSON Schema", 1439 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1442class CustomValidationStrategy(BaseModel): 1443 class Config: 1444 extra = Extra.allow 1445 1446 type: Literal["CustomValidationStrategy"] 1447 class_name: str = Field( 1448 ..., 1449 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1450 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1451 title="Class Name", 1452 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1455class ConfigRemapField(BaseModel): 1456 type: Literal["ConfigRemapField"] 1457 map: Union[Dict[str, Any], str] = Field( 1458 ..., 1459 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1460 examples=[ 1461 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1462 "{{ config['status_mapping'] }}", 1463 ], 1464 title="Value Mapping", 1465 ) 1466 field_path: List[str] = Field( 1467 ..., 1468 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1469 examples=[ 1470 ["status"], 1471 ["data", "status"], 1472 ["data", "{{ config.name }}", "status"], 1473 ["data", "*", "status"], 1474 ], 1475 title="Field Path", 1476 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1479class ConfigRemoveFields(BaseModel): 1480 type: Literal["ConfigRemoveFields"] 1481 field_pointers: List[List[str]] = Field( 1482 ..., 1483 description="A list of field pointers to be removed from the config.", 1484 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1485 title="Field Pointers", 1486 ) 1487 condition: Optional[str] = Field( 1488 "", 1489 description="Fields will be removed if expression is evaluated to True.", 1490 examples=[ 1491 "{{ config['environemnt'] == 'sandbox' }}", 1492 "{{ property is integer }}", 1493 "{{ property|length > 5 }}", 1494 "{{ property == 'some_string_to_match' }}", 1495 ], 1496 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1499class CustomConfigTransformation(BaseModel): 1500 type: Literal["CustomConfigTransformation"] 1501 class_name: str = Field( 1502 ..., 1503 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1504 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1505 ) 1506 parameters: Optional[Dict[str, Any]] = Field( 1507 None, 1508 alias="$parameters", 1509 description="Additional parameters to be passed to the custom config transformation.", 1510 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1513class AddedFieldDefinition(BaseModel): 1514 type: Literal["AddedFieldDefinition"] 1515 path: List[str] = Field( 1516 ..., 1517 description="List of strings defining the path where to add the value on the record.", 1518 examples=[["segment_id"], ["metadata", "segment_id"]], 1519 title="Path", 1520 ) 1521 value: str = Field( 1522 ..., 1523 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1524 examples=[ 1525 "{{ record['updates'] }}", 1526 "{{ record['MetaData']['LastUpdatedTime'] }}", 1527 "{{ stream_partition['segment_id'] }}", 1528 ], 1529 title="Value", 1530 ) 1531 value_type: Optional[ValueType] = Field( 1532 None, 1533 description="Type of the value. If not specified, the type will be inferred from the value.", 1534 title="Value Type", 1535 ) 1536 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1539class AddFields(BaseModel): 1540 type: Literal["AddFields"] 1541 fields: List[AddedFieldDefinition] = Field( 1542 ..., 1543 description="List of transformations (path and corresponding value) that will be added to the record.", 1544 title="Fields", 1545 ) 1546 condition: Optional[str] = Field( 1547 "", 1548 description="Fields will be added if expression is evaluated to True.", 1549 examples=[ 1550 "{{ property|string == '' }}", 1551 "{{ property is integer }}", 1552 "{{ property|length > 5 }}", 1553 "{{ property == 'some_string_to_match' }}", 1554 ], 1555 ) 1556 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1559class ApiKeyAuthenticator(BaseModel): 1560 type: Literal["ApiKeyAuthenticator"] 1561 api_token: Optional[str] = Field( 1562 None, 1563 description="The API key to inject in the request. Fill it in the user inputs.", 1564 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1565 title="API Key", 1566 ) 1567 header: Optional[str] = Field( 1568 None, 1569 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1570 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1571 title="Header Name", 1572 ) 1573 inject_into: Optional[RequestOption] = Field( 1574 None, 1575 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1576 examples=[ 1577 {"inject_into": "header", "field_name": "Authorization"}, 1578 {"inject_into": "request_parameter", "field_name": "authKey"}, 1579 ], 1580 title="Inject API Key Into Outgoing HTTP Request", 1581 ) 1582 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1585class AuthFlow(BaseModel): 1586 auth_flow_type: Optional[AuthFlowType] = Field( 1587 None, description="The type of auth to use", title="Auth flow type" 1588 ) 1589 predicate_key: Optional[List[str]] = Field( 1590 None, 1591 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1592 examples=[["credentials", "auth_type"]], 1593 title="Predicate key", 1594 ) 1595 predicate_value: Optional[str] = Field( 1596 None, 1597 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1598 examples=["Oauth"], 1599 title="Predicate value", 1600 ) 1601 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1604class CheckStream(BaseModel): 1605 type: Literal["CheckStream"] 1606 stream_names: Optional[List[str]] = Field( 1607 None, 1608 description="Names of the streams to try reading from when running a check operation.", 1609 examples=[["users"], ["users", "contacts"]], 1610 title="Stream Names", 1611 ) 1612 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1615class IncrementingCountCursor(BaseModel): 1616 type: Literal["IncrementingCountCursor"] 1617 cursor_field: str = Field( 1618 ..., 1619 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1620 examples=["created_at", "{{ config['record_cursor'] }}"], 1621 title="Cursor Field", 1622 ) 1623 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1624 None, 1625 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1626 title="Allow Catalog Defined Cursor Field", 1627 ) 1628 start_value: Optional[Union[str, int]] = Field( 1629 None, 1630 description="The value that determines the earliest record that should be synced.", 1631 examples=[0, "{{ config['start_value'] }}"], 1632 title="Start Value", 1633 ) 1634 start_value_option: Optional[RequestOption] = Field( 1635 None, 1636 description="Optionally configures how the start value will be sent in requests to the source API.", 1637 title="Inject Start Value Into Outgoing HTTP Request", 1638 ) 1639 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1642class DatetimeBasedCursor(BaseModel): 1643 type: Literal["DatetimeBasedCursor"] 1644 clamping: Optional[Clamping] = Field( 1645 None, 1646 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1647 title="Date Range Clamping", 1648 ) 1649 cursor_field: str = Field( 1650 ..., 1651 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1652 examples=["created_at", "{{ config['record_cursor'] }}"], 1653 title="Cursor Field", 1654 ) 1655 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1656 None, 1657 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1658 title="Allow Catalog Defined Cursor Field", 1659 ) 1660 cursor_datetime_formats: Optional[List[str]] = Field( 1661 None, 1662 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1663 examples=[ 1664 "%Y-%m-%d", 1665 "%Y-%m-%d %H:%M:%S", 1666 "%Y-%m-%dT%H:%M:%S", 1667 "%Y-%m-%dT%H:%M:%SZ", 1668 "%Y-%m-%dT%H:%M:%S%z", 1669 "%Y-%m-%dT%H:%M:%S.%fZ", 1670 "%Y-%m-%dT%H:%M:%S.%f%z", 1671 "%Y-%m-%d %H:%M:%S.%f+00:00", 1672 "%s", 1673 "%ms", 1674 ], 1675 title="Cursor Datetime Formats", 1676 ) 1677 start_datetime: Union[MinMaxDatetime, str] = Field( 1678 ..., 1679 description="The datetime that determines the earliest record that should be synced.", 1680 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1681 title="Start Datetime", 1682 ) 1683 start_time_option: Optional[RequestOption] = Field( 1684 None, 1685 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1686 title="Inject Start Time Into Outgoing HTTP Request", 1687 ) 1688 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1689 None, 1690 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1691 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1692 title="End Datetime", 1693 ) 1694 end_time_option: Optional[RequestOption] = Field( 1695 None, 1696 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1697 title="Inject End Time Into Outgoing HTTP Request", 1698 ) 1699 datetime_format: str = Field( 1700 ..., 1701 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1702 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1703 title="Outgoing Datetime Format", 1704 ) 1705 cursor_granularity: Optional[str] = Field( 1706 None, 1707 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1708 examples=["PT1S"], 1709 title="Cursor Granularity", 1710 ) 1711 is_data_feed: Optional[bool] = Field( 1712 None, 1713 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1714 title="Data Feed API", 1715 ) 1716 is_client_side_incremental: Optional[bool] = Field( 1717 None, 1718 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1719 title="Client-side Incremental Filtering", 1720 ) 1721 is_compare_strictly: Optional[bool] = Field( 1722 False, 1723 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1724 title="Strict Start-End Time Comparison", 1725 ) 1726 global_substream_cursor: Optional[bool] = Field( 1727 False, 1728 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1729 title="Global Substream Cursor", 1730 ) 1731 lookback_window: Optional[str] = Field( 1732 None, 1733 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1734 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1735 title="Lookback Window", 1736 ) 1737 partition_field_end: Optional[str] = Field( 1738 None, 1739 description="Name of the partition start time field.", 1740 examples=["ending_time"], 1741 title="Partition Field End", 1742 ) 1743 partition_field_start: Optional[str] = Field( 1744 None, 1745 description="Name of the partition end time field.", 1746 examples=["starting_time"], 1747 title="Partition Field Start", 1748 ) 1749 step: Optional[str] = Field( 1750 None, 1751 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1752 examples=["P1W", "{{ config['step_increment'] }}"], 1753 title="Step", 1754 ) 1755 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1758class JwtAuthenticator(BaseModel): 1759 type: Literal["JwtAuthenticator"] 1760 secret_key: str = Field( 1761 ..., 1762 description="Secret used to sign the JSON web token.", 1763 examples=["{{ config['secret_key'] }}"], 1764 title="Secret Key", 1765 ) 1766 base64_encode_secret_key: Optional[bool] = Field( 1767 False, 1768 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1769 title="Base64-encode Secret Key", 1770 ) 1771 algorithm: Algorithm = Field( 1772 ..., 1773 description="Algorithm used to sign the JSON web token.", 1774 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1775 title="Algorithm", 1776 ) 1777 token_duration: Optional[int] = Field( 1778 1200, 1779 description="The amount of time in seconds a JWT token can be valid after being issued.", 1780 examples=[1200, 3600], 1781 title="Token Duration", 1782 ) 1783 header_prefix: Optional[str] = Field( 1784 None, 1785 description="The prefix to be used within the Authentication header.", 1786 examples=["Bearer", "Basic"], 1787 title="Header Prefix", 1788 ) 1789 jwt_headers: Optional[JwtHeaders] = Field( 1790 None, 1791 description="JWT headers used when signing JSON web token.", 1792 title="JWT Headers", 1793 ) 1794 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1795 None, 1796 description="Additional headers to be included with the JWT headers object.", 1797 title="Additional JWT Headers", 1798 ) 1799 jwt_payload: Optional[JwtPayload] = Field( 1800 None, 1801 description="JWT Payload used when signing JSON web token.", 1802 title="JWT Payload", 1803 ) 1804 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1805 None, 1806 description="Additional properties to be added to the JWT payload.", 1807 title="Additional JWT Payload Properties", 1808 ) 1809 passphrase: Optional[str] = Field( 1810 None, 1811 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1812 examples=["{{ config['passphrase'] }}"], 1813 title="Passphrase", 1814 ) 1815 request_option: Optional[RequestOption] = Field( 1816 None, 1817 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1818 title="Request Option", 1819 ) 1820 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1823class OAuthAuthenticator(BaseModel): 1824 type: Literal["OAuthAuthenticator"] 1825 client_id_name: Optional[str] = Field( 1826 "client_id", 1827 description="The name of the property to use to refresh the `access_token`.", 1828 examples=["custom_app_id"], 1829 title="Client ID Property Name", 1830 ) 1831 client_id: Optional[str] = Field( 1832 None, 1833 description="The OAuth client ID. Fill it in the user inputs.", 1834 examples=[ 1835 "{{ config['client_id'] }}", 1836 "{{ config['credentials']['client_id }}", 1837 ], 1838 title="Client ID", 1839 ) 1840 client_secret_name: Optional[str] = Field( 1841 "client_secret", 1842 description="The name of the property to use to refresh the `access_token`.", 1843 examples=["custom_app_secret"], 1844 title="Client Secret Property Name", 1845 ) 1846 client_secret: Optional[str] = Field( 1847 None, 1848 description="The OAuth client secret. Fill it in the user inputs.", 1849 examples=[ 1850 "{{ config['client_secret'] }}", 1851 "{{ config['credentials']['client_secret }}", 1852 ], 1853 title="Client Secret", 1854 ) 1855 refresh_token_name: Optional[str] = Field( 1856 "refresh_token", 1857 description="The name of the property to use to refresh the `access_token`.", 1858 examples=["custom_app_refresh_value"], 1859 title="Refresh Token Property Name", 1860 ) 1861 refresh_token: Optional[str] = Field( 1862 None, 1863 description="Credential artifact used to get a new access token.", 1864 examples=[ 1865 "{{ config['refresh_token'] }}", 1866 "{{ config['credentials]['refresh_token'] }}", 1867 ], 1868 title="Refresh Token", 1869 ) 1870 token_refresh_endpoint: Optional[str] = Field( 1871 None, 1872 description="The full URL to call to obtain a new access token.", 1873 examples=["https://connect.squareup.com/oauth2/token"], 1874 title="Token Refresh Endpoint", 1875 ) 1876 access_token_name: Optional[str] = Field( 1877 "access_token", 1878 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1879 examples=["access_token"], 1880 title="Access Token Property Name", 1881 ) 1882 access_token_value: Optional[str] = Field( 1883 None, 1884 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1885 examples=["secret_access_token_value"], 1886 title="Access Token Value", 1887 ) 1888 expires_in_name: Optional[str] = Field( 1889 "expires_in", 1890 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1891 examples=["expires_in"], 1892 title="Token Expiry Property Name", 1893 ) 1894 grant_type_name: Optional[str] = Field( 1895 "grant_type", 1896 description="The name of the property to use to refresh the `access_token`.", 1897 examples=["custom_grant_type"], 1898 title="Grant Type Property Name", 1899 ) 1900 grant_type: Optional[str] = Field( 1901 "refresh_token", 1902 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1903 examples=["refresh_token", "client_credentials"], 1904 title="Grant Type", 1905 ) 1906 refresh_request_body: Optional[Dict[str, Any]] = Field( 1907 None, 1908 description="Body of the request sent to get a new access token.", 1909 examples=[ 1910 { 1911 "applicationId": "{{ config['application_id'] }}", 1912 "applicationSecret": "{{ config['application_secret'] }}", 1913 "token": "{{ config['token'] }}", 1914 } 1915 ], 1916 title="Refresh Request Body", 1917 ) 1918 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1919 None, 1920 description="Headers of the request sent to get a new access token.", 1921 examples=[ 1922 { 1923 "Authorization": "<AUTH_TOKEN>", 1924 "Content-Type": "application/x-www-form-urlencoded", 1925 } 1926 ], 1927 title="Refresh Request Headers", 1928 ) 1929 scopes: Optional[List[str]] = Field( 1930 None, 1931 description="List of scopes that should be granted to the access token.", 1932 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1933 title="Scopes", 1934 ) 1935 token_expiry_date: Optional[str] = Field( 1936 None, 1937 description="The access token expiry date.", 1938 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1939 title="Token Expiry Date", 1940 ) 1941 token_expiry_date_format: Optional[str] = Field( 1942 None, 1943 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1944 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1945 title="Token Expiry Date Format", 1946 ) 1947 refresh_token_error_status_codes: Optional[List[int]] = Field( 1948 None, 1949 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1950 examples=[[400, 500]], 1951 title="Refresh Token Error Status Codes", 1952 ) 1953 refresh_token_error_key: Optional[str] = Field( 1954 None, 1955 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1956 examples=["error"], 1957 title="Refresh Token Error Key", 1958 ) 1959 refresh_token_error_values: Optional[List[str]] = Field( 1960 None, 1961 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1962 examples=[["invalid_grant", "invalid_permissions"]], 1963 title="Refresh Token Error Values", 1964 ) 1965 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1966 None, 1967 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1968 title="Refresh Token Updater", 1969 ) 1970 profile_assertion: Optional[JwtAuthenticator] = Field( 1971 None, 1972 description="The authenticator being used to authenticate the client authenticator.", 1973 title="Profile Assertion", 1974 ) 1975 use_profile_assertion: Optional[bool] = Field( 1976 False, 1977 description="Enable using profile assertion as a flow for OAuth authorization.", 1978 title="Use Profile Assertion", 1979 ) 1980 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1983class FixedWindowCallRatePolicy(BaseModel): 1984 class Config: 1985 extra = Extra.allow 1986 1987 type: Literal["FixedWindowCallRatePolicy"] 1988 period: str = Field( 1989 ..., description="The time interval for the rate limit window.", title="Period" 1990 ) 1991 call_limit: int = Field( 1992 ..., 1993 description="The maximum number of calls allowed within the period.", 1994 title="Call Limit", 1995 ) 1996 matchers: List[HttpRequestRegexMatcher] = Field( 1997 ..., 1998 description="List of matchers that define which requests this policy applies to.", 1999 title="Matchers", 2000 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2003class MovingWindowCallRatePolicy(BaseModel): 2004 class Config: 2005 extra = Extra.allow 2006 2007 type: Literal["MovingWindowCallRatePolicy"] 2008 rates: List[Rate] = Field( 2009 ..., 2010 description="List of rates that define the call limits for different time intervals.", 2011 title="Rates", 2012 ) 2013 matchers: List[HttpRequestRegexMatcher] = Field( 2014 ..., 2015 description="List of matchers that define which requests this policy applies to.", 2016 title="Matchers", 2017 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2020class UnlimitedCallRatePolicy(BaseModel): 2021 class Config: 2022 extra = Extra.allow 2023 2024 type: Literal["UnlimitedCallRatePolicy"] 2025 matchers: List[HttpRequestRegexMatcher] = Field( 2026 ..., 2027 description="List of matchers that define which requests this policy applies to.", 2028 title="Matchers", 2029 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2032class DefaultErrorHandler(BaseModel): 2033 type: Literal["DefaultErrorHandler"] 2034 backoff_strategies: Optional[ 2035 List[ 2036 Union[ 2037 ConstantBackoffStrategy, 2038 ExponentialBackoffStrategy, 2039 WaitTimeFromHeader, 2040 WaitUntilTimeFromHeader, 2041 CustomBackoffStrategy, 2042 ] 2043 ] 2044 ] = Field( 2045 None, 2046 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2047 title="Backoff Strategies", 2048 ) 2049 max_retries: Optional[Union[int, str]] = Field( 2050 5, 2051 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2052 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2053 title="Max Retry Count", 2054 ) 2055 response_filters: Optional[List[HttpResponseFilter]] = Field( 2056 None, 2057 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2058 title="Response Filters", 2059 ) 2060 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2063class DefaultPaginator(BaseModel): 2064 type: Literal["DefaultPaginator"] 2065 pagination_strategy: Union[ 2066 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2067 ] = Field( 2068 ..., 2069 description="Strategy defining how records are paginated.", 2070 title="Pagination Strategy", 2071 ) 2072 page_size_option: Optional[RequestOption] = Field( 2073 None, title="Inject Page Size Into Outgoing HTTP Request" 2074 ) 2075 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2076 None, 2077 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2078 title="Inject Page Token Into Outgoing HTTP Request", 2079 ) 2080 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2083class RecordExpander(BaseModel): 2084 type: Literal["RecordExpander"] 2085 expand_records_from_field: List[str] = Field( 2086 ..., 2087 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2088 examples=[ 2089 ["lines", "data"], 2090 ["items"], 2091 ["nested", "array"], 2092 ["sections", "*", "items"], 2093 ], 2094 title="Expand Records From Field", 2095 ) 2096 remain_original_record: Optional[bool] = Field( 2097 False, 2098 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2099 title="Remain Original Record", 2100 ) 2101 on_no_records: Optional[OnNoRecords] = Field( 2102 OnNoRecords.skip, 2103 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2104 title="On No Records", 2105 ) 2106 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2109class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2110 type: Literal["ApiKey"] 2111 inject_into: RequestOption = Field( 2112 ..., 2113 description="Configure how the API Key will be sent in requests to the source API.", 2114 examples=[ 2115 {"inject_into": "header", "field_name": "Authorization"}, 2116 {"inject_into": "request_parameter", "field_name": "authKey"}, 2117 ], 2118 title="Inject API Key Into Outgoing HTTP Request", 2119 ) 2120 api_token: Optional[str] = Field( 2121 "{{ session_token }}", 2122 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2123 examples=[ 2124 "{{ session_token }}", 2125 "Token {{ session_token }}", 2126 "Bearer {{ session_token }}", 2127 ], 2128 title="API Token Template", 2129 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2132class JsonSchemaPropertySelector(BaseModel): 2133 type: Literal["JsonSchemaPropertySelector"] 2134 transformations: Optional[ 2135 List[ 2136 Union[ 2137 AddFields, 2138 RemoveFields, 2139 KeysToLower, 2140 KeysToSnakeCase, 2141 FlattenFields, 2142 DpathFlattenFields, 2143 KeysReplace, 2144 CustomTransformation, 2145 ] 2146 ] 2147 ] = Field( 2148 None, 2149 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2150 title="Transformations", 2151 ) 2152 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2155class ListPartitionRouter(BaseModel): 2156 type: Literal["ListPartitionRouter"] 2157 cursor_field: str = Field( 2158 ..., 2159 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2160 examples=["section", "{{ config['section_key'] }}"], 2161 title="Current Partition Value Identifier", 2162 ) 2163 values: Union[str, List[str]] = Field( 2164 ..., 2165 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2166 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2167 title="Partition Values", 2168 ) 2169 request_option: Optional[RequestOption] = Field( 2170 None, 2171 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2172 title="Inject Partition Value Into Outgoing HTTP Request", 2173 ) 2174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2177class PaginationReset(BaseModel): 2178 type: Literal["PaginationReset"] 2179 action: Action1 2180 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2183class GzipDecoder(BaseModel): 2184 type: Literal["GzipDecoder"] 2185 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2188class RequestBodyGraphQL(BaseModel): 2189 type: Literal["RequestBodyGraphQL"] 2190 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2193class DpathValidator(BaseModel): 2194 type: Literal["DpathValidator"] 2195 field_path: List[str] = Field( 2196 ..., 2197 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2198 examples=[ 2199 ["data"], 2200 ["data", "records"], 2201 ["data", "{{ parameters.name }}"], 2202 ["data", "*", "record"], 2203 ], 2204 title="Field Path", 2205 ) 2206 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2207 ..., 2208 description="The condition that the specified config value will be evaluated against", 2209 title="Validation Strategy", 2210 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2213class PredicateValidator(BaseModel): 2214 type: Literal["PredicateValidator"] 2215 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2216 ..., 2217 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2218 examples=[ 2219 "test-value", 2220 "{{ config['api_version'] }}", 2221 "{{ config['tenant_id'] }}", 2222 123, 2223 ], 2224 title="Value", 2225 ) 2226 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2227 ..., 2228 description="The validation strategy to apply to the value.", 2229 title="Validation Strategy", 2230 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2233class ConfigAddFields(BaseModel): 2234 type: Literal["ConfigAddFields"] 2235 fields: List[AddedFieldDefinition] = Field( 2236 ..., 2237 description="A list of transformations (path and corresponding value) that will be added to the config.", 2238 title="Fields", 2239 ) 2240 condition: Optional[str] = Field( 2241 "", 2242 description="Fields will be added if expression is evaluated to True.", 2243 examples=[ 2244 "{{ config['environemnt'] == 'sandbox' }}", 2245 "{{ property is integer }}", 2246 "{{ property|length > 5 }}", 2247 "{{ property == 'some_string_to_match' }}", 2248 ], 2249 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2252class CompositeErrorHandler(BaseModel): 2253 type: Literal["CompositeErrorHandler"] 2254 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2255 Field( 2256 ..., 2257 description="List of error handlers to iterate on to determine how to handle a failed response.", 2258 title="Error Handlers", 2259 ) 2260 ) 2261 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2264class HTTPAPIBudget(BaseModel): 2265 class Config: 2266 extra = Extra.allow 2267 2268 type: Literal["HTTPAPIBudget"] 2269 policies: List[ 2270 Union[ 2271 FixedWindowCallRatePolicy, 2272 MovingWindowCallRatePolicy, 2273 UnlimitedCallRatePolicy, 2274 ] 2275 ] = Field( 2276 ..., 2277 description="List of call rate policies that define how many calls are allowed.", 2278 title="Policies", 2279 ) 2280 ratelimit_reset_header: Optional[str] = Field( 2281 "ratelimit-reset", 2282 description="The HTTP response header name that indicates when the rate limit resets.", 2283 title="Rate Limit Reset Header", 2284 ) 2285 ratelimit_remaining_header: Optional[str] = Field( 2286 "ratelimit-remaining", 2287 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2288 title="Rate Limit Remaining Header", 2289 ) 2290 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2291 [429], 2292 description="List of HTTP status codes that indicate a rate limit has been hit.", 2293 title="Status Codes for Rate Limit Hit", 2294 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2297class DpathExtractor(BaseModel): 2298 type: Literal["DpathExtractor"] 2299 field_path: List[str] = Field( 2300 ..., 2301 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2302 examples=[ 2303 ["data"], 2304 ["data", "records"], 2305 ["data", "{{ parameters.name }}"], 2306 ["data", "*", "record"], 2307 ], 2308 title="Field Path", 2309 ) 2310 record_expander: Optional[RecordExpander] = Field( 2311 None, 2312 description="Optional component to expand records by extracting items from nested array fields.", 2313 title="Record Expander", 2314 ) 2315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2318class ZipfileDecoder(BaseModel): 2319 class Config: 2320 extra = Extra.allow 2321 2322 type: Literal["ZipfileDecoder"] 2323 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2324 ..., 2325 description="Parser to parse the decompressed data from the zipfile(s).", 2326 title="Parser", 2327 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2330class RecordSelector(BaseModel): 2331 type: Literal["RecordSelector"] 2332 extractor: Union[DpathExtractor, CustomRecordExtractor] 2333 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2334 None, 2335 description="Responsible for filtering records to be emitted by the Source.", 2336 title="Record Filter", 2337 ) 2338 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2339 None, 2340 description="Responsible for normalization according to the schema.", 2341 title="Schema Normalization", 2342 ) 2343 transform_before_filtering: Optional[bool] = Field( 2344 None, 2345 description="If true, transformation will be applied before record filtering.", 2346 title="Transform Before Filtering", 2347 ) 2348 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2351class ConfigMigration(BaseModel): 2352 type: Literal["ConfigMigration"] 2353 description: Optional[str] = Field( 2354 None, description="The description/purpose of the config migration." 2355 ) 2356 transformations: List[ 2357 Union[ 2358 ConfigRemapField, 2359 ConfigAddFields, 2360 ConfigRemoveFields, 2361 CustomConfigTransformation, 2362 ] 2363 ] = Field( 2364 ..., 2365 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2366 title="Transformations", 2367 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2370class ConfigNormalizationRules(BaseModel): 2371 class Config: 2372 extra = Extra.forbid 2373 2374 type: Literal["ConfigNormalizationRules"] 2375 config_migrations: Optional[List[ConfigMigration]] = Field( 2376 [], 2377 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2378 title="Config Migrations", 2379 ) 2380 transformations: Optional[ 2381 List[ 2382 Union[ 2383 ConfigRemapField, 2384 ConfigAddFields, 2385 ConfigRemoveFields, 2386 CustomConfigTransformation, 2387 ] 2388 ] 2389 ] = Field( 2390 [], 2391 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2392 title="Transformations", 2393 ) 2394 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2395 [], 2396 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2397 title="Validations", 2398 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2401class Spec(BaseModel): 2402 type: Literal["Spec"] 2403 connection_specification: Dict[str, Any] = Field( 2404 ..., 2405 description="A connection specification describing how a the connector can be configured.", 2406 title="Connection Specification", 2407 ) 2408 documentation_url: Optional[str] = Field( 2409 None, 2410 description="URL of the connector's documentation page.", 2411 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2412 title="Documentation URL", 2413 ) 2414 advanced_auth: Optional[AuthFlow] = Field( 2415 None, 2416 description="Advanced specification for configuring the authentication flow.", 2417 title="Advanced Auth", 2418 ) 2419 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2420 None, title="Config Normalization Rules" 2421 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2424class DeclarativeSource1(BaseModel): 2425 class Config: 2426 extra = Extra.forbid 2427 2428 type: Literal["DeclarativeSource"] 2429 check: Union[CheckStream, CheckDynamicStream] 2430 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2431 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2432 version: str = Field( 2433 ..., 2434 description="The version of the Airbyte CDK used to build and test the source.", 2435 ) 2436 schemas: Optional[Schemas] = None 2437 definitions: Optional[Dict[str, Any]] = None 2438 spec: Optional[Spec] = None 2439 concurrency_level: Optional[ConcurrencyLevel] = None 2440 api_budget: Optional[HTTPAPIBudget] = None 2441 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2442 None, 2443 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2444 title="Stream Groups", 2445 ) 2446 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2447 None, 2448 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2449 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2450 title="Maximum Concurrent Asynchronous Jobs", 2451 ) 2452 metadata: Optional[Dict[str, Any]] = Field( 2453 None, 2454 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2455 ) 2456 description: Optional[str] = Field( 2457 None, 2458 description="A description of the connector. It will be presented on the Source documentation page.", 2459 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2462class DeclarativeSource2(BaseModel): 2463 class Config: 2464 extra = Extra.forbid 2465 2466 type: Literal["DeclarativeSource"] 2467 check: Union[CheckStream, CheckDynamicStream] 2468 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2469 None 2470 ) 2471 dynamic_streams: List[DynamicDeclarativeStream] 2472 version: str = Field( 2473 ..., 2474 description="The version of the Airbyte CDK used to build and test the source.", 2475 ) 2476 schemas: Optional[Schemas] = None 2477 definitions: Optional[Dict[str, Any]] = None 2478 spec: Optional[Spec] = None 2479 concurrency_level: Optional[ConcurrencyLevel] = None 2480 api_budget: Optional[HTTPAPIBudget] = None 2481 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2482 None, 2483 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2484 title="Stream Groups", 2485 ) 2486 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2487 None, 2488 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2489 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2490 title="Maximum Concurrent Asynchronous Jobs", 2491 ) 2492 metadata: Optional[Dict[str, Any]] = Field( 2493 None, 2494 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2495 ) 2496 description: Optional[str] = Field( 2497 None, 2498 description="A description of the connector. It will be presented on the Source documentation page.", 2499 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2502class DeclarativeSource(BaseModel): 2503 class Config: 2504 extra = Extra.forbid 2505 2506 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2507 ..., 2508 description="An API source that extracts data according to its declarative components.", 2509 title="DeclarativeSource", 2510 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2513class SelectiveAuthenticator(BaseModel): 2514 class Config: 2515 extra = Extra.allow 2516 2517 type: Literal["SelectiveAuthenticator"] 2518 authenticator_selection_path: List[str] = Field( 2519 ..., 2520 description="Path of the field in config with selected authenticator name", 2521 examples=[["auth"], ["auth", "type"]], 2522 title="Authenticator Selection Path", 2523 ) 2524 authenticators: Dict[ 2525 str, 2526 Union[ 2527 ApiKeyAuthenticator, 2528 BasicHttpAuthenticator, 2529 BearerAuthenticator, 2530 OAuthAuthenticator, 2531 JwtAuthenticator, 2532 SessionTokenAuthenticator, 2533 LegacySessionTokenAuthenticator, 2534 CustomAuthenticator, 2535 NoAuth, 2536 ], 2537 ] = Field( 2538 ..., 2539 description="Authenticators to select from.", 2540 examples=[ 2541 { 2542 "authenticators": { 2543 "token": "#/definitions/ApiKeyAuthenticator", 2544 "oauth": "#/definitions/OAuthAuthenticator", 2545 "jwt": "#/definitions/JwtAuthenticator", 2546 } 2547 } 2548 ], 2549 title="Authenticators", 2550 ) 2551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2554class ConditionalStreams(BaseModel): 2555 type: Literal["ConditionalStreams"] 2556 condition: str = Field( 2557 ..., 2558 description="Condition that will be evaluated to determine if a set of streams should be available.", 2559 examples=["{{ config['is_sandbox'] }}"], 2560 title="Condition", 2561 ) 2562 streams: List[DeclarativeStream] = Field( 2563 ..., 2564 description="Streams that will be used during an operation based on the condition.", 2565 title="Streams", 2566 ) 2567 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2570class FileUploader(BaseModel): 2571 type: Literal["FileUploader"] 2572 requester: Union[HttpRequester, CustomRequester] = Field( 2573 ..., 2574 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2575 ) 2576 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2577 ..., 2578 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2579 ) 2580 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2581 None, 2582 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2583 ) 2584 filename_extractor: Optional[str] = Field( 2585 None, 2586 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2587 examples=[ 2588 "{{ record.id }}/{{ record.file_name }}/", 2589 "{{ record.id }}_{{ record.file_name }}/", 2590 ], 2591 ) 2592 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2595class DeclarativeStream(BaseModel): 2596 class Config: 2597 extra = Extra.allow 2598 2599 type: Literal["DeclarativeStream"] 2600 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2601 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2602 ..., 2603 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2604 title="Retriever", 2605 ) 2606 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2607 None, 2608 description="Component used to fetch data incrementally based on a time field in the data.", 2609 title="Incremental Sync", 2610 ) 2611 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2612 schema_loader: Optional[ 2613 Union[ 2614 InlineSchemaLoader, 2615 DynamicSchemaLoader, 2616 JsonFileSchemaLoader, 2617 List[ 2618 Union[ 2619 InlineSchemaLoader, 2620 DynamicSchemaLoader, 2621 JsonFileSchemaLoader, 2622 CustomSchemaLoader, 2623 ] 2624 ], 2625 CustomSchemaLoader, 2626 ] 2627 ] = Field( 2628 None, 2629 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2630 title="Schema Loader", 2631 ) 2632 transformations: Optional[ 2633 List[ 2634 Union[ 2635 AddFields, 2636 RemoveFields, 2637 KeysToLower, 2638 KeysToSnakeCase, 2639 FlattenFields, 2640 DpathFlattenFields, 2641 KeysReplace, 2642 CustomTransformation, 2643 ] 2644 ] 2645 ] = Field( 2646 None, 2647 description="A list of transformations to be applied to each output record.", 2648 title="Transformations", 2649 ) 2650 state_migrations: Optional[ 2651 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2652 ] = Field( 2653 [], 2654 description="Array of state migrations to be applied on the input state", 2655 title="State Migrations", 2656 ) 2657 file_uploader: Optional[FileUploader] = Field( 2658 None, 2659 description="(experimental) Describes how to fetch a file", 2660 title="File Uploader", 2661 ) 2662 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2665class SessionTokenAuthenticator(BaseModel): 2666 type: Literal["SessionTokenAuthenticator"] 2667 login_requester: HttpRequester = Field( 2668 ..., 2669 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2670 examples=[ 2671 { 2672 "type": "HttpRequester", 2673 "url_base": "https://my_api.com", 2674 "path": "/login", 2675 "authenticator": { 2676 "type": "BasicHttpAuthenticator", 2677 "username": "{{ config.username }}", 2678 "password": "{{ config.password }}", 2679 }, 2680 } 2681 ], 2682 title="Login Requester", 2683 ) 2684 session_token_path: List[str] = Field( 2685 ..., 2686 description="The path in the response body returned from the login requester to the session token.", 2687 examples=[["access_token"], ["result", "token"]], 2688 title="Session Token Path", 2689 ) 2690 expiration_duration: Optional[str] = Field( 2691 None, 2692 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2693 examples=["PT1H", "P1D"], 2694 title="Expiration Duration", 2695 ) 2696 request_authentication: Union[ 2697 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2698 ] = Field( 2699 ..., 2700 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2701 title="Data Request Authentication", 2702 ) 2703 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2704 None, description="Component used to decode the response.", title="Decoder" 2705 ) 2706 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2709class HttpRequester(BaseModelWithDeprecations): 2710 type: Literal["HttpRequester"] 2711 url_base: Optional[str] = Field( 2712 None, 2713 deprecated=True, 2714 deprecation_message="Use `url` field instead.", 2715 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2716 examples=[ 2717 "https://connect.squareup.com/v2", 2718 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2719 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2720 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2721 ], 2722 title="API Base URL", 2723 ) 2724 url: Optional[str] = Field( 2725 None, 2726 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2727 examples=[ 2728 "https://connect.squareup.com/v2", 2729 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2730 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2731 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2732 ], 2733 title="API Endpoint URL", 2734 ) 2735 path: Optional[str] = Field( 2736 None, 2737 deprecated=True, 2738 deprecation_message="Use `url` field instead.", 2739 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2740 examples=[ 2741 "/products", 2742 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2743 "/trades/{{ config['symbol_id'] }}/history", 2744 ], 2745 title="URL Path", 2746 ) 2747 http_method: Optional[HttpMethod] = Field( 2748 HttpMethod.GET, 2749 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2750 examples=["GET", "POST"], 2751 title="HTTP Method", 2752 ) 2753 authenticator: Optional[ 2754 Union[ 2755 ApiKeyAuthenticator, 2756 BasicHttpAuthenticator, 2757 BearerAuthenticator, 2758 OAuthAuthenticator, 2759 JwtAuthenticator, 2760 SessionTokenAuthenticator, 2761 SelectiveAuthenticator, 2762 CustomAuthenticator, 2763 NoAuth, 2764 LegacySessionTokenAuthenticator, 2765 ] 2766 ] = Field( 2767 None, 2768 description="Authentication method to use for requests sent to the API.", 2769 title="Authenticator", 2770 ) 2771 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2772 None, 2773 deprecated=True, 2774 deprecation_message="Use `query_properties` field instead.", 2775 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2776 title="Fetch Properties from Endpoint", 2777 ) 2778 query_properties: Optional[QueryProperties] = Field( 2779 None, 2780 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2781 title="Query Properties", 2782 ) 2783 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2784 None, 2785 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2786 examples=[ 2787 {"unit": "day"}, 2788 { 2789 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2790 }, 2791 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2792 {"sort_by[asc]": "updated_at"}, 2793 ], 2794 title="Query Parameters", 2795 ) 2796 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2797 None, 2798 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2799 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2800 title="Request Headers", 2801 ) 2802 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2803 None, 2804 deprecated=True, 2805 deprecation_message="Use `request_body` field instead.", 2806 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2807 examples=[ 2808 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2809 ], 2810 title="Request Body Payload (Non-JSON)", 2811 ) 2812 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2813 None, 2814 deprecated=True, 2815 deprecation_message="Use `request_body` field instead.", 2816 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2817 examples=[ 2818 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2819 {"key": "{{ config['value'] }}"}, 2820 {"sort": {"field": "updated_at", "order": "ascending"}}, 2821 ], 2822 title="Request Body JSON Payload", 2823 ) 2824 request_body: Optional[ 2825 Union[ 2826 RequestBodyPlainText, 2827 RequestBodyUrlEncodedForm, 2828 RequestBodyJsonObject, 2829 RequestBodyGraphQL, 2830 ] 2831 ] = Field( 2832 None, 2833 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2834 title="Request Body", 2835 ) 2836 error_handler: Optional[ 2837 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2838 ] = Field( 2839 None, 2840 description="Error handler component that defines how to handle errors.", 2841 title="Error Handler", 2842 ) 2843 use_cache: Optional[bool] = Field( 2844 False, 2845 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2846 title="Use Cache", 2847 ) 2848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2851class DynamicSchemaLoader(BaseModel): 2852 type: Literal["DynamicSchemaLoader"] 2853 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2854 ..., 2855 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2856 title="Retriever", 2857 ) 2858 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2859 None, 2860 description="Responsible for filtering fields to be added to json schema.", 2861 title="Schema Filter", 2862 ) 2863 schema_transformations: Optional[ 2864 List[ 2865 Union[ 2866 AddFields, 2867 RemoveFields, 2868 KeysToLower, 2869 KeysToSnakeCase, 2870 FlattenFields, 2871 DpathFlattenFields, 2872 KeysReplace, 2873 CustomTransformation, 2874 ] 2875 ] 2876 ] = Field( 2877 None, 2878 description="A list of transformations to be applied to the schema.", 2879 title="Schema Transformations", 2880 ) 2881 schema_type_identifier: SchemaTypeIdentifier 2882 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2885class ParentStreamConfig(BaseModel): 2886 type: Literal["ParentStreamConfig"] 2887 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2888 ..., description="Reference to the parent stream.", title="Parent Stream" 2889 ) 2890 parent_key: str = Field( 2891 ..., 2892 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2893 examples=["id", "{{ config['parent_record_id'] }}"], 2894 title="Parent Key", 2895 ) 2896 partition_field: str = Field( 2897 ..., 2898 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2899 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2900 title="Current Parent Key Value Identifier", 2901 ) 2902 request_option: Optional[RequestOption] = Field( 2903 None, 2904 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2905 title="Request Option", 2906 ) 2907 incremental_dependency: Optional[bool] = Field( 2908 False, 2909 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2910 title="Incremental Dependency", 2911 ) 2912 lazy_read_pointer: Optional[List[str]] = Field( 2913 [], 2914 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2915 title="Lazy Read Pointer", 2916 ) 2917 extra_fields: Optional[List[List[str]]] = Field( 2918 None, 2919 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2920 title="Extra Fields", 2921 ) 2922 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2925class PropertiesFromEndpoint(BaseModel): 2926 type: Literal["PropertiesFromEndpoint"] 2927 property_field_path: List[str] = Field( 2928 ..., 2929 description="Describes the path to the field that should be extracted", 2930 examples=[["name"]], 2931 ) 2932 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2933 ..., 2934 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2935 ) 2936 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2939class QueryProperties(BaseModel): 2940 type: Literal["QueryProperties"] 2941 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2942 ..., 2943 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2944 title="Property List", 2945 ) 2946 always_include_properties: Optional[List[str]] = Field( 2947 None, 2948 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2949 title="Always Include Properties", 2950 ) 2951 property_chunking: Optional[PropertyChunking] = Field( 2952 None, 2953 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2954 title="Property Chunking", 2955 ) 2956 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2957 None, 2958 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2959 title="Property Selector", 2960 ) 2961 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2964class StateDelegatingStream(BaseModel): 2965 type: Literal["StateDelegatingStream"] 2966 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2967 full_refresh_stream: DeclarativeStream = Field( 2968 ..., 2969 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2970 title="Full Refresh Stream", 2971 ) 2972 incremental_stream: DeclarativeStream = Field( 2973 ..., 2974 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2975 title="Incremental Stream", 2976 ) 2977 api_retention_period: Optional[str] = Field( 2978 None, 2979 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2980 examples=["P30D", "P90D", "P1Y"], 2981 title="API Retention Period", 2982 ) 2983 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2986class SimpleRetriever(BaseModel): 2987 type: Literal["SimpleRetriever"] 2988 requester: Union[HttpRequester, CustomRequester] = Field( 2989 ..., 2990 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2991 ) 2992 decoder: Optional[ 2993 Union[ 2994 JsonDecoder, 2995 XmlDecoder, 2996 CsvDecoder, 2997 JsonlDecoder, 2998 GzipDecoder, 2999 IterableDecoder, 3000 ZipfileDecoder, 3001 CustomDecoder, 3002 ] 3003 ] = Field( 3004 None, 3005 description="Component decoding the response so records can be extracted.", 3006 title="HTTP Response Format", 3007 ) 3008 record_selector: RecordSelector = Field( 3009 ..., 3010 description="Component that describes how to extract records from a HTTP response.", 3011 ) 3012 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3013 None, 3014 description="Paginator component that describes how to navigate through the API's pages.", 3015 ) 3016 pagination_reset: Optional[PaginationReset] = Field( 3017 None, 3018 description="Describes what triggers pagination reset and how to handle it.", 3019 ) 3020 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3021 False, 3022 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3023 ) 3024 partition_router: Optional[ 3025 Union[ 3026 SubstreamPartitionRouter, 3027 ListPartitionRouter, 3028 GroupingPartitionRouter, 3029 CustomPartitionRouter, 3030 List[ 3031 Union[ 3032 SubstreamPartitionRouter, 3033 ListPartitionRouter, 3034 GroupingPartitionRouter, 3035 CustomPartitionRouter, 3036 ] 3037 ], 3038 ] 3039 ] = Field( 3040 None, 3041 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3042 title="Partition Router", 3043 ) 3044 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3047class AsyncRetriever(BaseModel): 3048 type: Literal["AsyncRetriever"] 3049 record_selector: RecordSelector = Field( 3050 ..., 3051 description="Component that describes how to extract records from a HTTP response.", 3052 ) 3053 status_mapping: AsyncJobStatusMap = Field( 3054 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3055 ) 3056 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3057 ..., description="Responsible for fetching the actual status of the async job." 3058 ) 3059 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3060 None, 3061 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3062 ) 3063 download_extractor: Optional[ 3064 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3065 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3066 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3067 ..., 3068 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3069 ) 3070 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3071 ..., 3072 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3073 ) 3074 polling_job_timeout: Optional[Union[int, str]] = Field( 3075 None, 3076 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3077 ) 3078 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3079 None, 3080 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3081 ) 3082 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3083 None, 3084 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3085 ) 3086 download_requester: Union[HttpRequester, CustomRequester] = Field( 3087 ..., 3088 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3089 ) 3090 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3091 None, 3092 description="Paginator component that describes how to navigate through the API's pages during download.", 3093 ) 3094 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3095 None, 3096 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3097 ) 3098 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3099 None, 3100 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3101 ) 3102 partition_router: Optional[ 3103 Union[ 3104 ListPartitionRouter, 3105 SubstreamPartitionRouter, 3106 GroupingPartitionRouter, 3107 CustomPartitionRouter, 3108 List[ 3109 Union[ 3110 ListPartitionRouter, 3111 SubstreamPartitionRouter, 3112 GroupingPartitionRouter, 3113 CustomPartitionRouter, 3114 ] 3115 ], 3116 ] 3117 ] = Field( 3118 [], 3119 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3120 title="Partition Router", 3121 ) 3122 decoder: Optional[ 3123 Union[ 3124 CsvDecoder, 3125 GzipDecoder, 3126 JsonDecoder, 3127 JsonlDecoder, 3128 IterableDecoder, 3129 XmlDecoder, 3130 ZipfileDecoder, 3131 CustomDecoder, 3132 ] 3133 ] = Field( 3134 None, 3135 description="Component decoding the response so records can be extracted.", 3136 title="HTTP Response Format", 3137 ) 3138 download_decoder: Optional[ 3139 Union[ 3140 CsvDecoder, 3141 GzipDecoder, 3142 JsonDecoder, 3143 JsonlDecoder, 3144 IterableDecoder, 3145 XmlDecoder, 3146 ZipfileDecoder, 3147 CustomDecoder, 3148 ] 3149 ] = Field( 3150 None, 3151 description="Component decoding the download response so records can be extracted.", 3152 title="Download HTTP Response Format", 3153 ) 3154 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3161class StreamGroup(BaseModel): 3162 streams: List[str] = Field( 3163 ..., 3164 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3165 title="Streams", 3166 ) 3167 action: BlockSimultaneousSyncsAction = Field( 3168 ..., 3169 description="The action to apply to streams in this group.", 3170 title="Action", 3171 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3174class SubstreamPartitionRouter(BaseModel): 3175 type: Literal["SubstreamPartitionRouter"] 3176 parent_stream_configs: List[ParentStreamConfig] = Field( 3177 ..., 3178 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3179 title="Parent Stream Configs", 3180 ) 3181 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3184class GroupingPartitionRouter(BaseModel): 3185 type: Literal["GroupingPartitionRouter"] 3186 group_size: int = Field( 3187 ..., 3188 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3189 examples=[10, 50], 3190 title="Group Size", 3191 ) 3192 underlying_partition_router: Union[ 3193 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3194 ] = Field( 3195 ..., 3196 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3197 title="Underlying Partition Router", 3198 ) 3199 deduplicate: Optional[bool] = Field( 3200 True, 3201 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3202 title="Deduplicate Partitions", 3203 ) 3204 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3207class HttpComponentsResolver(BaseModel): 3208 type: Literal["HttpComponentsResolver"] 3209 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3210 ..., 3211 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3212 title="Retriever", 3213 ) 3214 components_mapping: List[ComponentMappingDefinition] 3215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3218class DynamicDeclarativeStream(BaseModel): 3219 type: Literal["DynamicDeclarativeStream"] 3220 name: Optional[str] = Field( 3221 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3222 ) 3223 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3224 ..., description="Reference to the stream template.", title="Stream Template" 3225 ) 3226 components_resolver: Union[ 3227 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3228 ] = Field( 3229 ..., 3230 description="Component resolve and populates stream templates with components values.", 3231 title="Components Resolver", 3232 ) 3233 use_parent_parameters: Optional[bool] = Field( 3234 True, 3235 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3236 title="Use Parent Parameters", 3237 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]