airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 108 109 110class CursorPagination(BaseModel): 111 type: Literal["CursorPagination"] 112 cursor_value: str = Field( 113 ..., 114 description="Value of the cursor defining the next page to fetch.", 115 examples=[ 116 "{{ headers.link.next.cursor }}", 117 "{{ last_record['key'] }}", 118 "{{ response['nextPage'] }}", 119 ], 120 title="Cursor Value", 121 ) 122 page_size: Optional[Union[int, str]] = Field( 123 None, 124 description="The number of records to include in each pages.", 125 examples=[100, "{{ config['page_size'] }}"], 126 title="Page Size", 127 ) 128 stop_condition: Optional[str] = Field( 129 None, 130 description="Template string evaluating when to stop paginating.", 131 examples=[ 132 "{{ response.data.has_more is false }}", 133 "{{ 'next' not in headers['link'] }}", 134 ], 135 title="Stop Condition", 136 ) 137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 138 139 140class CustomAuthenticator(BaseModel): 141 class Config: 142 extra = Extra.allow 143 144 type: Literal["CustomAuthenticator"] 145 class_name: str = Field( 146 ..., 147 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 148 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 149 title="Class Name", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 152 153 154class CustomBackoffStrategy(BaseModel): 155 class Config: 156 extra = Extra.allow 157 158 type: Literal["CustomBackoffStrategy"] 159 class_name: str = Field( 160 ..., 161 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 162 examples=["source_railz.components.MyCustomBackoffStrategy"], 163 title="Class Name", 164 ) 165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 166 167 168class CustomErrorHandler(BaseModel): 169 class Config: 170 extra = Extra.allow 171 172 type: Literal["CustomErrorHandler"] 173 class_name: str = Field( 174 ..., 175 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 176 examples=["source_railz.components.MyCustomErrorHandler"], 177 title="Class Name", 178 ) 179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 180 181 182class CustomPaginationStrategy(BaseModel): 183 class Config: 184 extra = Extra.allow 185 186 type: Literal["CustomPaginationStrategy"] 187 class_name: str = Field( 188 ..., 189 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 190 examples=["source_railz.components.MyCustomPaginationStrategy"], 191 title="Class Name", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 194 195 196class CustomRecordExtractor(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomRecordExtractor"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomRecordExtractor"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 208 209 210class CustomRecordFilter(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordFilter"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomCustomRecordFilter"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 222 223 224class CustomRequester(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRequester"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomRecordExtractor"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 236 237 238class CustomRetriever(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRetriever"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRetriever"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 250 251 252class CustomPartitionRouter(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomPartitionRouter"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomPartitionRouter"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 264 265 266class CustomSchemaLoader(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomSchemaLoader"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomSchemaLoader"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 278 279 280class CustomSchemaNormalization(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaNormalization"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 288 examples=[ 289 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 290 ], 291 title="Class Name", 292 ) 293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 294 295 296class CustomStateMigration(BaseModel): 297 class Config: 298 extra = Extra.allow 299 300 type: Literal["CustomStateMigration"] 301 class_name: str = Field( 302 ..., 303 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 304 examples=["source_railz.components.MyCustomStateMigration"], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 308 309 310class CustomTransformation(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomTransformation"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomTransformation"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 322 323 324class LegacyToPerPartitionStateMigration(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 329 330 331class Clamping(BaseModel): 332 target: str = Field( 333 ..., 334 description="The period of time that datetime windows will be clamped by", 335 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 336 title="Target", 337 ) 338 target_details: Optional[Dict[str, Any]] = None 339 340 341class Algorithm(Enum): 342 HS256 = "HS256" 343 HS384 = "HS384" 344 HS512 = "HS512" 345 ES256 = "ES256" 346 ES256K = "ES256K" 347 ES384 = "ES384" 348 ES512 = "ES512" 349 RS256 = "RS256" 350 RS384 = "RS384" 351 RS512 = "RS512" 352 PS256 = "PS256" 353 PS384 = "PS384" 354 PS512 = "PS512" 355 EdDSA = "EdDSA" 356 357 358class JwtHeaders(BaseModel): 359 class Config: 360 extra = Extra.forbid 361 362 kid: Optional[str] = Field( 363 None, 364 description="Private key ID for user account.", 365 examples=["{{ config['kid'] }}"], 366 title="Key Identifier", 367 ) 368 typ: Optional[str] = Field( 369 "JWT", 370 description="The media type of the complete JWT.", 371 examples=["JWT"], 372 title="Type", 373 ) 374 cty: Optional[str] = Field( 375 None, 376 description="Content type of JWT header.", 377 examples=["JWT"], 378 title="Content Type", 379 ) 380 381 382class JwtPayload(BaseModel): 383 class Config: 384 extra = Extra.forbid 385 386 iss: Optional[str] = Field( 387 None, 388 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 389 examples=["{{ config['iss'] }}"], 390 title="Issuer", 391 ) 392 sub: Optional[str] = Field( 393 None, 394 description="The subject of the JWT. Commonly defined by the API.", 395 title="Subject", 396 ) 397 aud: Optional[str] = Field( 398 None, 399 description="The recipient that the JWT is intended for. Commonly defined by the API.", 400 examples=["appstoreconnect-v1"], 401 title="Audience", 402 ) 403 404 405class RefreshTokenUpdater(BaseModel): 406 refresh_token_name: Optional[str] = Field( 407 "refresh_token", 408 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 409 examples=["refresh_token"], 410 title="Refresh Token Property Name", 411 ) 412 access_token_config_path: Optional[List[str]] = Field( 413 ["credentials", "access_token"], 414 description="Config path to the access token. Make sure the field actually exists in the config.", 415 examples=[["credentials", "access_token"], ["access_token"]], 416 title="Config Path To Access Token", 417 ) 418 refresh_token_config_path: Optional[List[str]] = Field( 419 ["credentials", "refresh_token"], 420 description="Config path to the access token. Make sure the field actually exists in the config.", 421 examples=[["credentials", "refresh_token"], ["refresh_token"]], 422 title="Config Path To Refresh Token", 423 ) 424 token_expiry_date_config_path: Optional[List[str]] = Field( 425 ["credentials", "token_expiry_date"], 426 description="Config path to the expiry date. Make sure actually exists in the config.", 427 examples=[["credentials", "token_expiry_date"]], 428 title="Config Path To Expiry Date", 429 ) 430 refresh_token_error_status_codes: Optional[List[int]] = Field( 431 [], 432 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 433 examples=[[400, 500]], 434 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 435 ) 436 refresh_token_error_key: Optional[str] = Field( 437 "", 438 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 439 examples=["error"], 440 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 441 ) 442 refresh_token_error_values: Optional[List[str]] = Field( 443 [], 444 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 445 examples=[["invalid_grant", "invalid_permissions"]], 446 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 447 ) 448 449 450class Rate(BaseModel): 451 class Config: 452 extra = Extra.allow 453 454 limit: Union[int, str] = Field( 455 ..., 456 description="The maximum number of calls allowed within the interval.", 457 title="Limit", 458 ) 459 interval: str = Field( 460 ..., 461 description="The time interval for the rate limit.", 462 examples=["PT1H", "P1D"], 463 title="Interval", 464 ) 465 466 467class HttpRequestRegexMatcher(BaseModel): 468 class Config: 469 extra = Extra.allow 470 471 method: Optional[str] = Field( 472 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 473 ) 474 url_base: Optional[str] = Field( 475 None, 476 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 477 title="URL Base", 478 ) 479 url_path_pattern: Optional[str] = Field( 480 None, 481 description="A regular expression pattern to match the URL path.", 482 title="URL Path Pattern", 483 ) 484 params: Optional[Dict[str, Any]] = Field( 485 None, description="The query parameters to match.", title="Parameters" 486 ) 487 headers: Optional[Dict[str, Any]] = Field( 488 None, description="The headers to match.", title="Headers" 489 ) 490 weight: Optional[Union[int, str]] = Field( 491 None, 492 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 493 title="Weight", 494 ) 495 496 497class ResponseToFileExtractor(BaseModel): 498 type: Literal["ResponseToFileExtractor"] 499 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 500 501 502class OnNoRecords(Enum): 503 skip = "skip" 504 emit_parent = "emit_parent" 505 506 507class ExponentialBackoffStrategy(BaseModel): 508 type: Literal["ExponentialBackoffStrategy"] 509 factor: Optional[Union[float, str]] = Field( 510 5, 511 description="Multiplicative constant applied on each retry.", 512 examples=[5, 5.5, "10"], 513 title="Factor", 514 ) 515 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 516 517 518class GroupByKeyMergeStrategy(BaseModel): 519 type: Literal["GroupByKeyMergeStrategy"] 520 key: Union[str, List[str]] = Field( 521 ..., 522 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 523 examples=["id", ["parent_id", "end_date"]], 524 title="Key", 525 ) 526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 527 528 529class SessionTokenRequestBearerAuthenticator(BaseModel): 530 type: Literal["Bearer"] 531 532 533class HttpMethod(Enum): 534 GET = "GET" 535 POST = "POST" 536 537 538class Action(Enum): 539 SUCCESS = "SUCCESS" 540 FAIL = "FAIL" 541 RETRY = "RETRY" 542 IGNORE = "IGNORE" 543 RESET_PAGINATION = "RESET_PAGINATION" 544 RATE_LIMITED = "RATE_LIMITED" 545 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 546 547 548class FailureType(Enum): 549 system_error = "system_error" 550 config_error = "config_error" 551 transient_error = "transient_error" 552 553 554class HttpResponseFilter(BaseModel): 555 type: Literal["HttpResponseFilter"] 556 action: Optional[Action] = Field( 557 None, 558 description="Action to execute if a response matches the filter.", 559 examples=[ 560 "SUCCESS", 561 "FAIL", 562 "RETRY", 563 "IGNORE", 564 "RESET_PAGINATION", 565 "RATE_LIMITED", 566 "REFRESH_TOKEN_THEN_RETRY", 567 ], 568 title="Action", 569 ) 570 failure_type: Optional[FailureType] = Field( 571 None, 572 description="Failure type of traced exception if a response matches the filter.", 573 examples=["system_error", "config_error", "transient_error"], 574 title="Failure Type", 575 ) 576 error_message: Optional[str] = Field( 577 None, 578 description="Error Message to display if the response matches the filter.", 579 title="Error Message", 580 ) 581 error_message_contains: Optional[str] = Field( 582 None, 583 description="Match the response if its error message contains the substring.", 584 example=["This API operation is not enabled for this site"], 585 title="Error Message Substring", 586 ) 587 http_codes: Optional[List[int]] = Field( 588 None, 589 description="Match the response if its HTTP code is included in this list.", 590 examples=[[420, 429], [500]], 591 title="HTTP Codes", 592 unique_items=True, 593 ) 594 predicate: Optional[str] = Field( 595 None, 596 description="Match the response if the predicate evaluates to true.", 597 examples=[ 598 "{{ 'Too much requests' in response }}", 599 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 600 ], 601 title="Predicate", 602 ) 603 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 604 605 606class ComplexFieldType(BaseModel): 607 field_type: str 608 items: Optional[Union[str, ComplexFieldType]] = None 609 610 611class TypesMap(BaseModel): 612 target_type: Union[str, List[str], ComplexFieldType] 613 current_type: Union[str, List[str]] 614 condition: Optional[str] = None 615 616 617class SchemaTypeIdentifier(BaseModel): 618 type: Optional[Literal["SchemaTypeIdentifier"]] = None 619 schema_pointer: Optional[List[str]] = Field( 620 [], 621 description="List of nested fields defining the schema field path to extract. Defaults to [].", 622 title="Schema Path", 623 ) 624 key_pointer: List[str] = Field( 625 ..., 626 description="List of potentially nested fields describing the full path of the field key to extract.", 627 title="Key Path", 628 ) 629 type_pointer: Optional[List[str]] = Field( 630 None, 631 description="List of potentially nested fields describing the full path of the field type to extract.", 632 title="Type Path", 633 ) 634 types_mapping: Optional[List[TypesMap]] = None 635 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 636 637 638class InlineSchemaLoader(BaseModel): 639 type: Literal["InlineSchemaLoader"] 640 schema_: Optional[Dict[str, Any]] = Field( 641 None, 642 alias="schema", 643 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 644 title="Schema", 645 ) 646 647 648class JsonFileSchemaLoader(BaseModel): 649 type: Literal["JsonFileSchemaLoader"] 650 file_path: Optional[str] = Field( 651 None, 652 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 653 example=["./schemas/users.json"], 654 title="File Path", 655 ) 656 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 657 658 659class JsonDecoder(BaseModel): 660 type: Literal["JsonDecoder"] 661 662 663class JsonlDecoder(BaseModel): 664 type: Literal["JsonlDecoder"] 665 666 667class KeysToLower(BaseModel): 668 type: Literal["KeysToLower"] 669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 670 671 672class KeysToSnakeCase(BaseModel): 673 type: Literal["KeysToSnakeCase"] 674 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 675 676 677class FlattenFields(BaseModel): 678 type: Literal["FlattenFields"] 679 flatten_lists: Optional[bool] = Field( 680 True, 681 description="Whether to flatten lists or leave it as is. Default is True.", 682 title="Flatten Lists", 683 ) 684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 685 686 687class KeyTransformation(BaseModel): 688 type: Literal["KeyTransformation"] 689 prefix: Optional[str] = Field( 690 None, 691 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 692 examples=["flattened_"], 693 title="Key Prefix", 694 ) 695 suffix: Optional[str] = Field( 696 None, 697 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 698 examples=["_flattened"], 699 title="Key Suffix", 700 ) 701 702 703class DpathFlattenFields(BaseModel): 704 type: Literal["DpathFlattenFields"] 705 field_path: List[str] = Field( 706 ..., 707 description="A path to field that needs to be flattened.", 708 examples=[["data"], ["data", "*", "field"]], 709 title="Field Path", 710 ) 711 delete_origin_value: Optional[bool] = Field( 712 None, 713 description="Whether to delete the origin value or keep it. Default is False.", 714 title="Delete Origin Value", 715 ) 716 replace_record: Optional[bool] = Field( 717 None, 718 description="Whether to replace the origin record or not. Default is False.", 719 title="Replace Origin Record", 720 ) 721 key_transformation: Optional[KeyTransformation] = Field( 722 None, 723 description="Transformation for object keys. If not provided, original key will be used.", 724 title="Key transformation", 725 ) 726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 727 728 729class KeysReplace(BaseModel): 730 type: Literal["KeysReplace"] 731 old: str = Field( 732 ..., 733 description="Old value to replace.", 734 examples=[ 735 " ", 736 "{{ record.id }}", 737 "{{ config['id'] }}", 738 "{{ stream_slice['id'] }}", 739 ], 740 title="Old value", 741 ) 742 new: str = Field( 743 ..., 744 description="New value to set.", 745 examples=[ 746 "_", 747 "{{ record.id }}", 748 "{{ config['id'] }}", 749 "{{ stream_slice['id'] }}", 750 ], 751 title="New value", 752 ) 753 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 754 755 756class IterableDecoder(BaseModel): 757 type: Literal["IterableDecoder"] 758 759 760class XmlDecoder(BaseModel): 761 type: Literal["XmlDecoder"] 762 763 764class CustomDecoder(BaseModel): 765 class Config: 766 extra = Extra.allow 767 768 type: Literal["CustomDecoder"] 769 class_name: str = Field( 770 ..., 771 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 772 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 773 title="Class Name", 774 ) 775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 776 777 778class MinMaxDatetime(BaseModel): 779 type: Literal["MinMaxDatetime"] 780 datetime: str = Field( 781 ..., 782 description="Datetime value.", 783 examples=[ 784 "2021-01-01", 785 "2021-01-01T00:00:00Z", 786 "{{ config['start_time'] }}", 787 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 788 ], 789 title="Datetime", 790 ) 791 datetime_format: Optional[str] = Field( 792 "", 793 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 794 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 795 title="Datetime Format", 796 ) 797 max_datetime: Optional[str] = Field( 798 None, 799 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 800 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 801 title="Max Datetime", 802 ) 803 min_datetime: Optional[str] = Field( 804 None, 805 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 806 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 807 title="Min Datetime", 808 ) 809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 810 811 812class NoAuth(BaseModel): 813 type: Literal["NoAuth"] 814 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 815 816 817class NoPagination(BaseModel): 818 type: Literal["NoPagination"] 819 820 821class State(BaseModel): 822 class Config: 823 extra = Extra.allow 824 825 min: int 826 max: int 827 828 829class OAuthScope(BaseModel): 830 class Config: 831 extra = Extra.allow 832 833 scope: str = Field( 834 ..., 835 description="The OAuth scope string to request from the provider.", 836 ) 837 838 839class OauthConnectorInputSpecification(BaseModel): 840 class Config: 841 extra = Extra.allow 842 843 consent_url: str = Field( 844 ..., 845 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 846 examples=[ 847 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 848 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 849 ], 850 title="Consent URL", 851 ) 852 scope: Optional[str] = Field( 853 None, 854 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 855 examples=["user:read user:read_orders workspaces:read"], 856 title="Scopes", 857 ) 858 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 859 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 860 # The CDK schema defines the manifest contract; the platform reads these fields 861 # during the OAuth consent flow to build the authorization URL. 862 scopes: Optional[List[OAuthScope]] = Field( 863 None, 864 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 865 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 866 title="Scopes", 867 ) 868 optional_scopes: Optional[List[OAuthScope]] = Field( 869 None, 870 description="Optional OAuth scope objects that may or may not be granted.", 871 examples=[[{"scope": "admin:read"}]], 872 title="Optional Scopes", 873 ) 874 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 875 ScopesJoinStrategy.space, 876 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 877 title="Scopes Join Strategy", 878 ) 879 access_token_url: str = Field( 880 ..., 881 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 882 examples=[ 883 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 884 ], 885 title="Access Token URL", 886 ) 887 access_token_headers: Optional[Dict[str, Any]] = Field( 888 None, 889 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 890 examples=[ 891 { 892 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 893 } 894 ], 895 title="Access Token Headers", 896 ) 897 access_token_params: Optional[Dict[str, Any]] = Field( 898 None, 899 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 900 examples=[ 901 { 902 "{{ auth_code_key }}": "{{ auth_code_value }}", 903 "{{ client_id_key }}": "{{ client_id_value }}", 904 "{{ client_secret_key }}": "{{ client_secret_value }}", 905 } 906 ], 907 title="Access Token Query Params (Json Encoded)", 908 ) 909 extract_output: Optional[List[str]] = Field( 910 None, 911 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 912 examples=[["access_token", "refresh_token", "other_field"]], 913 title="Extract Output", 914 ) 915 state: Optional[State] = Field( 916 None, 917 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 918 examples=[{"min": 7, "max": 128}], 919 title="Configurable State Query Param", 920 ) 921 client_id_key: Optional[str] = Field( 922 None, 923 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 924 examples=["my_custom_client_id_key_name"], 925 title="Client ID Key Override", 926 ) 927 client_secret_key: Optional[str] = Field( 928 None, 929 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 930 examples=["my_custom_client_secret_key_name"], 931 title="Client Secret Key Override", 932 ) 933 scope_key: Optional[str] = Field( 934 None, 935 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 936 examples=["my_custom_scope_key_key_name"], 937 title="Scopes Key Override", 938 ) 939 state_key: Optional[str] = Field( 940 None, 941 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 942 examples=["my_custom_state_key_key_name"], 943 title="State Key Override", 944 ) 945 auth_code_key: Optional[str] = Field( 946 None, 947 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 948 examples=["my_custom_auth_code_key_name"], 949 title="Auth Code Key Override", 950 ) 951 redirect_uri_key: Optional[str] = Field( 952 None, 953 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 954 examples=["my_custom_redirect_uri_key_name"], 955 title="Redirect URI Key Override", 956 ) 957 958 959class OAuthConfigSpecification(BaseModel): 960 class Config: 961 extra = Extra.allow 962 963 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 964 None, 965 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 966 examples=[ 967 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 968 { 969 "app_id": { 970 "type": "string", 971 "path_in_connector_config": ["info", "app_id"], 972 } 973 }, 974 ], 975 title="OAuth user input", 976 ) 977 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 978 None, 979 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 980 title="DeclarativeOAuth Connector Specification", 981 ) 982 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 983 None, 984 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 985 examples=[ 986 { 987 "refresh_token": { 988 "type": "string,", 989 "path_in_connector_config": ["credentials", "refresh_token"], 990 } 991 } 992 ], 993 title="OAuth output specification", 994 ) 995 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 996 None, 997 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 998 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 999 title="OAuth input specification", 1000 ) 1001 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1002 None, 1003 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1004 examples=[ 1005 { 1006 "client_id": { 1007 "type": "string,", 1008 "path_in_connector_config": ["credentials", "client_id"], 1009 }, 1010 "client_secret": { 1011 "type": "string,", 1012 "path_in_connector_config": ["credentials", "client_secret"], 1013 }, 1014 } 1015 ], 1016 title="OAuth server output specification", 1017 ) 1018 1019 1020class OffsetIncrement(BaseModel): 1021 type: Literal["OffsetIncrement"] 1022 page_size: Optional[Union[int, str]] = Field( 1023 None, 1024 description="The number of records to include in each pages.", 1025 examples=[100, "{{ config['page_size'] }}"], 1026 title="Limit", 1027 ) 1028 inject_on_first_request: Optional[bool] = Field( 1029 False, 1030 description="Using the `offset` with value `0` during the first request", 1031 title="Inject Offset on First Request", 1032 ) 1033 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1034 1035 1036class PageIncrement(BaseModel): 1037 type: Literal["PageIncrement"] 1038 page_size: Optional[Union[int, str]] = Field( 1039 None, 1040 description="The number of records to include in each pages.", 1041 examples=[100, "100", "{{ config['page_size'] }}"], 1042 title="Page Size", 1043 ) 1044 start_from_page: Optional[int] = Field( 1045 0, 1046 description="Index of the first page to request.", 1047 examples=[0, 1], 1048 title="Start From Page", 1049 ) 1050 inject_on_first_request: Optional[bool] = Field( 1051 False, 1052 description="Using the `page number` with value defined by `start_from_page` during the first request", 1053 title="Inject Page Number on First Request", 1054 ) 1055 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1056 1057 1058class PrimaryKey(BaseModel): 1059 __root__: Union[str, List[str], List[List[str]]] = Field( 1060 ..., 1061 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1062 examples=["id", ["code", "type"]], 1063 title="Primary Key", 1064 ) 1065 1066 1067class PropertyLimitType(Enum): 1068 characters = "characters" 1069 property_count = "property_count" 1070 1071 1072class PropertyChunking(BaseModel): 1073 type: Literal["PropertyChunking"] 1074 property_limit_type: PropertyLimitType = Field( 1075 ..., 1076 description="The type used to determine the maximum number of properties per chunk", 1077 title="Property Limit Type", 1078 ) 1079 property_limit: Optional[int] = Field( 1080 None, 1081 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1082 title="Property Limit", 1083 ) 1084 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1085 None, 1086 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1087 title="Record Merge Strategy", 1088 ) 1089 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1090 1091 1092class RecordFilter(BaseModel): 1093 type: Literal["RecordFilter"] 1094 condition: Optional[str] = Field( 1095 "", 1096 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1097 examples=[ 1098 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1099 "{{ record.status in ['active', 'expired'] }}", 1100 ], 1101 title="Condition", 1102 ) 1103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1104 1105 1106class SchemaNormalization(Enum): 1107 Default = "Default" 1108 None_ = "None" 1109 1110 1111class RemoveFields(BaseModel): 1112 type: Literal["RemoveFields"] 1113 condition: Optional[str] = Field( 1114 "", 1115 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1116 examples=[ 1117 "{{ property|string == '' }}", 1118 "{{ property is integer }}", 1119 "{{ property|length > 5 }}", 1120 "{{ property == 'some_string_to_match' }}", 1121 ], 1122 ) 1123 field_pointers: List[List[str]] = Field( 1124 ..., 1125 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1126 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1127 title="Field Paths", 1128 ) 1129 1130 1131class RequestPath(BaseModel): 1132 type: Literal["RequestPath"] 1133 1134 1135class InjectInto(Enum): 1136 request_parameter = "request_parameter" 1137 header = "header" 1138 body_data = "body_data" 1139 body_json = "body_json" 1140 1141 1142class RequestOption(BaseModel): 1143 type: Literal["RequestOption"] 1144 inject_into: InjectInto = Field( 1145 ..., 1146 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1147 examples=["request_parameter", "header", "body_data", "body_json"], 1148 title="Inject Into", 1149 ) 1150 field_name: Optional[str] = Field( 1151 None, 1152 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1153 examples=["segment_id"], 1154 title="Field Name", 1155 ) 1156 field_path: Optional[List[str]] = Field( 1157 None, 1158 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1159 examples=[["data", "viewer", "id"]], 1160 title="Field Path", 1161 ) 1162 1163 1164class Schemas(BaseModel): 1165 pass 1166 1167 class Config: 1168 extra = Extra.allow 1169 1170 1171class LegacySessionTokenAuthenticator(BaseModel): 1172 type: Literal["LegacySessionTokenAuthenticator"] 1173 header: str = Field( 1174 ..., 1175 description="The name of the session token header that will be injected in the request", 1176 examples=["X-Session"], 1177 title="Session Request Header", 1178 ) 1179 login_url: str = Field( 1180 ..., 1181 description="Path of the login URL (do not include the base URL)", 1182 examples=["session"], 1183 title="Login Path", 1184 ) 1185 session_token: Optional[str] = Field( 1186 None, 1187 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1188 example=["{{ config['session_token'] }}"], 1189 title="Session Token", 1190 ) 1191 session_token_response_key: str = Field( 1192 ..., 1193 description="Name of the key of the session token to be extracted from the response", 1194 examples=["id"], 1195 title="Response Token Response Key", 1196 ) 1197 username: Optional[str] = Field( 1198 None, 1199 description="Username used to authenticate and obtain a session token", 1200 examples=[" {{ config['username'] }}"], 1201 title="Username", 1202 ) 1203 password: Optional[str] = Field( 1204 "", 1205 description="Password used to authenticate and obtain a session token", 1206 examples=["{{ config['password'] }}", ""], 1207 title="Password", 1208 ) 1209 validate_session_url: str = Field( 1210 ..., 1211 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1212 examples=["user/current"], 1213 title="Validate Session Path", 1214 ) 1215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1216 1217 1218class Action1(Enum): 1219 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1220 RESET = "RESET" 1221 1222 1223class PaginationResetLimits(BaseModel): 1224 type: Literal["PaginationResetLimits"] 1225 number_of_records: Optional[int] = None 1226 1227 1228class CsvDecoder(BaseModel): 1229 type: Literal["CsvDecoder"] 1230 encoding: Optional[str] = "utf-8" 1231 delimiter: Optional[str] = "," 1232 set_values_to_none: Optional[List[str]] = None 1233 1234 1235class AsyncJobStatusMap(BaseModel): 1236 type: Optional[Literal["AsyncJobStatusMap"]] = None 1237 running: List[str] 1238 completed: List[str] 1239 failed: List[str] 1240 timeout: List[str] 1241 1242 1243class ValueType(Enum): 1244 string = "string" 1245 number = "number" 1246 integer = "integer" 1247 boolean = "boolean" 1248 1249 1250class WaitTimeFromHeader(BaseModel): 1251 type: Literal["WaitTimeFromHeader"] 1252 header: str = Field( 1253 ..., 1254 description="The name of the response header defining how long to wait before retrying.", 1255 examples=["Retry-After"], 1256 title="Response Header Name", 1257 ) 1258 regex: Optional[str] = Field( 1259 None, 1260 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1261 examples=["([-+]?\\d+)"], 1262 title="Extraction Regex", 1263 ) 1264 max_waiting_time_in_seconds: Optional[float] = Field( 1265 None, 1266 description="Given the value extracted from the header is greater than this value, stop the stream.", 1267 examples=[3600], 1268 title="Max Waiting Time in Seconds", 1269 ) 1270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1271 1272 1273class WaitUntilTimeFromHeader(BaseModel): 1274 type: Literal["WaitUntilTimeFromHeader"] 1275 header: str = Field( 1276 ..., 1277 description="The name of the response header defining how long to wait before retrying.", 1278 examples=["wait_time"], 1279 title="Response Header", 1280 ) 1281 min_wait: Optional[Union[float, str]] = Field( 1282 None, 1283 description="Minimum time to wait before retrying.", 1284 examples=[10, "60"], 1285 title="Minimum Wait Time", 1286 ) 1287 regex: Optional[str] = Field( 1288 None, 1289 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1290 examples=["([-+]?\\d+)"], 1291 title="Extraction Regex", 1292 ) 1293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1294 1295 1296class ComponentMappingDefinition(BaseModel): 1297 type: Literal["ComponentMappingDefinition"] 1298 field_path: List[str] = Field( 1299 ..., 1300 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1301 examples=[ 1302 ["name"], 1303 ["retriever", "requester", "url"], 1304 ["retriever", "requester", "{{ components_values.field }}"], 1305 ["*", "**", "name"], 1306 ], 1307 title="Field Path", 1308 ) 1309 value: str = Field( 1310 ..., 1311 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1312 examples=[ 1313 "{{ components_values['updates'] }}", 1314 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1315 "{{ config['segment_id'] }}", 1316 "{{ stream_slice['parent_id'] }}", 1317 "{{ stream_slice['extra_fields']['name'] }}", 1318 ], 1319 title="Value", 1320 ) 1321 value_type: Optional[ValueType] = Field( 1322 None, 1323 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1324 title="Value Type", 1325 ) 1326 create_or_update: Optional[bool] = Field( 1327 False, 1328 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1329 title="Create or Update", 1330 ) 1331 condition: Optional[str] = Field( 1332 None, 1333 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1334 examples=[ 1335 "{{ components_values.get('cursor_field', None) }}", 1336 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1337 ], 1338 title="Condition", 1339 ) 1340 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1341 1342 1343class StreamConfig(BaseModel): 1344 type: Literal["StreamConfig"] 1345 configs_pointer: List[str] = Field( 1346 ..., 1347 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1348 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1349 title="Configs Pointer", 1350 ) 1351 default_values: Optional[List[Dict[str, Any]]] = Field( 1352 None, 1353 description="A list of default values, each matching the structure expected from the parsed component value.", 1354 title="Default Values", 1355 ) 1356 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1357 1358 1359class ConfigComponentsResolver(BaseModel): 1360 type: Literal["ConfigComponentsResolver"] 1361 stream_config: Union[List[StreamConfig], StreamConfig] 1362 components_mapping: List[ComponentMappingDefinition] 1363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1364 1365 1366class StreamParametersDefinition(BaseModel): 1367 type: Literal["StreamParametersDefinition"] 1368 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1369 ..., 1370 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1371 examples=[ 1372 [ 1373 { 1374 "name": "test stream", 1375 "$parameters": {"entity": "test entity"}, 1376 "primary_key": "test key", 1377 } 1378 ] 1379 ], 1380 title="Stream Parameters", 1381 ) 1382 1383 1384class ParametrizedComponentsResolver(BaseModel): 1385 type: Literal["ParametrizedComponentsResolver"] 1386 stream_parameters: StreamParametersDefinition 1387 components_mapping: List[ComponentMappingDefinition] 1388 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1389 1390 1391class RequestBodyPlainText(BaseModel): 1392 type: Literal["RequestBodyPlainText"] 1393 value: str 1394 1395 1396class RequestBodyUrlEncodedForm(BaseModel): 1397 type: Literal["RequestBodyUrlEncodedForm"] 1398 value: Dict[str, str] 1399 1400 1401class RequestBodyJsonObject(BaseModel): 1402 type: Literal["RequestBodyJsonObject"] 1403 value: Dict[str, Any] 1404 1405 1406class RequestBodyGraphQlQuery(BaseModel): 1407 class Config: 1408 extra = Extra.allow 1409 1410 query: str = Field(..., description="The GraphQL query to be executed") 1411 1412 1413class ValidateAdheresToSchema(BaseModel): 1414 type: Literal["ValidateAdheresToSchema"] 1415 base_schema: Union[str, Dict[str, Any]] = Field( 1416 ..., 1417 description="The base JSON schema against which the user-provided schema will be validated.", 1418 examples=[ 1419 "{{ config['report_validation_schema'] }}", 1420 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1421 { 1422 "$schema": "http://json-schema.org/draft-07/schema#", 1423 "title": "Person", 1424 "type": "object", 1425 "properties": { 1426 "name": {"type": "string", "description": "The person's name"}, 1427 "age": { 1428 "type": "integer", 1429 "minimum": 0, 1430 "description": "The person's age", 1431 }, 1432 }, 1433 "required": ["name", "age"], 1434 }, 1435 ], 1436 title="Base JSON Schema", 1437 ) 1438 1439 1440class CustomValidationStrategy(BaseModel): 1441 class Config: 1442 extra = Extra.allow 1443 1444 type: Literal["CustomValidationStrategy"] 1445 class_name: str = Field( 1446 ..., 1447 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1448 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1449 title="Class Name", 1450 ) 1451 1452 1453class ConfigRemapField(BaseModel): 1454 type: Literal["ConfigRemapField"] 1455 map: Union[Dict[str, Any], str] = Field( 1456 ..., 1457 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1458 examples=[ 1459 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1460 "{{ config['status_mapping'] }}", 1461 ], 1462 title="Value Mapping", 1463 ) 1464 field_path: List[str] = Field( 1465 ..., 1466 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1467 examples=[ 1468 ["status"], 1469 ["data", "status"], 1470 ["data", "{{ config.name }}", "status"], 1471 ["data", "*", "status"], 1472 ], 1473 title="Field Path", 1474 ) 1475 1476 1477class ConfigRemoveFields(BaseModel): 1478 type: Literal["ConfigRemoveFields"] 1479 field_pointers: List[List[str]] = Field( 1480 ..., 1481 description="A list of field pointers to be removed from the config.", 1482 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1483 title="Field Pointers", 1484 ) 1485 condition: Optional[str] = Field( 1486 "", 1487 description="Fields will be removed if expression is evaluated to True.", 1488 examples=[ 1489 "{{ config['environemnt'] == 'sandbox' }}", 1490 "{{ property is integer }}", 1491 "{{ property|length > 5 }}", 1492 "{{ property == 'some_string_to_match' }}", 1493 ], 1494 ) 1495 1496 1497class CustomConfigTransformation(BaseModel): 1498 type: Literal["CustomConfigTransformation"] 1499 class_name: str = Field( 1500 ..., 1501 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1502 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1503 ) 1504 parameters: Optional[Dict[str, Any]] = Field( 1505 None, 1506 alias="$parameters", 1507 description="Additional parameters to be passed to the custom config transformation.", 1508 ) 1509 1510 1511class AddedFieldDefinition(BaseModel): 1512 type: Literal["AddedFieldDefinition"] 1513 path: List[str] = Field( 1514 ..., 1515 description="List of strings defining the path where to add the value on the record.", 1516 examples=[["segment_id"], ["metadata", "segment_id"]], 1517 title="Path", 1518 ) 1519 value: str = Field( 1520 ..., 1521 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1522 examples=[ 1523 "{{ record['updates'] }}", 1524 "{{ record['MetaData']['LastUpdatedTime'] }}", 1525 "{{ stream_partition['segment_id'] }}", 1526 ], 1527 title="Value", 1528 ) 1529 value_type: Optional[ValueType] = Field( 1530 None, 1531 description="Type of the value. If not specified, the type will be inferred from the value.", 1532 title="Value Type", 1533 ) 1534 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1535 1536 1537class AddFields(BaseModel): 1538 type: Literal["AddFields"] 1539 fields: List[AddedFieldDefinition] = Field( 1540 ..., 1541 description="List of transformations (path and corresponding value) that will be added to the record.", 1542 title="Fields", 1543 ) 1544 condition: Optional[str] = Field( 1545 "", 1546 description="Fields will be added if expression is evaluated to True.", 1547 examples=[ 1548 "{{ property|string == '' }}", 1549 "{{ property is integer }}", 1550 "{{ property|length > 5 }}", 1551 "{{ property == 'some_string_to_match' }}", 1552 ], 1553 ) 1554 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1555 1556 1557class ApiKeyAuthenticator(BaseModel): 1558 type: Literal["ApiKeyAuthenticator"] 1559 api_token: Optional[str] = Field( 1560 None, 1561 description="The API key to inject in the request. Fill it in the user inputs.", 1562 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1563 title="API Key", 1564 ) 1565 header: Optional[str] = Field( 1566 None, 1567 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1568 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1569 title="Header Name", 1570 ) 1571 inject_into: Optional[RequestOption] = Field( 1572 None, 1573 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1574 examples=[ 1575 {"inject_into": "header", "field_name": "Authorization"}, 1576 {"inject_into": "request_parameter", "field_name": "authKey"}, 1577 ], 1578 title="Inject API Key Into Outgoing HTTP Request", 1579 ) 1580 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1581 1582 1583class AuthFlow(BaseModel): 1584 auth_flow_type: Optional[AuthFlowType] = Field( 1585 None, description="The type of auth to use", title="Auth flow type" 1586 ) 1587 predicate_key: Optional[List[str]] = Field( 1588 None, 1589 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1590 examples=[["credentials", "auth_type"]], 1591 title="Predicate key", 1592 ) 1593 predicate_value: Optional[str] = Field( 1594 None, 1595 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1596 examples=["Oauth"], 1597 title="Predicate value", 1598 ) 1599 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1600 1601 1602class CheckStream(BaseModel): 1603 type: Literal["CheckStream"] 1604 stream_names: Optional[List[str]] = Field( 1605 None, 1606 description="Names of the streams to try reading from when running a check operation.", 1607 examples=[["users"], ["users", "contacts"]], 1608 title="Stream Names", 1609 ) 1610 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1611 1612 1613class IncrementingCountCursor(BaseModel): 1614 type: Literal["IncrementingCountCursor"] 1615 cursor_field: str = Field( 1616 ..., 1617 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1618 examples=["created_at", "{{ config['record_cursor'] }}"], 1619 title="Cursor Field", 1620 ) 1621 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1622 None, 1623 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1624 title="Allow Catalog Defined Cursor Field", 1625 ) 1626 start_value: Optional[Union[str, int]] = Field( 1627 None, 1628 description="The value that determines the earliest record that should be synced.", 1629 examples=[0, "{{ config['start_value'] }}"], 1630 title="Start Value", 1631 ) 1632 start_value_option: Optional[RequestOption] = Field( 1633 None, 1634 description="Optionally configures how the start value will be sent in requests to the source API.", 1635 title="Inject Start Value Into Outgoing HTTP Request", 1636 ) 1637 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1638 1639 1640class DatetimeBasedCursor(BaseModel): 1641 type: Literal["DatetimeBasedCursor"] 1642 clamping: Optional[Clamping] = Field( 1643 None, 1644 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1645 title="Date Range Clamping", 1646 ) 1647 cursor_field: str = Field( 1648 ..., 1649 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1650 examples=["created_at", "{{ config['record_cursor'] }}"], 1651 title="Cursor Field", 1652 ) 1653 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1654 None, 1655 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1656 title="Allow Catalog Defined Cursor Field", 1657 ) 1658 cursor_datetime_formats: Optional[List[str]] = Field( 1659 None, 1660 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1661 examples=[ 1662 "%Y-%m-%d", 1663 "%Y-%m-%d %H:%M:%S", 1664 "%Y-%m-%dT%H:%M:%S", 1665 "%Y-%m-%dT%H:%M:%SZ", 1666 "%Y-%m-%dT%H:%M:%S%z", 1667 "%Y-%m-%dT%H:%M:%S.%fZ", 1668 "%Y-%m-%dT%H:%M:%S.%f%z", 1669 "%Y-%m-%d %H:%M:%S.%f+00:00", 1670 "%s", 1671 "%ms", 1672 ], 1673 title="Cursor Datetime Formats", 1674 ) 1675 start_datetime: Union[MinMaxDatetime, str] = Field( 1676 ..., 1677 description="The datetime that determines the earliest record that should be synced.", 1678 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1679 title="Start Datetime", 1680 ) 1681 start_time_option: Optional[RequestOption] = Field( 1682 None, 1683 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1684 title="Inject Start Time Into Outgoing HTTP Request", 1685 ) 1686 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1687 None, 1688 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1689 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1690 title="End Datetime", 1691 ) 1692 end_time_option: Optional[RequestOption] = Field( 1693 None, 1694 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1695 title="Inject End Time Into Outgoing HTTP Request", 1696 ) 1697 datetime_format: str = Field( 1698 ..., 1699 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1700 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1701 title="Outgoing Datetime Format", 1702 ) 1703 cursor_granularity: Optional[str] = Field( 1704 None, 1705 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1706 examples=["PT1S"], 1707 title="Cursor Granularity", 1708 ) 1709 is_data_feed: Optional[bool] = Field( 1710 None, 1711 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1712 title="Data Feed API", 1713 ) 1714 is_client_side_incremental: Optional[bool] = Field( 1715 None, 1716 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1717 title="Client-side Incremental Filtering", 1718 ) 1719 is_compare_strictly: Optional[bool] = Field( 1720 False, 1721 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1722 title="Strict Start-End Time Comparison", 1723 ) 1724 global_substream_cursor: Optional[bool] = Field( 1725 False, 1726 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1727 title="Global Substream Cursor", 1728 ) 1729 lookback_window: Optional[str] = Field( 1730 None, 1731 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1732 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1733 title="Lookback Window", 1734 ) 1735 partition_field_end: Optional[str] = Field( 1736 None, 1737 description="Name of the partition start time field.", 1738 examples=["ending_time"], 1739 title="Partition Field End", 1740 ) 1741 partition_field_start: Optional[str] = Field( 1742 None, 1743 description="Name of the partition end time field.", 1744 examples=["starting_time"], 1745 title="Partition Field Start", 1746 ) 1747 step: Optional[str] = Field( 1748 None, 1749 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1750 examples=["P1W", "{{ config['step_increment'] }}"], 1751 title="Step", 1752 ) 1753 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1754 1755 1756class JwtAuthenticator(BaseModel): 1757 type: Literal["JwtAuthenticator"] 1758 secret_key: str = Field( 1759 ..., 1760 description="Secret used to sign the JSON web token.", 1761 examples=["{{ config['secret_key'] }}"], 1762 title="Secret Key", 1763 ) 1764 base64_encode_secret_key: Optional[bool] = Field( 1765 False, 1766 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1767 title="Base64-encode Secret Key", 1768 ) 1769 algorithm: Algorithm = Field( 1770 ..., 1771 description="Algorithm used to sign the JSON web token.", 1772 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1773 title="Algorithm", 1774 ) 1775 token_duration: Optional[int] = Field( 1776 1200, 1777 description="The amount of time in seconds a JWT token can be valid after being issued.", 1778 examples=[1200, 3600], 1779 title="Token Duration", 1780 ) 1781 header_prefix: Optional[str] = Field( 1782 None, 1783 description="The prefix to be used within the Authentication header.", 1784 examples=["Bearer", "Basic"], 1785 title="Header Prefix", 1786 ) 1787 jwt_headers: Optional[JwtHeaders] = Field( 1788 None, 1789 description="JWT headers used when signing JSON web token.", 1790 title="JWT Headers", 1791 ) 1792 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1793 None, 1794 description="Additional headers to be included with the JWT headers object.", 1795 title="Additional JWT Headers", 1796 ) 1797 jwt_payload: Optional[JwtPayload] = Field( 1798 None, 1799 description="JWT Payload used when signing JSON web token.", 1800 title="JWT Payload", 1801 ) 1802 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1803 None, 1804 description="Additional properties to be added to the JWT payload.", 1805 title="Additional JWT Payload Properties", 1806 ) 1807 passphrase: Optional[str] = Field( 1808 None, 1809 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1810 examples=["{{ config['passphrase'] }}"], 1811 title="Passphrase", 1812 ) 1813 request_option: Optional[RequestOption] = Field( 1814 None, 1815 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1816 title="Request Option", 1817 ) 1818 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1819 1820 1821class OAuthAuthenticator(BaseModel): 1822 type: Literal["OAuthAuthenticator"] 1823 client_id_name: Optional[str] = Field( 1824 "client_id", 1825 description="The name of the property to use to refresh the `access_token`.", 1826 examples=["custom_app_id"], 1827 title="Client ID Property Name", 1828 ) 1829 client_id: Optional[str] = Field( 1830 None, 1831 description="The OAuth client ID. Fill it in the user inputs.", 1832 examples=[ 1833 "{{ config['client_id'] }}", 1834 "{{ config['credentials']['client_id }}", 1835 ], 1836 title="Client ID", 1837 ) 1838 client_secret_name: Optional[str] = Field( 1839 "client_secret", 1840 description="The name of the property to use to refresh the `access_token`.", 1841 examples=["custom_app_secret"], 1842 title="Client Secret Property Name", 1843 ) 1844 client_secret: Optional[str] = Field( 1845 None, 1846 description="The OAuth client secret. Fill it in the user inputs.", 1847 examples=[ 1848 "{{ config['client_secret'] }}", 1849 "{{ config['credentials']['client_secret }}", 1850 ], 1851 title="Client Secret", 1852 ) 1853 refresh_token_name: Optional[str] = Field( 1854 "refresh_token", 1855 description="The name of the property to use to refresh the `access_token`.", 1856 examples=["custom_app_refresh_value"], 1857 title="Refresh Token Property Name", 1858 ) 1859 refresh_token: Optional[str] = Field( 1860 None, 1861 description="Credential artifact used to get a new access token.", 1862 examples=[ 1863 "{{ config['refresh_token'] }}", 1864 "{{ config['credentials]['refresh_token'] }}", 1865 ], 1866 title="Refresh Token", 1867 ) 1868 token_refresh_endpoint: Optional[str] = Field( 1869 None, 1870 description="The full URL to call to obtain a new access token.", 1871 examples=["https://connect.squareup.com/oauth2/token"], 1872 title="Token Refresh Endpoint", 1873 ) 1874 access_token_name: Optional[str] = Field( 1875 "access_token", 1876 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1877 examples=["access_token"], 1878 title="Access Token Property Name", 1879 ) 1880 access_token_value: Optional[str] = Field( 1881 None, 1882 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1883 examples=["secret_access_token_value"], 1884 title="Access Token Value", 1885 ) 1886 expires_in_name: Optional[str] = Field( 1887 "expires_in", 1888 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1889 examples=["expires_in"], 1890 title="Token Expiry Property Name", 1891 ) 1892 grant_type_name: Optional[str] = Field( 1893 "grant_type", 1894 description="The name of the property to use to refresh the `access_token`.", 1895 examples=["custom_grant_type"], 1896 title="Grant Type Property Name", 1897 ) 1898 grant_type: Optional[str] = Field( 1899 "refresh_token", 1900 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1901 examples=["refresh_token", "client_credentials"], 1902 title="Grant Type", 1903 ) 1904 refresh_request_body: Optional[Dict[str, Any]] = Field( 1905 None, 1906 description="Body of the request sent to get a new access token.", 1907 examples=[ 1908 { 1909 "applicationId": "{{ config['application_id'] }}", 1910 "applicationSecret": "{{ config['application_secret'] }}", 1911 "token": "{{ config['token'] }}", 1912 } 1913 ], 1914 title="Refresh Request Body", 1915 ) 1916 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1917 None, 1918 description="Headers of the request sent to get a new access token.", 1919 examples=[ 1920 { 1921 "Authorization": "<AUTH_TOKEN>", 1922 "Content-Type": "application/x-www-form-urlencoded", 1923 } 1924 ], 1925 title="Refresh Request Headers", 1926 ) 1927 scopes: Optional[List[str]] = Field( 1928 None, 1929 description="List of scopes that should be granted to the access token.", 1930 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1931 title="Scopes", 1932 ) 1933 token_expiry_date: Optional[str] = Field( 1934 None, 1935 description="The access token expiry date.", 1936 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1937 title="Token Expiry Date", 1938 ) 1939 token_expiry_date_format: Optional[str] = Field( 1940 None, 1941 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1942 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1943 title="Token Expiry Date Format", 1944 ) 1945 refresh_token_error_status_codes: Optional[List[int]] = Field( 1946 None, 1947 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1948 examples=[[400, 500]], 1949 title="Refresh Token Error Status Codes", 1950 ) 1951 refresh_token_error_key: Optional[str] = Field( 1952 None, 1953 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1954 examples=["error"], 1955 title="Refresh Token Error Key", 1956 ) 1957 refresh_token_error_values: Optional[List[str]] = Field( 1958 None, 1959 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1960 examples=[["invalid_grant", "invalid_permissions"]], 1961 title="Refresh Token Error Values", 1962 ) 1963 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1964 None, 1965 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1966 title="Refresh Token Updater", 1967 ) 1968 profile_assertion: Optional[JwtAuthenticator] = Field( 1969 None, 1970 description="The authenticator being used to authenticate the client authenticator.", 1971 title="Profile Assertion", 1972 ) 1973 use_profile_assertion: Optional[bool] = Field( 1974 False, 1975 description="Enable using profile assertion as a flow for OAuth authorization.", 1976 title="Use Profile Assertion", 1977 ) 1978 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1979 1980 1981class FixedWindowCallRatePolicy(BaseModel): 1982 class Config: 1983 extra = Extra.allow 1984 1985 type: Literal["FixedWindowCallRatePolicy"] 1986 period: str = Field( 1987 ..., description="The time interval for the rate limit window.", title="Period" 1988 ) 1989 call_limit: int = Field( 1990 ..., 1991 description="The maximum number of calls allowed within the period.", 1992 title="Call Limit", 1993 ) 1994 matchers: List[HttpRequestRegexMatcher] = Field( 1995 ..., 1996 description="List of matchers that define which requests this policy applies to.", 1997 title="Matchers", 1998 ) 1999 2000 2001class MovingWindowCallRatePolicy(BaseModel): 2002 class Config: 2003 extra = Extra.allow 2004 2005 type: Literal["MovingWindowCallRatePolicy"] 2006 rates: List[Rate] = Field( 2007 ..., 2008 description="List of rates that define the call limits for different time intervals.", 2009 title="Rates", 2010 ) 2011 matchers: List[HttpRequestRegexMatcher] = Field( 2012 ..., 2013 description="List of matchers that define which requests this policy applies to.", 2014 title="Matchers", 2015 ) 2016 2017 2018class UnlimitedCallRatePolicy(BaseModel): 2019 class Config: 2020 extra = Extra.allow 2021 2022 type: Literal["UnlimitedCallRatePolicy"] 2023 matchers: List[HttpRequestRegexMatcher] = Field( 2024 ..., 2025 description="List of matchers that define which requests this policy applies to.", 2026 title="Matchers", 2027 ) 2028 2029 2030class DefaultErrorHandler(BaseModel): 2031 type: Literal["DefaultErrorHandler"] 2032 backoff_strategies: Optional[ 2033 List[ 2034 Union[ 2035 ConstantBackoffStrategy, 2036 ExponentialBackoffStrategy, 2037 WaitTimeFromHeader, 2038 WaitUntilTimeFromHeader, 2039 CustomBackoffStrategy, 2040 ] 2041 ] 2042 ] = Field( 2043 None, 2044 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2045 title="Backoff Strategies", 2046 ) 2047 max_retries: Optional[int] = Field( 2048 5, 2049 description="The maximum number of time to retry a retryable request before giving up and failing.", 2050 examples=[5, 0, 10], 2051 title="Max Retry Count", 2052 ) 2053 response_filters: Optional[List[HttpResponseFilter]] = Field( 2054 None, 2055 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2056 title="Response Filters", 2057 ) 2058 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2059 2060 2061class DefaultPaginator(BaseModel): 2062 type: Literal["DefaultPaginator"] 2063 pagination_strategy: Union[ 2064 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2065 ] = Field( 2066 ..., 2067 description="Strategy defining how records are paginated.", 2068 title="Pagination Strategy", 2069 ) 2070 page_size_option: Optional[RequestOption] = Field( 2071 None, title="Inject Page Size Into Outgoing HTTP Request" 2072 ) 2073 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2074 None, 2075 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2076 title="Inject Page Token Into Outgoing HTTP Request", 2077 ) 2078 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2079 2080 2081class RecordExpander(BaseModel): 2082 type: Literal["RecordExpander"] 2083 expand_records_from_field: List[str] = Field( 2084 ..., 2085 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2086 examples=[ 2087 ["lines", "data"], 2088 ["items"], 2089 ["nested", "array"], 2090 ["sections", "*", "items"], 2091 ], 2092 title="Expand Records From Field", 2093 ) 2094 remain_original_record: Optional[bool] = Field( 2095 False, 2096 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2097 title="Remain Original Record", 2098 ) 2099 on_no_records: Optional[OnNoRecords] = Field( 2100 OnNoRecords.skip, 2101 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2102 title="On No Records", 2103 ) 2104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2105 2106 2107class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2108 type: Literal["ApiKey"] 2109 inject_into: RequestOption = Field( 2110 ..., 2111 description="Configure how the API Key will be sent in requests to the source API.", 2112 examples=[ 2113 {"inject_into": "header", "field_name": "Authorization"}, 2114 {"inject_into": "request_parameter", "field_name": "authKey"}, 2115 ], 2116 title="Inject API Key Into Outgoing HTTP Request", 2117 ) 2118 api_token: Optional[str] = Field( 2119 "{{ session_token }}", 2120 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2121 examples=[ 2122 "{{ session_token }}", 2123 "Token {{ session_token }}", 2124 "Bearer {{ session_token }}", 2125 ], 2126 title="API Token Template", 2127 ) 2128 2129 2130class JsonSchemaPropertySelector(BaseModel): 2131 type: Literal["JsonSchemaPropertySelector"] 2132 transformations: Optional[ 2133 List[ 2134 Union[ 2135 AddFields, 2136 RemoveFields, 2137 KeysToLower, 2138 KeysToSnakeCase, 2139 FlattenFields, 2140 DpathFlattenFields, 2141 KeysReplace, 2142 CustomTransformation, 2143 ] 2144 ] 2145 ] = Field( 2146 None, 2147 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2148 title="Transformations", 2149 ) 2150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2151 2152 2153class ListPartitionRouter(BaseModel): 2154 type: Literal["ListPartitionRouter"] 2155 cursor_field: str = Field( 2156 ..., 2157 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2158 examples=["section", "{{ config['section_key'] }}"], 2159 title="Current Partition Value Identifier", 2160 ) 2161 values: Union[str, List[str]] = Field( 2162 ..., 2163 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2164 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2165 title="Partition Values", 2166 ) 2167 request_option: Optional[RequestOption] = Field( 2168 None, 2169 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2170 title="Inject Partition Value Into Outgoing HTTP Request", 2171 ) 2172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2173 2174 2175class PaginationReset(BaseModel): 2176 type: Literal["PaginationReset"] 2177 action: Action1 2178 limits: Optional[PaginationResetLimits] = None 2179 2180 2181class GzipDecoder(BaseModel): 2182 type: Literal["GzipDecoder"] 2183 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2184 2185 2186class RequestBodyGraphQL(BaseModel): 2187 type: Literal["RequestBodyGraphQL"] 2188 value: RequestBodyGraphQlQuery 2189 2190 2191class DpathValidator(BaseModel): 2192 type: Literal["DpathValidator"] 2193 field_path: List[str] = Field( 2194 ..., 2195 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2196 examples=[ 2197 ["data"], 2198 ["data", "records"], 2199 ["data", "{{ parameters.name }}"], 2200 ["data", "*", "record"], 2201 ], 2202 title="Field Path", 2203 ) 2204 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2205 ..., 2206 description="The condition that the specified config value will be evaluated against", 2207 title="Validation Strategy", 2208 ) 2209 2210 2211class PredicateValidator(BaseModel): 2212 type: Literal["PredicateValidator"] 2213 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2214 ..., 2215 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2216 examples=[ 2217 "test-value", 2218 "{{ config['api_version'] }}", 2219 "{{ config['tenant_id'] }}", 2220 123, 2221 ], 2222 title="Value", 2223 ) 2224 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2225 ..., 2226 description="The validation strategy to apply to the value.", 2227 title="Validation Strategy", 2228 ) 2229 2230 2231class ConfigAddFields(BaseModel): 2232 type: Literal["ConfigAddFields"] 2233 fields: List[AddedFieldDefinition] = Field( 2234 ..., 2235 description="A list of transformations (path and corresponding value) that will be added to the config.", 2236 title="Fields", 2237 ) 2238 condition: Optional[str] = Field( 2239 "", 2240 description="Fields will be added if expression is evaluated to True.", 2241 examples=[ 2242 "{{ config['environemnt'] == 'sandbox' }}", 2243 "{{ property is integer }}", 2244 "{{ property|length > 5 }}", 2245 "{{ property == 'some_string_to_match' }}", 2246 ], 2247 ) 2248 2249 2250class CompositeErrorHandler(BaseModel): 2251 type: Literal["CompositeErrorHandler"] 2252 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2253 Field( 2254 ..., 2255 description="List of error handlers to iterate on to determine how to handle a failed response.", 2256 title="Error Handlers", 2257 ) 2258 ) 2259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2260 2261 2262class HTTPAPIBudget(BaseModel): 2263 class Config: 2264 extra = Extra.allow 2265 2266 type: Literal["HTTPAPIBudget"] 2267 policies: List[ 2268 Union[ 2269 FixedWindowCallRatePolicy, 2270 MovingWindowCallRatePolicy, 2271 UnlimitedCallRatePolicy, 2272 ] 2273 ] = Field( 2274 ..., 2275 description="List of call rate policies that define how many calls are allowed.", 2276 title="Policies", 2277 ) 2278 ratelimit_reset_header: Optional[str] = Field( 2279 "ratelimit-reset", 2280 description="The HTTP response header name that indicates when the rate limit resets.", 2281 title="Rate Limit Reset Header", 2282 ) 2283 ratelimit_remaining_header: Optional[str] = Field( 2284 "ratelimit-remaining", 2285 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2286 title="Rate Limit Remaining Header", 2287 ) 2288 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2289 [429], 2290 description="List of HTTP status codes that indicate a rate limit has been hit.", 2291 title="Status Codes for Rate Limit Hit", 2292 ) 2293 2294 2295class DpathExtractor(BaseModel): 2296 type: Literal["DpathExtractor"] 2297 field_path: List[str] = Field( 2298 ..., 2299 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2300 examples=[ 2301 ["data"], 2302 ["data", "records"], 2303 ["data", "{{ parameters.name }}"], 2304 ["data", "*", "record"], 2305 ], 2306 title="Field Path", 2307 ) 2308 record_expander: Optional[RecordExpander] = Field( 2309 None, 2310 description="Optional component to expand records by extracting items from nested array fields.", 2311 title="Record Expander", 2312 ) 2313 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2314 2315 2316class ZipfileDecoder(BaseModel): 2317 class Config: 2318 extra = Extra.allow 2319 2320 type: Literal["ZipfileDecoder"] 2321 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2322 ..., 2323 description="Parser to parse the decompressed data from the zipfile(s).", 2324 title="Parser", 2325 ) 2326 2327 2328class RecordSelector(BaseModel): 2329 type: Literal["RecordSelector"] 2330 extractor: Union[DpathExtractor, CustomRecordExtractor] 2331 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2332 None, 2333 description="Responsible for filtering records to be emitted by the Source.", 2334 title="Record Filter", 2335 ) 2336 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2337 None, 2338 description="Responsible for normalization according to the schema.", 2339 title="Schema Normalization", 2340 ) 2341 transform_before_filtering: Optional[bool] = Field( 2342 None, 2343 description="If true, transformation will be applied before record filtering.", 2344 title="Transform Before Filtering", 2345 ) 2346 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2347 2348 2349class ConfigMigration(BaseModel): 2350 type: Literal["ConfigMigration"] 2351 description: Optional[str] = Field( 2352 None, description="The description/purpose of the config migration." 2353 ) 2354 transformations: List[ 2355 Union[ 2356 ConfigRemapField, 2357 ConfigAddFields, 2358 ConfigRemoveFields, 2359 CustomConfigTransformation, 2360 ] 2361 ] = Field( 2362 ..., 2363 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2364 title="Transformations", 2365 ) 2366 2367 2368class ConfigNormalizationRules(BaseModel): 2369 class Config: 2370 extra = Extra.forbid 2371 2372 type: Literal["ConfigNormalizationRules"] 2373 config_migrations: Optional[List[ConfigMigration]] = Field( 2374 [], 2375 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2376 title="Config Migrations", 2377 ) 2378 transformations: Optional[ 2379 List[ 2380 Union[ 2381 ConfigRemapField, 2382 ConfigAddFields, 2383 ConfigRemoveFields, 2384 CustomConfigTransformation, 2385 ] 2386 ] 2387 ] = Field( 2388 [], 2389 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2390 title="Transformations", 2391 ) 2392 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2393 [], 2394 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2395 title="Validations", 2396 ) 2397 2398 2399class Spec(BaseModel): 2400 type: Literal["Spec"] 2401 connection_specification: Dict[str, Any] = Field( 2402 ..., 2403 description="A connection specification describing how a the connector can be configured.", 2404 title="Connection Specification", 2405 ) 2406 documentation_url: Optional[str] = Field( 2407 None, 2408 description="URL of the connector's documentation page.", 2409 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2410 title="Documentation URL", 2411 ) 2412 advanced_auth: Optional[AuthFlow] = Field( 2413 None, 2414 description="Advanced specification for configuring the authentication flow.", 2415 title="Advanced Auth", 2416 ) 2417 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2418 None, title="Config Normalization Rules" 2419 ) 2420 2421 2422class DeclarativeSource1(BaseModel): 2423 class Config: 2424 extra = Extra.forbid 2425 2426 type: Literal["DeclarativeSource"] 2427 check: Union[CheckStream, CheckDynamicStream] 2428 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2429 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2430 version: str = Field( 2431 ..., 2432 description="The version of the Airbyte CDK used to build and test the source.", 2433 ) 2434 schemas: Optional[Schemas] = None 2435 definitions: Optional[Dict[str, Any]] = None 2436 spec: Optional[Spec] = None 2437 concurrency_level: Optional[ConcurrencyLevel] = None 2438 api_budget: Optional[HTTPAPIBudget] = None 2439 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2440 None, 2441 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2442 title="Stream Groups", 2443 ) 2444 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2445 None, 2446 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2447 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2448 title="Maximum Concurrent Asynchronous Jobs", 2449 ) 2450 metadata: Optional[Dict[str, Any]] = Field( 2451 None, 2452 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2453 ) 2454 description: Optional[str] = Field( 2455 None, 2456 description="A description of the connector. It will be presented on the Source documentation page.", 2457 ) 2458 2459 2460class DeclarativeSource2(BaseModel): 2461 class Config: 2462 extra = Extra.forbid 2463 2464 type: Literal["DeclarativeSource"] 2465 check: Union[CheckStream, CheckDynamicStream] 2466 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2467 None 2468 ) 2469 dynamic_streams: List[DynamicDeclarativeStream] 2470 version: str = Field( 2471 ..., 2472 description="The version of the Airbyte CDK used to build and test the source.", 2473 ) 2474 schemas: Optional[Schemas] = None 2475 definitions: Optional[Dict[str, Any]] = None 2476 spec: Optional[Spec] = None 2477 concurrency_level: Optional[ConcurrencyLevel] = None 2478 api_budget: Optional[HTTPAPIBudget] = None 2479 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2480 None, 2481 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2482 title="Stream Groups", 2483 ) 2484 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2485 None, 2486 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2487 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2488 title="Maximum Concurrent Asynchronous Jobs", 2489 ) 2490 metadata: Optional[Dict[str, Any]] = Field( 2491 None, 2492 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2493 ) 2494 description: Optional[str] = Field( 2495 None, 2496 description="A description of the connector. It will be presented on the Source documentation page.", 2497 ) 2498 2499 2500class DeclarativeSource(BaseModel): 2501 class Config: 2502 extra = Extra.forbid 2503 2504 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2505 ..., 2506 description="An API source that extracts data according to its declarative components.", 2507 title="DeclarativeSource", 2508 ) 2509 2510 2511class SelectiveAuthenticator(BaseModel): 2512 class Config: 2513 extra = Extra.allow 2514 2515 type: Literal["SelectiveAuthenticator"] 2516 authenticator_selection_path: List[str] = Field( 2517 ..., 2518 description="Path of the field in config with selected authenticator name", 2519 examples=[["auth"], ["auth", "type"]], 2520 title="Authenticator Selection Path", 2521 ) 2522 authenticators: Dict[ 2523 str, 2524 Union[ 2525 ApiKeyAuthenticator, 2526 BasicHttpAuthenticator, 2527 BearerAuthenticator, 2528 OAuthAuthenticator, 2529 JwtAuthenticator, 2530 SessionTokenAuthenticator, 2531 LegacySessionTokenAuthenticator, 2532 CustomAuthenticator, 2533 NoAuth, 2534 ], 2535 ] = Field( 2536 ..., 2537 description="Authenticators to select from.", 2538 examples=[ 2539 { 2540 "authenticators": { 2541 "token": "#/definitions/ApiKeyAuthenticator", 2542 "oauth": "#/definitions/OAuthAuthenticator", 2543 "jwt": "#/definitions/JwtAuthenticator", 2544 } 2545 } 2546 ], 2547 title="Authenticators", 2548 ) 2549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2550 2551 2552class ConditionalStreams(BaseModel): 2553 type: Literal["ConditionalStreams"] 2554 condition: str = Field( 2555 ..., 2556 description="Condition that will be evaluated to determine if a set of streams should be available.", 2557 examples=["{{ config['is_sandbox'] }}"], 2558 title="Condition", 2559 ) 2560 streams: List[DeclarativeStream] = Field( 2561 ..., 2562 description="Streams that will be used during an operation based on the condition.", 2563 title="Streams", 2564 ) 2565 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2566 2567 2568class FileUploader(BaseModel): 2569 type: Literal["FileUploader"] 2570 requester: Union[HttpRequester, CustomRequester] = Field( 2571 ..., 2572 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2573 ) 2574 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2575 ..., 2576 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2577 ) 2578 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2579 None, 2580 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2581 ) 2582 filename_extractor: Optional[str] = Field( 2583 None, 2584 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2585 examples=[ 2586 "{{ record.id }}/{{ record.file_name }}/", 2587 "{{ record.id }}_{{ record.file_name }}/", 2588 ], 2589 ) 2590 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2591 2592 2593class DeclarativeStream(BaseModel): 2594 class Config: 2595 extra = Extra.allow 2596 2597 type: Literal["DeclarativeStream"] 2598 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2599 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2600 ..., 2601 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2602 title="Retriever", 2603 ) 2604 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2605 None, 2606 description="Component used to fetch data incrementally based on a time field in the data.", 2607 title="Incremental Sync", 2608 ) 2609 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2610 schema_loader: Optional[ 2611 Union[ 2612 InlineSchemaLoader, 2613 DynamicSchemaLoader, 2614 JsonFileSchemaLoader, 2615 List[ 2616 Union[ 2617 InlineSchemaLoader, 2618 DynamicSchemaLoader, 2619 JsonFileSchemaLoader, 2620 CustomSchemaLoader, 2621 ] 2622 ], 2623 CustomSchemaLoader, 2624 ] 2625 ] = Field( 2626 None, 2627 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2628 title="Schema Loader", 2629 ) 2630 transformations: Optional[ 2631 List[ 2632 Union[ 2633 AddFields, 2634 RemoveFields, 2635 KeysToLower, 2636 KeysToSnakeCase, 2637 FlattenFields, 2638 DpathFlattenFields, 2639 KeysReplace, 2640 CustomTransformation, 2641 ] 2642 ] 2643 ] = Field( 2644 None, 2645 description="A list of transformations to be applied to each output record.", 2646 title="Transformations", 2647 ) 2648 state_migrations: Optional[ 2649 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2650 ] = Field( 2651 [], 2652 description="Array of state migrations to be applied on the input state", 2653 title="State Migrations", 2654 ) 2655 file_uploader: Optional[FileUploader] = Field( 2656 None, 2657 description="(experimental) Describes how to fetch a file", 2658 title="File Uploader", 2659 ) 2660 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2661 2662 2663class SessionTokenAuthenticator(BaseModel): 2664 type: Literal["SessionTokenAuthenticator"] 2665 login_requester: HttpRequester = Field( 2666 ..., 2667 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2668 examples=[ 2669 { 2670 "type": "HttpRequester", 2671 "url_base": "https://my_api.com", 2672 "path": "/login", 2673 "authenticator": { 2674 "type": "BasicHttpAuthenticator", 2675 "username": "{{ config.username }}", 2676 "password": "{{ config.password }}", 2677 }, 2678 } 2679 ], 2680 title="Login Requester", 2681 ) 2682 session_token_path: List[str] = Field( 2683 ..., 2684 description="The path in the response body returned from the login requester to the session token.", 2685 examples=[["access_token"], ["result", "token"]], 2686 title="Session Token Path", 2687 ) 2688 expiration_duration: Optional[str] = Field( 2689 None, 2690 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2691 examples=["PT1H", "P1D"], 2692 title="Expiration Duration", 2693 ) 2694 request_authentication: Union[ 2695 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2696 ] = Field( 2697 ..., 2698 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2699 title="Data Request Authentication", 2700 ) 2701 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2702 None, description="Component used to decode the response.", title="Decoder" 2703 ) 2704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2705 2706 2707class HttpRequester(BaseModelWithDeprecations): 2708 type: Literal["HttpRequester"] 2709 url_base: Optional[str] = Field( 2710 None, 2711 deprecated=True, 2712 deprecation_message="Use `url` field instead.", 2713 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2714 examples=[ 2715 "https://connect.squareup.com/v2", 2716 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2717 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2718 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2719 ], 2720 title="API Base URL", 2721 ) 2722 url: Optional[str] = Field( 2723 None, 2724 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2725 examples=[ 2726 "https://connect.squareup.com/v2", 2727 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2728 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2729 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2730 ], 2731 title="API Endpoint URL", 2732 ) 2733 path: Optional[str] = Field( 2734 None, 2735 deprecated=True, 2736 deprecation_message="Use `url` field instead.", 2737 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2738 examples=[ 2739 "/products", 2740 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2741 "/trades/{{ config['symbol_id'] }}/history", 2742 ], 2743 title="URL Path", 2744 ) 2745 http_method: Optional[HttpMethod] = Field( 2746 HttpMethod.GET, 2747 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2748 examples=["GET", "POST"], 2749 title="HTTP Method", 2750 ) 2751 authenticator: Optional[ 2752 Union[ 2753 ApiKeyAuthenticator, 2754 BasicHttpAuthenticator, 2755 BearerAuthenticator, 2756 OAuthAuthenticator, 2757 JwtAuthenticator, 2758 SessionTokenAuthenticator, 2759 SelectiveAuthenticator, 2760 CustomAuthenticator, 2761 NoAuth, 2762 LegacySessionTokenAuthenticator, 2763 ] 2764 ] = Field( 2765 None, 2766 description="Authentication method to use for requests sent to the API.", 2767 title="Authenticator", 2768 ) 2769 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2770 None, 2771 deprecated=True, 2772 deprecation_message="Use `query_properties` field instead.", 2773 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2774 title="Fetch Properties from Endpoint", 2775 ) 2776 query_properties: Optional[QueryProperties] = Field( 2777 None, 2778 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2779 title="Query Properties", 2780 ) 2781 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2782 None, 2783 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2784 examples=[ 2785 {"unit": "day"}, 2786 { 2787 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2788 }, 2789 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2790 {"sort_by[asc]": "updated_at"}, 2791 ], 2792 title="Query Parameters", 2793 ) 2794 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2795 None, 2796 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2797 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2798 title="Request Headers", 2799 ) 2800 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2801 None, 2802 deprecated=True, 2803 deprecation_message="Use `request_body` field instead.", 2804 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2805 examples=[ 2806 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2807 ], 2808 title="Request Body Payload (Non-JSON)", 2809 ) 2810 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2811 None, 2812 deprecated=True, 2813 deprecation_message="Use `request_body` field instead.", 2814 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2815 examples=[ 2816 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2817 {"key": "{{ config['value'] }}"}, 2818 {"sort": {"field": "updated_at", "order": "ascending"}}, 2819 ], 2820 title="Request Body JSON Payload", 2821 ) 2822 request_body: Optional[ 2823 Union[ 2824 RequestBodyPlainText, 2825 RequestBodyUrlEncodedForm, 2826 RequestBodyJsonObject, 2827 RequestBodyGraphQL, 2828 ] 2829 ] = Field( 2830 None, 2831 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2832 title="Request Body", 2833 ) 2834 error_handler: Optional[ 2835 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2836 ] = Field( 2837 None, 2838 description="Error handler component that defines how to handle errors.", 2839 title="Error Handler", 2840 ) 2841 use_cache: Optional[bool] = Field( 2842 False, 2843 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2844 title="Use Cache", 2845 ) 2846 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2847 2848 2849class DynamicSchemaLoader(BaseModel): 2850 type: Literal["DynamicSchemaLoader"] 2851 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2852 ..., 2853 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2854 title="Retriever", 2855 ) 2856 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2857 None, 2858 description="Responsible for filtering fields to be added to json schema.", 2859 title="Schema Filter", 2860 ) 2861 schema_transformations: Optional[ 2862 List[ 2863 Union[ 2864 AddFields, 2865 RemoveFields, 2866 KeysToLower, 2867 KeysToSnakeCase, 2868 FlattenFields, 2869 DpathFlattenFields, 2870 KeysReplace, 2871 CustomTransformation, 2872 ] 2873 ] 2874 ] = Field( 2875 None, 2876 description="A list of transformations to be applied to the schema.", 2877 title="Schema Transformations", 2878 ) 2879 schema_type_identifier: SchemaTypeIdentifier 2880 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2881 2882 2883class ParentStreamConfig(BaseModel): 2884 type: Literal["ParentStreamConfig"] 2885 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2886 ..., description="Reference to the parent stream.", title="Parent Stream" 2887 ) 2888 parent_key: str = Field( 2889 ..., 2890 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2891 examples=["id", "{{ config['parent_record_id'] }}"], 2892 title="Parent Key", 2893 ) 2894 partition_field: str = Field( 2895 ..., 2896 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2897 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2898 title="Current Parent Key Value Identifier", 2899 ) 2900 request_option: Optional[RequestOption] = Field( 2901 None, 2902 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2903 title="Request Option", 2904 ) 2905 incremental_dependency: Optional[bool] = Field( 2906 False, 2907 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2908 title="Incremental Dependency", 2909 ) 2910 lazy_read_pointer: Optional[List[str]] = Field( 2911 [], 2912 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2913 title="Lazy Read Pointer", 2914 ) 2915 extra_fields: Optional[List[List[str]]] = Field( 2916 None, 2917 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2918 title="Extra Fields", 2919 ) 2920 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2921 2922 2923class PropertiesFromEndpoint(BaseModel): 2924 type: Literal["PropertiesFromEndpoint"] 2925 property_field_path: List[str] = Field( 2926 ..., 2927 description="Describes the path to the field that should be extracted", 2928 examples=[["name"]], 2929 ) 2930 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2931 ..., 2932 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2933 ) 2934 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2935 2936 2937class QueryProperties(BaseModel): 2938 type: Literal["QueryProperties"] 2939 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2940 ..., 2941 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2942 title="Property List", 2943 ) 2944 always_include_properties: Optional[List[str]] = Field( 2945 None, 2946 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2947 title="Always Include Properties", 2948 ) 2949 property_chunking: Optional[PropertyChunking] = Field( 2950 None, 2951 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2952 title="Property Chunking", 2953 ) 2954 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2955 None, 2956 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2957 title="Property Selector", 2958 ) 2959 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2960 2961 2962class StateDelegatingStream(BaseModel): 2963 type: Literal["StateDelegatingStream"] 2964 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2965 full_refresh_stream: DeclarativeStream = Field( 2966 ..., 2967 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2968 title="Full Refresh Stream", 2969 ) 2970 incremental_stream: DeclarativeStream = Field( 2971 ..., 2972 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2973 title="Incremental Stream", 2974 ) 2975 api_retention_period: Optional[str] = Field( 2976 None, 2977 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2978 examples=["P30D", "P90D", "P1Y"], 2979 title="API Retention Period", 2980 ) 2981 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2982 2983 2984class SimpleRetriever(BaseModel): 2985 type: Literal["SimpleRetriever"] 2986 requester: Union[HttpRequester, CustomRequester] = Field( 2987 ..., 2988 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2989 ) 2990 decoder: Optional[ 2991 Union[ 2992 JsonDecoder, 2993 XmlDecoder, 2994 CsvDecoder, 2995 JsonlDecoder, 2996 GzipDecoder, 2997 IterableDecoder, 2998 ZipfileDecoder, 2999 CustomDecoder, 3000 ] 3001 ] = Field( 3002 None, 3003 description="Component decoding the response so records can be extracted.", 3004 title="HTTP Response Format", 3005 ) 3006 record_selector: RecordSelector = Field( 3007 ..., 3008 description="Component that describes how to extract records from a HTTP response.", 3009 ) 3010 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3011 None, 3012 description="Paginator component that describes how to navigate through the API's pages.", 3013 ) 3014 pagination_reset: Optional[PaginationReset] = Field( 3015 None, 3016 description="Describes what triggers pagination reset and how to handle it.", 3017 ) 3018 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3019 False, 3020 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3021 ) 3022 partition_router: Optional[ 3023 Union[ 3024 SubstreamPartitionRouter, 3025 ListPartitionRouter, 3026 GroupingPartitionRouter, 3027 CustomPartitionRouter, 3028 List[ 3029 Union[ 3030 SubstreamPartitionRouter, 3031 ListPartitionRouter, 3032 GroupingPartitionRouter, 3033 CustomPartitionRouter, 3034 ] 3035 ], 3036 ] 3037 ] = Field( 3038 None, 3039 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3040 title="Partition Router", 3041 ) 3042 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3043 3044 3045class AsyncRetriever(BaseModel): 3046 type: Literal["AsyncRetriever"] 3047 record_selector: RecordSelector = Field( 3048 ..., 3049 description="Component that describes how to extract records from a HTTP response.", 3050 ) 3051 status_mapping: AsyncJobStatusMap = Field( 3052 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3053 ) 3054 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3055 ..., description="Responsible for fetching the actual status of the async job." 3056 ) 3057 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3058 None, 3059 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3060 ) 3061 download_extractor: Optional[ 3062 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3063 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3064 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3065 ..., 3066 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3067 ) 3068 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3069 ..., 3070 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3071 ) 3072 polling_job_timeout: Optional[Union[int, str]] = Field( 3073 None, 3074 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3075 ) 3076 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3077 None, 3078 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3079 ) 3080 download_requester: Union[HttpRequester, CustomRequester] = Field( 3081 ..., 3082 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3083 ) 3084 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3085 None, 3086 description="Paginator component that describes how to navigate through the API's pages during download.", 3087 ) 3088 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3089 None, 3090 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3091 ) 3092 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3093 None, 3094 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3095 ) 3096 partition_router: Optional[ 3097 Union[ 3098 ListPartitionRouter, 3099 SubstreamPartitionRouter, 3100 GroupingPartitionRouter, 3101 CustomPartitionRouter, 3102 List[ 3103 Union[ 3104 ListPartitionRouter, 3105 SubstreamPartitionRouter, 3106 GroupingPartitionRouter, 3107 CustomPartitionRouter, 3108 ] 3109 ], 3110 ] 3111 ] = Field( 3112 [], 3113 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3114 title="Partition Router", 3115 ) 3116 decoder: Optional[ 3117 Union[ 3118 CsvDecoder, 3119 GzipDecoder, 3120 JsonDecoder, 3121 JsonlDecoder, 3122 IterableDecoder, 3123 XmlDecoder, 3124 ZipfileDecoder, 3125 CustomDecoder, 3126 ] 3127 ] = Field( 3128 None, 3129 description="Component decoding the response so records can be extracted.", 3130 title="HTTP Response Format", 3131 ) 3132 download_decoder: Optional[ 3133 Union[ 3134 CsvDecoder, 3135 GzipDecoder, 3136 JsonDecoder, 3137 JsonlDecoder, 3138 IterableDecoder, 3139 XmlDecoder, 3140 ZipfileDecoder, 3141 CustomDecoder, 3142 ] 3143 ] = Field( 3144 None, 3145 description="Component decoding the download response so records can be extracted.", 3146 title="Download HTTP Response Format", 3147 ) 3148 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3149 3150 3151class BlockSimultaneousSyncsAction(BaseModel): 3152 type: Literal["BlockSimultaneousSyncsAction"] 3153 3154 3155class StreamGroup(BaseModel): 3156 streams: List[str] = Field( 3157 ..., 3158 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3159 title="Streams", 3160 ) 3161 action: BlockSimultaneousSyncsAction = Field( 3162 ..., 3163 description="The action to apply to streams in this group.", 3164 title="Action", 3165 ) 3166 3167 3168class SubstreamPartitionRouter(BaseModel): 3169 type: Literal["SubstreamPartitionRouter"] 3170 parent_stream_configs: List[ParentStreamConfig] = Field( 3171 ..., 3172 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3173 title="Parent Stream Configs", 3174 ) 3175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3176 3177 3178class GroupingPartitionRouter(BaseModel): 3179 type: Literal["GroupingPartitionRouter"] 3180 group_size: int = Field( 3181 ..., 3182 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3183 examples=[10, 50], 3184 title="Group Size", 3185 ) 3186 underlying_partition_router: Union[ 3187 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3188 ] = Field( 3189 ..., 3190 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3191 title="Underlying Partition Router", 3192 ) 3193 deduplicate: Optional[bool] = Field( 3194 True, 3195 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3196 title="Deduplicate Partitions", 3197 ) 3198 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3199 3200 3201class HttpComponentsResolver(BaseModel): 3202 type: Literal["HttpComponentsResolver"] 3203 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3204 ..., 3205 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3206 title="Retriever", 3207 ) 3208 components_mapping: List[ComponentMappingDefinition] 3209 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3210 3211 3212class DynamicDeclarativeStream(BaseModel): 3213 type: Literal["DynamicDeclarativeStream"] 3214 name: Optional[str] = Field( 3215 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3216 ) 3217 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3218 ..., description="Reference to the stream template.", title="Stream Template" 3219 ) 3220 components_resolver: Union[ 3221 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3222 ] = Field( 3223 ..., 3224 description="Component resolve and populates stream templates with components values.", 3225 title="Components Resolver", 3226 ) 3227 use_parent_parameters: Optional[bool] = Field( 3228 True, 3229 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3230 title="Use Parent Parameters", 3231 ) 3232 3233 3234ComplexFieldType.update_forward_refs() 3235GzipDecoder.update_forward_refs() 3236CompositeErrorHandler.update_forward_refs() 3237DeclarativeSource1.update_forward_refs() 3238DeclarativeSource2.update_forward_refs() 3239SelectiveAuthenticator.update_forward_refs() 3240ConditionalStreams.update_forward_refs() 3241FileUploader.update_forward_refs() 3242DeclarativeStream.update_forward_refs() 3243SessionTokenAuthenticator.update_forward_refs() 3244HttpRequester.update_forward_refs() 3245DynamicSchemaLoader.update_forward_refs() 3246ParentStreamConfig.update_forward_refs() 3247PropertiesFromEndpoint.update_forward_refs() 3248SimpleRetriever.update_forward_refs() 3249AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
111class CursorPagination(BaseModel): 112 type: Literal["CursorPagination"] 113 cursor_value: str = Field( 114 ..., 115 description="Value of the cursor defining the next page to fetch.", 116 examples=[ 117 "{{ headers.link.next.cursor }}", 118 "{{ last_record['key'] }}", 119 "{{ response['nextPage'] }}", 120 ], 121 title="Cursor Value", 122 ) 123 page_size: Optional[Union[int, str]] = Field( 124 None, 125 description="The number of records to include in each pages.", 126 examples=[100, "{{ config['page_size'] }}"], 127 title="Page Size", 128 ) 129 stop_condition: Optional[str] = Field( 130 None, 131 description="Template string evaluating when to stop paginating.", 132 examples=[ 133 "{{ response.data.has_more is false }}", 134 "{{ 'next' not in headers['link'] }}", 135 ], 136 title="Stop Condition", 137 ) 138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
141class CustomAuthenticator(BaseModel): 142 class Config: 143 extra = Extra.allow 144 145 type: Literal["CustomAuthenticator"] 146 class_name: str = Field( 147 ..., 148 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 149 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 150 title="Class Name", 151 ) 152 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
155class CustomBackoffStrategy(BaseModel): 156 class Config: 157 extra = Extra.allow 158 159 type: Literal["CustomBackoffStrategy"] 160 class_name: str = Field( 161 ..., 162 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 163 examples=["source_railz.components.MyCustomBackoffStrategy"], 164 title="Class Name", 165 ) 166 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
169class CustomErrorHandler(BaseModel): 170 class Config: 171 extra = Extra.allow 172 173 type: Literal["CustomErrorHandler"] 174 class_name: str = Field( 175 ..., 176 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 177 examples=["source_railz.components.MyCustomErrorHandler"], 178 title="Class Name", 179 ) 180 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
183class CustomPaginationStrategy(BaseModel): 184 class Config: 185 extra = Extra.allow 186 187 type: Literal["CustomPaginationStrategy"] 188 class_name: str = Field( 189 ..., 190 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 191 examples=["source_railz.components.MyCustomPaginationStrategy"], 192 title="Class Name", 193 ) 194 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
197class CustomRecordExtractor(BaseModel): 198 class Config: 199 extra = Extra.allow 200 201 type: Literal["CustomRecordExtractor"] 202 class_name: str = Field( 203 ..., 204 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 205 examples=["source_railz.components.MyCustomRecordExtractor"], 206 title="Class Name", 207 ) 208 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
211class CustomRecordFilter(BaseModel): 212 class Config: 213 extra = Extra.allow 214 215 type: Literal["CustomRecordFilter"] 216 class_name: str = Field( 217 ..., 218 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 219 examples=["source_railz.components.MyCustomCustomRecordFilter"], 220 title="Class Name", 221 ) 222 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
225class CustomRequester(BaseModel): 226 class Config: 227 extra = Extra.allow 228 229 type: Literal["CustomRequester"] 230 class_name: str = Field( 231 ..., 232 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 233 examples=["source_railz.components.MyCustomRecordExtractor"], 234 title="Class Name", 235 ) 236 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
239class CustomRetriever(BaseModel): 240 class Config: 241 extra = Extra.allow 242 243 type: Literal["CustomRetriever"] 244 class_name: str = Field( 245 ..., 246 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 247 examples=["source_railz.components.MyCustomRetriever"], 248 title="Class Name", 249 ) 250 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
253class CustomPartitionRouter(BaseModel): 254 class Config: 255 extra = Extra.allow 256 257 type: Literal["CustomPartitionRouter"] 258 class_name: str = Field( 259 ..., 260 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 261 examples=["source_railz.components.MyCustomPartitionRouter"], 262 title="Class Name", 263 ) 264 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
267class CustomSchemaLoader(BaseModel): 268 class Config: 269 extra = Extra.allow 270 271 type: Literal["CustomSchemaLoader"] 272 class_name: str = Field( 273 ..., 274 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 275 examples=["source_railz.components.MyCustomSchemaLoader"], 276 title="Class Name", 277 ) 278 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
281class CustomSchemaNormalization(BaseModel): 282 class Config: 283 extra = Extra.allow 284 285 type: Literal["CustomSchemaNormalization"] 286 class_name: str = Field( 287 ..., 288 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 289 examples=[ 290 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 291 ], 292 title="Class Name", 293 ) 294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
297class CustomStateMigration(BaseModel): 298 class Config: 299 extra = Extra.allow 300 301 type: Literal["CustomStateMigration"] 302 class_name: str = Field( 303 ..., 304 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 305 examples=["source_railz.components.MyCustomStateMigration"], 306 title="Class Name", 307 ) 308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
311class CustomTransformation(BaseModel): 312 class Config: 313 extra = Extra.allow 314 315 type: Literal["CustomTransformation"] 316 class_name: str = Field( 317 ..., 318 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 319 examples=["source_railz.components.MyCustomTransformation"], 320 title="Class Name", 321 ) 322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
342class Algorithm(Enum): 343 HS256 = "HS256" 344 HS384 = "HS384" 345 HS512 = "HS512" 346 ES256 = "ES256" 347 ES256K = "ES256K" 348 ES384 = "ES384" 349 ES512 = "ES512" 350 RS256 = "RS256" 351 RS384 = "RS384" 352 RS512 = "RS512" 353 PS256 = "PS256" 354 PS384 = "PS384" 355 PS512 = "PS512" 356 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
359class JwtHeaders(BaseModel): 360 class Config: 361 extra = Extra.forbid 362 363 kid: Optional[str] = Field( 364 None, 365 description="Private key ID for user account.", 366 examples=["{{ config['kid'] }}"], 367 title="Key Identifier", 368 ) 369 typ: Optional[str] = Field( 370 "JWT", 371 description="The media type of the complete JWT.", 372 examples=["JWT"], 373 title="Type", 374 ) 375 cty: Optional[str] = Field( 376 None, 377 description="Content type of JWT header.", 378 examples=["JWT"], 379 title="Content Type", 380 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
383class JwtPayload(BaseModel): 384 class Config: 385 extra = Extra.forbid 386 387 iss: Optional[str] = Field( 388 None, 389 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 390 examples=["{{ config['iss'] }}"], 391 title="Issuer", 392 ) 393 sub: Optional[str] = Field( 394 None, 395 description="The subject of the JWT. Commonly defined by the API.", 396 title="Subject", 397 ) 398 aud: Optional[str] = Field( 399 None, 400 description="The recipient that the JWT is intended for. Commonly defined by the API.", 401 examples=["appstoreconnect-v1"], 402 title="Audience", 403 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
406class RefreshTokenUpdater(BaseModel): 407 refresh_token_name: Optional[str] = Field( 408 "refresh_token", 409 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 410 examples=["refresh_token"], 411 title="Refresh Token Property Name", 412 ) 413 access_token_config_path: Optional[List[str]] = Field( 414 ["credentials", "access_token"], 415 description="Config path to the access token. Make sure the field actually exists in the config.", 416 examples=[["credentials", "access_token"], ["access_token"]], 417 title="Config Path To Access Token", 418 ) 419 refresh_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "refresh_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "refresh_token"], ["refresh_token"]], 423 title="Config Path To Refresh Token", 424 ) 425 token_expiry_date_config_path: Optional[List[str]] = Field( 426 ["credentials", "token_expiry_date"], 427 description="Config path to the expiry date. Make sure actually exists in the config.", 428 examples=[["credentials", "token_expiry_date"]], 429 title="Config Path To Expiry Date", 430 ) 431 refresh_token_error_status_codes: Optional[List[int]] = Field( 432 [], 433 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 434 examples=[[400, 500]], 435 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 436 ) 437 refresh_token_error_key: Optional[str] = Field( 438 "", 439 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 440 examples=["error"], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 442 ) 443 refresh_token_error_values: Optional[List[str]] = Field( 444 [], 445 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 446 examples=[["invalid_grant", "invalid_permissions"]], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 448 )
class
Rate(pydantic.v1.main.BaseModel):
451class Rate(BaseModel): 452 class Config: 453 extra = Extra.allow 454 455 limit: Union[int, str] = Field( 456 ..., 457 description="The maximum number of calls allowed within the interval.", 458 title="Limit", 459 ) 460 interval: str = Field( 461 ..., 462 description="The time interval for the rate limit.", 463 examples=["PT1H", "P1D"], 464 title="Interval", 465 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
468class HttpRequestRegexMatcher(BaseModel): 469 class Config: 470 extra = Extra.allow 471 472 method: Optional[str] = Field( 473 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 474 ) 475 url_base: Optional[str] = Field( 476 None, 477 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 478 title="URL Base", 479 ) 480 url_path_pattern: Optional[str] = Field( 481 None, 482 description="A regular expression pattern to match the URL path.", 483 title="URL Path Pattern", 484 ) 485 params: Optional[Dict[str, Any]] = Field( 486 None, description="The query parameters to match.", title="Parameters" 487 ) 488 headers: Optional[Dict[str, Any]] = Field( 489 None, description="The headers to match.", title="Headers" 490 ) 491 weight: Optional[Union[int, str]] = Field( 492 None, 493 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 494 title="Weight", 495 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
OnNoRecords(enum.Enum):
An enumeration.
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
508class ExponentialBackoffStrategy(BaseModel): 509 type: Literal["ExponentialBackoffStrategy"] 510 factor: Optional[Union[float, str]] = Field( 511 5, 512 description="Multiplicative constant applied on each retry.", 513 examples=[5, 5.5, "10"], 514 title="Factor", 515 ) 516 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
519class GroupByKeyMergeStrategy(BaseModel): 520 type: Literal["GroupByKeyMergeStrategy"] 521 key: Union[str, List[str]] = Field( 522 ..., 523 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 524 examples=["id", ["parent_id", "end_date"]], 525 title="Key", 526 ) 527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
539class Action(Enum): 540 SUCCESS = "SUCCESS" 541 FAIL = "FAIL" 542 RETRY = "RETRY" 543 IGNORE = "IGNORE" 544 RESET_PAGINATION = "RESET_PAGINATION" 545 RATE_LIMITED = "RATE_LIMITED" 546 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
549class FailureType(Enum): 550 system_error = "system_error" 551 config_error = "config_error" 552 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
555class HttpResponseFilter(BaseModel): 556 type: Literal["HttpResponseFilter"] 557 action: Optional[Action] = Field( 558 None, 559 description="Action to execute if a response matches the filter.", 560 examples=[ 561 "SUCCESS", 562 "FAIL", 563 "RETRY", 564 "IGNORE", 565 "RESET_PAGINATION", 566 "RATE_LIMITED", 567 "REFRESH_TOKEN_THEN_RETRY", 568 ], 569 title="Action", 570 ) 571 failure_type: Optional[FailureType] = Field( 572 None, 573 description="Failure type of traced exception if a response matches the filter.", 574 examples=["system_error", "config_error", "transient_error"], 575 title="Failure Type", 576 ) 577 error_message: Optional[str] = Field( 578 None, 579 description="Error Message to display if the response matches the filter.", 580 title="Error Message", 581 ) 582 error_message_contains: Optional[str] = Field( 583 None, 584 description="Match the response if its error message contains the substring.", 585 example=["This API operation is not enabled for this site"], 586 title="Error Message Substring", 587 ) 588 http_codes: Optional[List[int]] = Field( 589 None, 590 description="Match the response if its HTTP code is included in this list.", 591 examples=[[420, 429], [500]], 592 title="HTTP Codes", 593 unique_items=True, 594 ) 595 predicate: Optional[str] = Field( 596 None, 597 description="Match the response if the predicate evaluates to true.", 598 examples=[ 599 "{{ 'Too much requests' in response }}", 600 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 601 ], 602 title="Predicate", 603 ) 604 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
607class ComplexFieldType(BaseModel): 608 field_type: str 609 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
612class TypesMap(BaseModel): 613 target_type: Union[str, List[str], ComplexFieldType] 614 current_type: Union[str, List[str]] 615 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
618class SchemaTypeIdentifier(BaseModel): 619 type: Optional[Literal["SchemaTypeIdentifier"]] = None 620 schema_pointer: Optional[List[str]] = Field( 621 [], 622 description="List of nested fields defining the schema field path to extract. Defaults to [].", 623 title="Schema Path", 624 ) 625 key_pointer: List[str] = Field( 626 ..., 627 description="List of potentially nested fields describing the full path of the field key to extract.", 628 title="Key Path", 629 ) 630 type_pointer: Optional[List[str]] = Field( 631 None, 632 description="List of potentially nested fields describing the full path of the field type to extract.", 633 title="Type Path", 634 ) 635 types_mapping: Optional[List[TypesMap]] = None 636 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
639class InlineSchemaLoader(BaseModel): 640 type: Literal["InlineSchemaLoader"] 641 schema_: Optional[Dict[str, Any]] = Field( 642 None, 643 alias="schema", 644 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 645 title="Schema", 646 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
649class JsonFileSchemaLoader(BaseModel): 650 type: Literal["JsonFileSchemaLoader"] 651 file_path: Optional[str] = Field( 652 None, 653 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 654 example=["./schemas/users.json"], 655 title="File Path", 656 ) 657 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
678class FlattenFields(BaseModel): 679 type: Literal["FlattenFields"] 680 flatten_lists: Optional[bool] = Field( 681 True, 682 description="Whether to flatten lists or leave it as is. Default is True.", 683 title="Flatten Lists", 684 ) 685 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
688class KeyTransformation(BaseModel): 689 type: Literal["KeyTransformation"] 690 prefix: Optional[str] = Field( 691 None, 692 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 693 examples=["flattened_"], 694 title="Key Prefix", 695 ) 696 suffix: Optional[str] = Field( 697 None, 698 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 699 examples=["_flattened"], 700 title="Key Suffix", 701 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
704class DpathFlattenFields(BaseModel): 705 type: Literal["DpathFlattenFields"] 706 field_path: List[str] = Field( 707 ..., 708 description="A path to field that needs to be flattened.", 709 examples=[["data"], ["data", "*", "field"]], 710 title="Field Path", 711 ) 712 delete_origin_value: Optional[bool] = Field( 713 None, 714 description="Whether to delete the origin value or keep it. Default is False.", 715 title="Delete Origin Value", 716 ) 717 replace_record: Optional[bool] = Field( 718 None, 719 description="Whether to replace the origin record or not. Default is False.", 720 title="Replace Origin Record", 721 ) 722 key_transformation: Optional[KeyTransformation] = Field( 723 None, 724 description="Transformation for object keys. If not provided, original key will be used.", 725 title="Key transformation", 726 ) 727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
730class KeysReplace(BaseModel): 731 type: Literal["KeysReplace"] 732 old: str = Field( 733 ..., 734 description="Old value to replace.", 735 examples=[ 736 " ", 737 "{{ record.id }}", 738 "{{ config['id'] }}", 739 "{{ stream_slice['id'] }}", 740 ], 741 title="Old value", 742 ) 743 new: str = Field( 744 ..., 745 description="New value to set.", 746 examples=[ 747 "_", 748 "{{ record.id }}", 749 "{{ config['id'] }}", 750 "{{ stream_slice['id'] }}", 751 ], 752 title="New value", 753 ) 754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
765class CustomDecoder(BaseModel): 766 class Config: 767 extra = Extra.allow 768 769 type: Literal["CustomDecoder"] 770 class_name: str = Field( 771 ..., 772 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 773 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 774 title="Class Name", 775 ) 776 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
779class MinMaxDatetime(BaseModel): 780 type: Literal["MinMaxDatetime"] 781 datetime: str = Field( 782 ..., 783 description="Datetime value.", 784 examples=[ 785 "2021-01-01", 786 "2021-01-01T00:00:00Z", 787 "{{ config['start_time'] }}", 788 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 789 ], 790 title="Datetime", 791 ) 792 datetime_format: Optional[str] = Field( 793 "", 794 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 795 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 796 title="Datetime Format", 797 ) 798 max_datetime: Optional[str] = Field( 799 None, 800 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 801 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 802 title="Max Datetime", 803 ) 804 min_datetime: Optional[str] = Field( 805 None, 806 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 807 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 808 title="Min Datetime", 809 ) 810 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
840class OauthConnectorInputSpecification(BaseModel): 841 class Config: 842 extra = Extra.allow 843 844 consent_url: str = Field( 845 ..., 846 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 847 examples=[ 848 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 849 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 850 ], 851 title="Consent URL", 852 ) 853 scope: Optional[str] = Field( 854 None, 855 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 856 examples=["user:read user:read_orders workspaces:read"], 857 title="Scopes", 858 ) 859 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 860 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 861 # The CDK schema defines the manifest contract; the platform reads these fields 862 # during the OAuth consent flow to build the authorization URL. 863 scopes: Optional[List[OAuthScope]] = Field( 864 None, 865 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 866 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 867 title="Scopes", 868 ) 869 optional_scopes: Optional[List[OAuthScope]] = Field( 870 None, 871 description="Optional OAuth scope objects that may or may not be granted.", 872 examples=[[{"scope": "admin:read"}]], 873 title="Optional Scopes", 874 ) 875 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 876 ScopesJoinStrategy.space, 877 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 878 title="Scopes Join Strategy", 879 ) 880 access_token_url: str = Field( 881 ..., 882 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 883 examples=[ 884 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 885 ], 886 title="Access Token URL", 887 ) 888 access_token_headers: Optional[Dict[str, Any]] = Field( 889 None, 890 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 891 examples=[ 892 { 893 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 894 } 895 ], 896 title="Access Token Headers", 897 ) 898 access_token_params: Optional[Dict[str, Any]] = Field( 899 None, 900 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 901 examples=[ 902 { 903 "{{ auth_code_key }}": "{{ auth_code_value }}", 904 "{{ client_id_key }}": "{{ client_id_value }}", 905 "{{ client_secret_key }}": "{{ client_secret_value }}", 906 } 907 ], 908 title="Access Token Query Params (Json Encoded)", 909 ) 910 extract_output: Optional[List[str]] = Field( 911 None, 912 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 913 examples=[["access_token", "refresh_token", "other_field"]], 914 title="Extract Output", 915 ) 916 state: Optional[State] = Field( 917 None, 918 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 919 examples=[{"min": 7, "max": 128}], 920 title="Configurable State Query Param", 921 ) 922 client_id_key: Optional[str] = Field( 923 None, 924 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 925 examples=["my_custom_client_id_key_name"], 926 title="Client ID Key Override", 927 ) 928 client_secret_key: Optional[str] = Field( 929 None, 930 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 931 examples=["my_custom_client_secret_key_name"], 932 title="Client Secret Key Override", 933 ) 934 scope_key: Optional[str] = Field( 935 None, 936 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 937 examples=["my_custom_scope_key_key_name"], 938 title="Scopes Key Override", 939 ) 940 state_key: Optional[str] = Field( 941 None, 942 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 943 examples=["my_custom_state_key_key_name"], 944 title="State Key Override", 945 ) 946 auth_code_key: Optional[str] = Field( 947 None, 948 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 949 examples=["my_custom_auth_code_key_name"], 950 title="Auth Code Key Override", 951 ) 952 redirect_uri_key: Optional[str] = Field( 953 None, 954 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 955 examples=["my_custom_redirect_uri_key_name"], 956 title="Redirect URI Key Override", 957 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
960class OAuthConfigSpecification(BaseModel): 961 class Config: 962 extra = Extra.allow 963 964 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 965 None, 966 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 967 examples=[ 968 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 969 { 970 "app_id": { 971 "type": "string", 972 "path_in_connector_config": ["info", "app_id"], 973 } 974 }, 975 ], 976 title="OAuth user input", 977 ) 978 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 979 None, 980 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 981 title="DeclarativeOAuth Connector Specification", 982 ) 983 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 984 None, 985 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 986 examples=[ 987 { 988 "refresh_token": { 989 "type": "string,", 990 "path_in_connector_config": ["credentials", "refresh_token"], 991 } 992 } 993 ], 994 title="OAuth output specification", 995 ) 996 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 997 None, 998 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 999 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1000 title="OAuth input specification", 1001 ) 1002 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1003 None, 1004 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1005 examples=[ 1006 { 1007 "client_id": { 1008 "type": "string,", 1009 "path_in_connector_config": ["credentials", "client_id"], 1010 }, 1011 "client_secret": { 1012 "type": "string,", 1013 "path_in_connector_config": ["credentials", "client_secret"], 1014 }, 1015 } 1016 ], 1017 title="OAuth server output specification", 1018 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1021class OffsetIncrement(BaseModel): 1022 type: Literal["OffsetIncrement"] 1023 page_size: Optional[Union[int, str]] = Field( 1024 None, 1025 description="The number of records to include in each pages.", 1026 examples=[100, "{{ config['page_size'] }}"], 1027 title="Limit", 1028 ) 1029 inject_on_first_request: Optional[bool] = Field( 1030 False, 1031 description="Using the `offset` with value `0` during the first request", 1032 title="Inject Offset on First Request", 1033 ) 1034 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1037class PageIncrement(BaseModel): 1038 type: Literal["PageIncrement"] 1039 page_size: Optional[Union[int, str]] = Field( 1040 None, 1041 description="The number of records to include in each pages.", 1042 examples=[100, "100", "{{ config['page_size'] }}"], 1043 title="Page Size", 1044 ) 1045 start_from_page: Optional[int] = Field( 1046 0, 1047 description="Index of the first page to request.", 1048 examples=[0, 1], 1049 title="Start From Page", 1050 ) 1051 inject_on_first_request: Optional[bool] = Field( 1052 False, 1053 description="Using the `page number` with value defined by `start_from_page` during the first request", 1054 title="Inject Page Number on First Request", 1055 ) 1056 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1059class PrimaryKey(BaseModel): 1060 __root__: Union[str, List[str], List[List[str]]] = Field( 1061 ..., 1062 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1063 examples=["id", ["code", "type"]], 1064 title="Primary Key", 1065 )
class
PropertyLimitType(enum.Enum):
1068class PropertyLimitType(Enum): 1069 characters = "characters" 1070 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1073class PropertyChunking(BaseModel): 1074 type: Literal["PropertyChunking"] 1075 property_limit_type: PropertyLimitType = Field( 1076 ..., 1077 description="The type used to determine the maximum number of properties per chunk", 1078 title="Property Limit Type", 1079 ) 1080 property_limit: Optional[int] = Field( 1081 None, 1082 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1083 title="Property Limit", 1084 ) 1085 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1086 None, 1087 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1088 title="Record Merge Strategy", 1089 ) 1090 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1093class RecordFilter(BaseModel): 1094 type: Literal["RecordFilter"] 1095 condition: Optional[str] = Field( 1096 "", 1097 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1098 examples=[ 1099 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1100 "{{ record.status in ['active', 'expired'] }}", 1101 ], 1102 title="Condition", 1103 ) 1104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1112class RemoveFields(BaseModel): 1113 type: Literal["RemoveFields"] 1114 condition: Optional[str] = Field( 1115 "", 1116 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1117 examples=[ 1118 "{{ property|string == '' }}", 1119 "{{ property is integer }}", 1120 "{{ property|length > 5 }}", 1121 "{{ property == 'some_string_to_match' }}", 1122 ], 1123 ) 1124 field_pointers: List[List[str]] = Field( 1125 ..., 1126 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1127 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1128 title="Field Paths", 1129 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1136class InjectInto(Enum): 1137 request_parameter = "request_parameter" 1138 header = "header" 1139 body_data = "body_data" 1140 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1143class RequestOption(BaseModel): 1144 type: Literal["RequestOption"] 1145 inject_into: InjectInto = Field( 1146 ..., 1147 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1148 examples=["request_parameter", "header", "body_data", "body_json"], 1149 title="Inject Into", 1150 ) 1151 field_name: Optional[str] = Field( 1152 None, 1153 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1154 examples=["segment_id"], 1155 title="Field Name", 1156 ) 1157 field_path: Optional[List[str]] = Field( 1158 None, 1159 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1160 examples=[["data", "viewer", "id"]], 1161 title="Field Path", 1162 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1172class LegacySessionTokenAuthenticator(BaseModel): 1173 type: Literal["LegacySessionTokenAuthenticator"] 1174 header: str = Field( 1175 ..., 1176 description="The name of the session token header that will be injected in the request", 1177 examples=["X-Session"], 1178 title="Session Request Header", 1179 ) 1180 login_url: str = Field( 1181 ..., 1182 description="Path of the login URL (do not include the base URL)", 1183 examples=["session"], 1184 title="Login Path", 1185 ) 1186 session_token: Optional[str] = Field( 1187 None, 1188 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1189 example=["{{ config['session_token'] }}"], 1190 title="Session Token", 1191 ) 1192 session_token_response_key: str = Field( 1193 ..., 1194 description="Name of the key of the session token to be extracted from the response", 1195 examples=["id"], 1196 title="Response Token Response Key", 1197 ) 1198 username: Optional[str] = Field( 1199 None, 1200 description="Username used to authenticate and obtain a session token", 1201 examples=[" {{ config['username'] }}"], 1202 title="Username", 1203 ) 1204 password: Optional[str] = Field( 1205 "", 1206 description="Password used to authenticate and obtain a session token", 1207 examples=["{{ config['password'] }}", ""], 1208 title="Password", 1209 ) 1210 validate_session_url: str = Field( 1211 ..., 1212 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1213 examples=["user/current"], 1214 title="Validate Session Path", 1215 ) 1216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1244class ValueType(Enum): 1245 string = "string" 1246 number = "number" 1247 integer = "integer" 1248 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1251class WaitTimeFromHeader(BaseModel): 1252 type: Literal["WaitTimeFromHeader"] 1253 header: str = Field( 1254 ..., 1255 description="The name of the response header defining how long to wait before retrying.", 1256 examples=["Retry-After"], 1257 title="Response Header Name", 1258 ) 1259 regex: Optional[str] = Field( 1260 None, 1261 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1262 examples=["([-+]?\\d+)"], 1263 title="Extraction Regex", 1264 ) 1265 max_waiting_time_in_seconds: Optional[float] = Field( 1266 None, 1267 description="Given the value extracted from the header is greater than this value, stop the stream.", 1268 examples=[3600], 1269 title="Max Waiting Time in Seconds", 1270 ) 1271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1274class WaitUntilTimeFromHeader(BaseModel): 1275 type: Literal["WaitUntilTimeFromHeader"] 1276 header: str = Field( 1277 ..., 1278 description="The name of the response header defining how long to wait before retrying.", 1279 examples=["wait_time"], 1280 title="Response Header", 1281 ) 1282 min_wait: Optional[Union[float, str]] = Field( 1283 None, 1284 description="Minimum time to wait before retrying.", 1285 examples=[10, "60"], 1286 title="Minimum Wait Time", 1287 ) 1288 regex: Optional[str] = Field( 1289 None, 1290 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1291 examples=["([-+]?\\d+)"], 1292 title="Extraction Regex", 1293 ) 1294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1297class ComponentMappingDefinition(BaseModel): 1298 type: Literal["ComponentMappingDefinition"] 1299 field_path: List[str] = Field( 1300 ..., 1301 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1302 examples=[ 1303 ["name"], 1304 ["retriever", "requester", "url"], 1305 ["retriever", "requester", "{{ components_values.field }}"], 1306 ["*", "**", "name"], 1307 ], 1308 title="Field Path", 1309 ) 1310 value: str = Field( 1311 ..., 1312 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1313 examples=[ 1314 "{{ components_values['updates'] }}", 1315 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1316 "{{ config['segment_id'] }}", 1317 "{{ stream_slice['parent_id'] }}", 1318 "{{ stream_slice['extra_fields']['name'] }}", 1319 ], 1320 title="Value", 1321 ) 1322 value_type: Optional[ValueType] = Field( 1323 None, 1324 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1325 title="Value Type", 1326 ) 1327 create_or_update: Optional[bool] = Field( 1328 False, 1329 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1330 title="Create or Update", 1331 ) 1332 condition: Optional[str] = Field( 1333 None, 1334 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1335 examples=[ 1336 "{{ components_values.get('cursor_field', None) }}", 1337 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1338 ], 1339 title="Condition", 1340 ) 1341 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1344class StreamConfig(BaseModel): 1345 type: Literal["StreamConfig"] 1346 configs_pointer: List[str] = Field( 1347 ..., 1348 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1349 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1350 title="Configs Pointer", 1351 ) 1352 default_values: Optional[List[Dict[str, Any]]] = Field( 1353 None, 1354 description="A list of default values, each matching the structure expected from the parsed component value.", 1355 title="Default Values", 1356 ) 1357 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1360class ConfigComponentsResolver(BaseModel): 1361 type: Literal["ConfigComponentsResolver"] 1362 stream_config: Union[List[StreamConfig], StreamConfig] 1363 components_mapping: List[ComponentMappingDefinition] 1364 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1367class StreamParametersDefinition(BaseModel): 1368 type: Literal["StreamParametersDefinition"] 1369 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1370 ..., 1371 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1372 examples=[ 1373 [ 1374 { 1375 "name": "test stream", 1376 "$parameters": {"entity": "test entity"}, 1377 "primary_key": "test key", 1378 } 1379 ] 1380 ], 1381 title="Stream Parameters", 1382 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1385class ParametrizedComponentsResolver(BaseModel): 1386 type: Literal["ParametrizedComponentsResolver"] 1387 stream_parameters: StreamParametersDefinition 1388 components_mapping: List[ComponentMappingDefinition] 1389 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1414class ValidateAdheresToSchema(BaseModel): 1415 type: Literal["ValidateAdheresToSchema"] 1416 base_schema: Union[str, Dict[str, Any]] = Field( 1417 ..., 1418 description="The base JSON schema against which the user-provided schema will be validated.", 1419 examples=[ 1420 "{{ config['report_validation_schema'] }}", 1421 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1422 { 1423 "$schema": "http://json-schema.org/draft-07/schema#", 1424 "title": "Person", 1425 "type": "object", 1426 "properties": { 1427 "name": {"type": "string", "description": "The person's name"}, 1428 "age": { 1429 "type": "integer", 1430 "minimum": 0, 1431 "description": "The person's age", 1432 }, 1433 }, 1434 "required": ["name", "age"], 1435 }, 1436 ], 1437 title="Base JSON Schema", 1438 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1441class CustomValidationStrategy(BaseModel): 1442 class Config: 1443 extra = Extra.allow 1444 1445 type: Literal["CustomValidationStrategy"] 1446 class_name: str = Field( 1447 ..., 1448 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1449 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1450 title="Class Name", 1451 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1454class ConfigRemapField(BaseModel): 1455 type: Literal["ConfigRemapField"] 1456 map: Union[Dict[str, Any], str] = Field( 1457 ..., 1458 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1459 examples=[ 1460 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1461 "{{ config['status_mapping'] }}", 1462 ], 1463 title="Value Mapping", 1464 ) 1465 field_path: List[str] = Field( 1466 ..., 1467 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1468 examples=[ 1469 ["status"], 1470 ["data", "status"], 1471 ["data", "{{ config.name }}", "status"], 1472 ["data", "*", "status"], 1473 ], 1474 title="Field Path", 1475 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1478class ConfigRemoveFields(BaseModel): 1479 type: Literal["ConfigRemoveFields"] 1480 field_pointers: List[List[str]] = Field( 1481 ..., 1482 description="A list of field pointers to be removed from the config.", 1483 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1484 title="Field Pointers", 1485 ) 1486 condition: Optional[str] = Field( 1487 "", 1488 description="Fields will be removed if expression is evaluated to True.", 1489 examples=[ 1490 "{{ config['environemnt'] == 'sandbox' }}", 1491 "{{ property is integer }}", 1492 "{{ property|length > 5 }}", 1493 "{{ property == 'some_string_to_match' }}", 1494 ], 1495 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1498class CustomConfigTransformation(BaseModel): 1499 type: Literal["CustomConfigTransformation"] 1500 class_name: str = Field( 1501 ..., 1502 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1503 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field( 1506 None, 1507 alias="$parameters", 1508 description="Additional parameters to be passed to the custom config transformation.", 1509 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1512class AddedFieldDefinition(BaseModel): 1513 type: Literal["AddedFieldDefinition"] 1514 path: List[str] = Field( 1515 ..., 1516 description="List of strings defining the path where to add the value on the record.", 1517 examples=[["segment_id"], ["metadata", "segment_id"]], 1518 title="Path", 1519 ) 1520 value: str = Field( 1521 ..., 1522 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1523 examples=[ 1524 "{{ record['updates'] }}", 1525 "{{ record['MetaData']['LastUpdatedTime'] }}", 1526 "{{ stream_partition['segment_id'] }}", 1527 ], 1528 title="Value", 1529 ) 1530 value_type: Optional[ValueType] = Field( 1531 None, 1532 description="Type of the value. If not specified, the type will be inferred from the value.", 1533 title="Value Type", 1534 ) 1535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1538class AddFields(BaseModel): 1539 type: Literal["AddFields"] 1540 fields: List[AddedFieldDefinition] = Field( 1541 ..., 1542 description="List of transformations (path and corresponding value) that will be added to the record.", 1543 title="Fields", 1544 ) 1545 condition: Optional[str] = Field( 1546 "", 1547 description="Fields will be added if expression is evaluated to True.", 1548 examples=[ 1549 "{{ property|string == '' }}", 1550 "{{ property is integer }}", 1551 "{{ property|length > 5 }}", 1552 "{{ property == 'some_string_to_match' }}", 1553 ], 1554 ) 1555 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1558class ApiKeyAuthenticator(BaseModel): 1559 type: Literal["ApiKeyAuthenticator"] 1560 api_token: Optional[str] = Field( 1561 None, 1562 description="The API key to inject in the request. Fill it in the user inputs.", 1563 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1564 title="API Key", 1565 ) 1566 header: Optional[str] = Field( 1567 None, 1568 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1569 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1570 title="Header Name", 1571 ) 1572 inject_into: Optional[RequestOption] = Field( 1573 None, 1574 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1575 examples=[ 1576 {"inject_into": "header", "field_name": "Authorization"}, 1577 {"inject_into": "request_parameter", "field_name": "authKey"}, 1578 ], 1579 title="Inject API Key Into Outgoing HTTP Request", 1580 ) 1581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1584class AuthFlow(BaseModel): 1585 auth_flow_type: Optional[AuthFlowType] = Field( 1586 None, description="The type of auth to use", title="Auth flow type" 1587 ) 1588 predicate_key: Optional[List[str]] = Field( 1589 None, 1590 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1591 examples=[["credentials", "auth_type"]], 1592 title="Predicate key", 1593 ) 1594 predicate_value: Optional[str] = Field( 1595 None, 1596 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1597 examples=["Oauth"], 1598 title="Predicate value", 1599 ) 1600 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1603class CheckStream(BaseModel): 1604 type: Literal["CheckStream"] 1605 stream_names: Optional[List[str]] = Field( 1606 None, 1607 description="Names of the streams to try reading from when running a check operation.", 1608 examples=[["users"], ["users", "contacts"]], 1609 title="Stream Names", 1610 ) 1611 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1614class IncrementingCountCursor(BaseModel): 1615 type: Literal["IncrementingCountCursor"] 1616 cursor_field: str = Field( 1617 ..., 1618 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1619 examples=["created_at", "{{ config['record_cursor'] }}"], 1620 title="Cursor Field", 1621 ) 1622 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1623 None, 1624 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1625 title="Allow Catalog Defined Cursor Field", 1626 ) 1627 start_value: Optional[Union[str, int]] = Field( 1628 None, 1629 description="The value that determines the earliest record that should be synced.", 1630 examples=[0, "{{ config['start_value'] }}"], 1631 title="Start Value", 1632 ) 1633 start_value_option: Optional[RequestOption] = Field( 1634 None, 1635 description="Optionally configures how the start value will be sent in requests to the source API.", 1636 title="Inject Start Value Into Outgoing HTTP Request", 1637 ) 1638 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1641class DatetimeBasedCursor(BaseModel): 1642 type: Literal["DatetimeBasedCursor"] 1643 clamping: Optional[Clamping] = Field( 1644 None, 1645 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1646 title="Date Range Clamping", 1647 ) 1648 cursor_field: str = Field( 1649 ..., 1650 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1651 examples=["created_at", "{{ config['record_cursor'] }}"], 1652 title="Cursor Field", 1653 ) 1654 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1655 None, 1656 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1657 title="Allow Catalog Defined Cursor Field", 1658 ) 1659 cursor_datetime_formats: Optional[List[str]] = Field( 1660 None, 1661 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1662 examples=[ 1663 "%Y-%m-%d", 1664 "%Y-%m-%d %H:%M:%S", 1665 "%Y-%m-%dT%H:%M:%S", 1666 "%Y-%m-%dT%H:%M:%SZ", 1667 "%Y-%m-%dT%H:%M:%S%z", 1668 "%Y-%m-%dT%H:%M:%S.%fZ", 1669 "%Y-%m-%dT%H:%M:%S.%f%z", 1670 "%Y-%m-%d %H:%M:%S.%f+00:00", 1671 "%s", 1672 "%ms", 1673 ], 1674 title="Cursor Datetime Formats", 1675 ) 1676 start_datetime: Union[MinMaxDatetime, str] = Field( 1677 ..., 1678 description="The datetime that determines the earliest record that should be synced.", 1679 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1680 title="Start Datetime", 1681 ) 1682 start_time_option: Optional[RequestOption] = Field( 1683 None, 1684 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1685 title="Inject Start Time Into Outgoing HTTP Request", 1686 ) 1687 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1688 None, 1689 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1690 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1691 title="End Datetime", 1692 ) 1693 end_time_option: Optional[RequestOption] = Field( 1694 None, 1695 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1696 title="Inject End Time Into Outgoing HTTP Request", 1697 ) 1698 datetime_format: str = Field( 1699 ..., 1700 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1701 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1702 title="Outgoing Datetime Format", 1703 ) 1704 cursor_granularity: Optional[str] = Field( 1705 None, 1706 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1707 examples=["PT1S"], 1708 title="Cursor Granularity", 1709 ) 1710 is_data_feed: Optional[bool] = Field( 1711 None, 1712 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1713 title="Data Feed API", 1714 ) 1715 is_client_side_incremental: Optional[bool] = Field( 1716 None, 1717 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1718 title="Client-side Incremental Filtering", 1719 ) 1720 is_compare_strictly: Optional[bool] = Field( 1721 False, 1722 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1723 title="Strict Start-End Time Comparison", 1724 ) 1725 global_substream_cursor: Optional[bool] = Field( 1726 False, 1727 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1728 title="Global Substream Cursor", 1729 ) 1730 lookback_window: Optional[str] = Field( 1731 None, 1732 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1733 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1734 title="Lookback Window", 1735 ) 1736 partition_field_end: Optional[str] = Field( 1737 None, 1738 description="Name of the partition start time field.", 1739 examples=["ending_time"], 1740 title="Partition Field End", 1741 ) 1742 partition_field_start: Optional[str] = Field( 1743 None, 1744 description="Name of the partition end time field.", 1745 examples=["starting_time"], 1746 title="Partition Field Start", 1747 ) 1748 step: Optional[str] = Field( 1749 None, 1750 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1751 examples=["P1W", "{{ config['step_increment'] }}"], 1752 title="Step", 1753 ) 1754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1757class JwtAuthenticator(BaseModel): 1758 type: Literal["JwtAuthenticator"] 1759 secret_key: str = Field( 1760 ..., 1761 description="Secret used to sign the JSON web token.", 1762 examples=["{{ config['secret_key'] }}"], 1763 title="Secret Key", 1764 ) 1765 base64_encode_secret_key: Optional[bool] = Field( 1766 False, 1767 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1768 title="Base64-encode Secret Key", 1769 ) 1770 algorithm: Algorithm = Field( 1771 ..., 1772 description="Algorithm used to sign the JSON web token.", 1773 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1774 title="Algorithm", 1775 ) 1776 token_duration: Optional[int] = Field( 1777 1200, 1778 description="The amount of time in seconds a JWT token can be valid after being issued.", 1779 examples=[1200, 3600], 1780 title="Token Duration", 1781 ) 1782 header_prefix: Optional[str] = Field( 1783 None, 1784 description="The prefix to be used within the Authentication header.", 1785 examples=["Bearer", "Basic"], 1786 title="Header Prefix", 1787 ) 1788 jwt_headers: Optional[JwtHeaders] = Field( 1789 None, 1790 description="JWT headers used when signing JSON web token.", 1791 title="JWT Headers", 1792 ) 1793 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1794 None, 1795 description="Additional headers to be included with the JWT headers object.", 1796 title="Additional JWT Headers", 1797 ) 1798 jwt_payload: Optional[JwtPayload] = Field( 1799 None, 1800 description="JWT Payload used when signing JSON web token.", 1801 title="JWT Payload", 1802 ) 1803 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1804 None, 1805 description="Additional properties to be added to the JWT payload.", 1806 title="Additional JWT Payload Properties", 1807 ) 1808 passphrase: Optional[str] = Field( 1809 None, 1810 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1811 examples=["{{ config['passphrase'] }}"], 1812 title="Passphrase", 1813 ) 1814 request_option: Optional[RequestOption] = Field( 1815 None, 1816 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1817 title="Request Option", 1818 ) 1819 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1822class OAuthAuthenticator(BaseModel): 1823 type: Literal["OAuthAuthenticator"] 1824 client_id_name: Optional[str] = Field( 1825 "client_id", 1826 description="The name of the property to use to refresh the `access_token`.", 1827 examples=["custom_app_id"], 1828 title="Client ID Property Name", 1829 ) 1830 client_id: Optional[str] = Field( 1831 None, 1832 description="The OAuth client ID. Fill it in the user inputs.", 1833 examples=[ 1834 "{{ config['client_id'] }}", 1835 "{{ config['credentials']['client_id }}", 1836 ], 1837 title="Client ID", 1838 ) 1839 client_secret_name: Optional[str] = Field( 1840 "client_secret", 1841 description="The name of the property to use to refresh the `access_token`.", 1842 examples=["custom_app_secret"], 1843 title="Client Secret Property Name", 1844 ) 1845 client_secret: Optional[str] = Field( 1846 None, 1847 description="The OAuth client secret. Fill it in the user inputs.", 1848 examples=[ 1849 "{{ config['client_secret'] }}", 1850 "{{ config['credentials']['client_secret }}", 1851 ], 1852 title="Client Secret", 1853 ) 1854 refresh_token_name: Optional[str] = Field( 1855 "refresh_token", 1856 description="The name of the property to use to refresh the `access_token`.", 1857 examples=["custom_app_refresh_value"], 1858 title="Refresh Token Property Name", 1859 ) 1860 refresh_token: Optional[str] = Field( 1861 None, 1862 description="Credential artifact used to get a new access token.", 1863 examples=[ 1864 "{{ config['refresh_token'] }}", 1865 "{{ config['credentials]['refresh_token'] }}", 1866 ], 1867 title="Refresh Token", 1868 ) 1869 token_refresh_endpoint: Optional[str] = Field( 1870 None, 1871 description="The full URL to call to obtain a new access token.", 1872 examples=["https://connect.squareup.com/oauth2/token"], 1873 title="Token Refresh Endpoint", 1874 ) 1875 access_token_name: Optional[str] = Field( 1876 "access_token", 1877 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1878 examples=["access_token"], 1879 title="Access Token Property Name", 1880 ) 1881 access_token_value: Optional[str] = Field( 1882 None, 1883 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1884 examples=["secret_access_token_value"], 1885 title="Access Token Value", 1886 ) 1887 expires_in_name: Optional[str] = Field( 1888 "expires_in", 1889 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1890 examples=["expires_in"], 1891 title="Token Expiry Property Name", 1892 ) 1893 grant_type_name: Optional[str] = Field( 1894 "grant_type", 1895 description="The name of the property to use to refresh the `access_token`.", 1896 examples=["custom_grant_type"], 1897 title="Grant Type Property Name", 1898 ) 1899 grant_type: Optional[str] = Field( 1900 "refresh_token", 1901 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1902 examples=["refresh_token", "client_credentials"], 1903 title="Grant Type", 1904 ) 1905 refresh_request_body: Optional[Dict[str, Any]] = Field( 1906 None, 1907 description="Body of the request sent to get a new access token.", 1908 examples=[ 1909 { 1910 "applicationId": "{{ config['application_id'] }}", 1911 "applicationSecret": "{{ config['application_secret'] }}", 1912 "token": "{{ config['token'] }}", 1913 } 1914 ], 1915 title="Refresh Request Body", 1916 ) 1917 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1918 None, 1919 description="Headers of the request sent to get a new access token.", 1920 examples=[ 1921 { 1922 "Authorization": "<AUTH_TOKEN>", 1923 "Content-Type": "application/x-www-form-urlencoded", 1924 } 1925 ], 1926 title="Refresh Request Headers", 1927 ) 1928 scopes: Optional[List[str]] = Field( 1929 None, 1930 description="List of scopes that should be granted to the access token.", 1931 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1932 title="Scopes", 1933 ) 1934 token_expiry_date: Optional[str] = Field( 1935 None, 1936 description="The access token expiry date.", 1937 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1938 title="Token Expiry Date", 1939 ) 1940 token_expiry_date_format: Optional[str] = Field( 1941 None, 1942 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1943 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1944 title="Token Expiry Date Format", 1945 ) 1946 refresh_token_error_status_codes: Optional[List[int]] = Field( 1947 None, 1948 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1949 examples=[[400, 500]], 1950 title="Refresh Token Error Status Codes", 1951 ) 1952 refresh_token_error_key: Optional[str] = Field( 1953 None, 1954 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1955 examples=["error"], 1956 title="Refresh Token Error Key", 1957 ) 1958 refresh_token_error_values: Optional[List[str]] = Field( 1959 None, 1960 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1961 examples=[["invalid_grant", "invalid_permissions"]], 1962 title="Refresh Token Error Values", 1963 ) 1964 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1965 None, 1966 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1967 title="Refresh Token Updater", 1968 ) 1969 profile_assertion: Optional[JwtAuthenticator] = Field( 1970 None, 1971 description="The authenticator being used to authenticate the client authenticator.", 1972 title="Profile Assertion", 1973 ) 1974 use_profile_assertion: Optional[bool] = Field( 1975 False, 1976 description="Enable using profile assertion as a flow for OAuth authorization.", 1977 title="Use Profile Assertion", 1978 ) 1979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1982class FixedWindowCallRatePolicy(BaseModel): 1983 class Config: 1984 extra = Extra.allow 1985 1986 type: Literal["FixedWindowCallRatePolicy"] 1987 period: str = Field( 1988 ..., description="The time interval for the rate limit window.", title="Period" 1989 ) 1990 call_limit: int = Field( 1991 ..., 1992 description="The maximum number of calls allowed within the period.", 1993 title="Call Limit", 1994 ) 1995 matchers: List[HttpRequestRegexMatcher] = Field( 1996 ..., 1997 description="List of matchers that define which requests this policy applies to.", 1998 title="Matchers", 1999 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2002class MovingWindowCallRatePolicy(BaseModel): 2003 class Config: 2004 extra = Extra.allow 2005 2006 type: Literal["MovingWindowCallRatePolicy"] 2007 rates: List[Rate] = Field( 2008 ..., 2009 description="List of rates that define the call limits for different time intervals.", 2010 title="Rates", 2011 ) 2012 matchers: List[HttpRequestRegexMatcher] = Field( 2013 ..., 2014 description="List of matchers that define which requests this policy applies to.", 2015 title="Matchers", 2016 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2019class UnlimitedCallRatePolicy(BaseModel): 2020 class Config: 2021 extra = Extra.allow 2022 2023 type: Literal["UnlimitedCallRatePolicy"] 2024 matchers: List[HttpRequestRegexMatcher] = Field( 2025 ..., 2026 description="List of matchers that define which requests this policy applies to.", 2027 title="Matchers", 2028 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2031class DefaultErrorHandler(BaseModel): 2032 type: Literal["DefaultErrorHandler"] 2033 backoff_strategies: Optional[ 2034 List[ 2035 Union[ 2036 ConstantBackoffStrategy, 2037 ExponentialBackoffStrategy, 2038 WaitTimeFromHeader, 2039 WaitUntilTimeFromHeader, 2040 CustomBackoffStrategy, 2041 ] 2042 ] 2043 ] = Field( 2044 None, 2045 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2046 title="Backoff Strategies", 2047 ) 2048 max_retries: Optional[int] = Field( 2049 5, 2050 description="The maximum number of time to retry a retryable request before giving up and failing.", 2051 examples=[5, 0, 10], 2052 title="Max Retry Count", 2053 ) 2054 response_filters: Optional[List[HttpResponseFilter]] = Field( 2055 None, 2056 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2057 title="Response Filters", 2058 ) 2059 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2062class DefaultPaginator(BaseModel): 2063 type: Literal["DefaultPaginator"] 2064 pagination_strategy: Union[ 2065 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2066 ] = Field( 2067 ..., 2068 description="Strategy defining how records are paginated.", 2069 title="Pagination Strategy", 2070 ) 2071 page_size_option: Optional[RequestOption] = Field( 2072 None, title="Inject Page Size Into Outgoing HTTP Request" 2073 ) 2074 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2075 None, 2076 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2077 title="Inject Page Token Into Outgoing HTTP Request", 2078 ) 2079 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2082class RecordExpander(BaseModel): 2083 type: Literal["RecordExpander"] 2084 expand_records_from_field: List[str] = Field( 2085 ..., 2086 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2087 examples=[ 2088 ["lines", "data"], 2089 ["items"], 2090 ["nested", "array"], 2091 ["sections", "*", "items"], 2092 ], 2093 title="Expand Records From Field", 2094 ) 2095 remain_original_record: Optional[bool] = Field( 2096 False, 2097 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2098 title="Remain Original Record", 2099 ) 2100 on_no_records: Optional[OnNoRecords] = Field( 2101 OnNoRecords.skip, 2102 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2103 title="On No Records", 2104 ) 2105 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2108class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2109 type: Literal["ApiKey"] 2110 inject_into: RequestOption = Field( 2111 ..., 2112 description="Configure how the API Key will be sent in requests to the source API.", 2113 examples=[ 2114 {"inject_into": "header", "field_name": "Authorization"}, 2115 {"inject_into": "request_parameter", "field_name": "authKey"}, 2116 ], 2117 title="Inject API Key Into Outgoing HTTP Request", 2118 ) 2119 api_token: Optional[str] = Field( 2120 "{{ session_token }}", 2121 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2122 examples=[ 2123 "{{ session_token }}", 2124 "Token {{ session_token }}", 2125 "Bearer {{ session_token }}", 2126 ], 2127 title="API Token Template", 2128 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2131class JsonSchemaPropertySelector(BaseModel): 2132 type: Literal["JsonSchemaPropertySelector"] 2133 transformations: Optional[ 2134 List[ 2135 Union[ 2136 AddFields, 2137 RemoveFields, 2138 KeysToLower, 2139 KeysToSnakeCase, 2140 FlattenFields, 2141 DpathFlattenFields, 2142 KeysReplace, 2143 CustomTransformation, 2144 ] 2145 ] 2146 ] = Field( 2147 None, 2148 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2149 title="Transformations", 2150 ) 2151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2154class ListPartitionRouter(BaseModel): 2155 type: Literal["ListPartitionRouter"] 2156 cursor_field: str = Field( 2157 ..., 2158 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2159 examples=["section", "{{ config['section_key'] }}"], 2160 title="Current Partition Value Identifier", 2161 ) 2162 values: Union[str, List[str]] = Field( 2163 ..., 2164 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2165 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2166 title="Partition Values", 2167 ) 2168 request_option: Optional[RequestOption] = Field( 2169 None, 2170 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2171 title="Inject Partition Value Into Outgoing HTTP Request", 2172 ) 2173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2176class PaginationReset(BaseModel): 2177 type: Literal["PaginationReset"] 2178 action: Action1 2179 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2182class GzipDecoder(BaseModel): 2183 type: Literal["GzipDecoder"] 2184 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2187class RequestBodyGraphQL(BaseModel): 2188 type: Literal["RequestBodyGraphQL"] 2189 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2192class DpathValidator(BaseModel): 2193 type: Literal["DpathValidator"] 2194 field_path: List[str] = Field( 2195 ..., 2196 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2197 examples=[ 2198 ["data"], 2199 ["data", "records"], 2200 ["data", "{{ parameters.name }}"], 2201 ["data", "*", "record"], 2202 ], 2203 title="Field Path", 2204 ) 2205 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2206 ..., 2207 description="The condition that the specified config value will be evaluated against", 2208 title="Validation Strategy", 2209 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2212class PredicateValidator(BaseModel): 2213 type: Literal["PredicateValidator"] 2214 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2215 ..., 2216 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2217 examples=[ 2218 "test-value", 2219 "{{ config['api_version'] }}", 2220 "{{ config['tenant_id'] }}", 2221 123, 2222 ], 2223 title="Value", 2224 ) 2225 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2226 ..., 2227 description="The validation strategy to apply to the value.", 2228 title="Validation Strategy", 2229 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2232class ConfigAddFields(BaseModel): 2233 type: Literal["ConfigAddFields"] 2234 fields: List[AddedFieldDefinition] = Field( 2235 ..., 2236 description="A list of transformations (path and corresponding value) that will be added to the config.", 2237 title="Fields", 2238 ) 2239 condition: Optional[str] = Field( 2240 "", 2241 description="Fields will be added if expression is evaluated to True.", 2242 examples=[ 2243 "{{ config['environemnt'] == 'sandbox' }}", 2244 "{{ property is integer }}", 2245 "{{ property|length > 5 }}", 2246 "{{ property == 'some_string_to_match' }}", 2247 ], 2248 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2251class CompositeErrorHandler(BaseModel): 2252 type: Literal["CompositeErrorHandler"] 2253 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2254 Field( 2255 ..., 2256 description="List of error handlers to iterate on to determine how to handle a failed response.", 2257 title="Error Handlers", 2258 ) 2259 ) 2260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2263class HTTPAPIBudget(BaseModel): 2264 class Config: 2265 extra = Extra.allow 2266 2267 type: Literal["HTTPAPIBudget"] 2268 policies: List[ 2269 Union[ 2270 FixedWindowCallRatePolicy, 2271 MovingWindowCallRatePolicy, 2272 UnlimitedCallRatePolicy, 2273 ] 2274 ] = Field( 2275 ..., 2276 description="List of call rate policies that define how many calls are allowed.", 2277 title="Policies", 2278 ) 2279 ratelimit_reset_header: Optional[str] = Field( 2280 "ratelimit-reset", 2281 description="The HTTP response header name that indicates when the rate limit resets.", 2282 title="Rate Limit Reset Header", 2283 ) 2284 ratelimit_remaining_header: Optional[str] = Field( 2285 "ratelimit-remaining", 2286 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2287 title="Rate Limit Remaining Header", 2288 ) 2289 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2290 [429], 2291 description="List of HTTP status codes that indicate a rate limit has been hit.", 2292 title="Status Codes for Rate Limit Hit", 2293 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2296class DpathExtractor(BaseModel): 2297 type: Literal["DpathExtractor"] 2298 field_path: List[str] = Field( 2299 ..., 2300 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2301 examples=[ 2302 ["data"], 2303 ["data", "records"], 2304 ["data", "{{ parameters.name }}"], 2305 ["data", "*", "record"], 2306 ], 2307 title="Field Path", 2308 ) 2309 record_expander: Optional[RecordExpander] = Field( 2310 None, 2311 description="Optional component to expand records by extracting items from nested array fields.", 2312 title="Record Expander", 2313 ) 2314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2317class ZipfileDecoder(BaseModel): 2318 class Config: 2319 extra = Extra.allow 2320 2321 type: Literal["ZipfileDecoder"] 2322 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2323 ..., 2324 description="Parser to parse the decompressed data from the zipfile(s).", 2325 title="Parser", 2326 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2329class RecordSelector(BaseModel): 2330 type: Literal["RecordSelector"] 2331 extractor: Union[DpathExtractor, CustomRecordExtractor] 2332 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2333 None, 2334 description="Responsible for filtering records to be emitted by the Source.", 2335 title="Record Filter", 2336 ) 2337 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2338 None, 2339 description="Responsible for normalization according to the schema.", 2340 title="Schema Normalization", 2341 ) 2342 transform_before_filtering: Optional[bool] = Field( 2343 None, 2344 description="If true, transformation will be applied before record filtering.", 2345 title="Transform Before Filtering", 2346 ) 2347 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2350class ConfigMigration(BaseModel): 2351 type: Literal["ConfigMigration"] 2352 description: Optional[str] = Field( 2353 None, description="The description/purpose of the config migration." 2354 ) 2355 transformations: List[ 2356 Union[ 2357 ConfigRemapField, 2358 ConfigAddFields, 2359 ConfigRemoveFields, 2360 CustomConfigTransformation, 2361 ] 2362 ] = Field( 2363 ..., 2364 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2365 title="Transformations", 2366 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2369class ConfigNormalizationRules(BaseModel): 2370 class Config: 2371 extra = Extra.forbid 2372 2373 type: Literal["ConfigNormalizationRules"] 2374 config_migrations: Optional[List[ConfigMigration]] = Field( 2375 [], 2376 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2377 title="Config Migrations", 2378 ) 2379 transformations: Optional[ 2380 List[ 2381 Union[ 2382 ConfigRemapField, 2383 ConfigAddFields, 2384 ConfigRemoveFields, 2385 CustomConfigTransformation, 2386 ] 2387 ] 2388 ] = Field( 2389 [], 2390 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2391 title="Transformations", 2392 ) 2393 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2394 [], 2395 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2396 title="Validations", 2397 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2400class Spec(BaseModel): 2401 type: Literal["Spec"] 2402 connection_specification: Dict[str, Any] = Field( 2403 ..., 2404 description="A connection specification describing how a the connector can be configured.", 2405 title="Connection Specification", 2406 ) 2407 documentation_url: Optional[str] = Field( 2408 None, 2409 description="URL of the connector's documentation page.", 2410 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2411 title="Documentation URL", 2412 ) 2413 advanced_auth: Optional[AuthFlow] = Field( 2414 None, 2415 description="Advanced specification for configuring the authentication flow.", 2416 title="Advanced Auth", 2417 ) 2418 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2419 None, title="Config Normalization Rules" 2420 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2423class DeclarativeSource1(BaseModel): 2424 class Config: 2425 extra = Extra.forbid 2426 2427 type: Literal["DeclarativeSource"] 2428 check: Union[CheckStream, CheckDynamicStream] 2429 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2430 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2431 version: str = Field( 2432 ..., 2433 description="The version of the Airbyte CDK used to build and test the source.", 2434 ) 2435 schemas: Optional[Schemas] = None 2436 definitions: Optional[Dict[str, Any]] = None 2437 spec: Optional[Spec] = None 2438 concurrency_level: Optional[ConcurrencyLevel] = None 2439 api_budget: Optional[HTTPAPIBudget] = None 2440 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2441 None, 2442 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2443 title="Stream Groups", 2444 ) 2445 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2446 None, 2447 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2448 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2449 title="Maximum Concurrent Asynchronous Jobs", 2450 ) 2451 metadata: Optional[Dict[str, Any]] = Field( 2452 None, 2453 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2454 ) 2455 description: Optional[str] = Field( 2456 None, 2457 description="A description of the connector. It will be presented on the Source documentation page.", 2458 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2461class DeclarativeSource2(BaseModel): 2462 class Config: 2463 extra = Extra.forbid 2464 2465 type: Literal["DeclarativeSource"] 2466 check: Union[CheckStream, CheckDynamicStream] 2467 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2468 None 2469 ) 2470 dynamic_streams: List[DynamicDeclarativeStream] 2471 version: str = Field( 2472 ..., 2473 description="The version of the Airbyte CDK used to build and test the source.", 2474 ) 2475 schemas: Optional[Schemas] = None 2476 definitions: Optional[Dict[str, Any]] = None 2477 spec: Optional[Spec] = None 2478 concurrency_level: Optional[ConcurrencyLevel] = None 2479 api_budget: Optional[HTTPAPIBudget] = None 2480 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2481 None, 2482 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2483 title="Stream Groups", 2484 ) 2485 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2486 None, 2487 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2488 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2489 title="Maximum Concurrent Asynchronous Jobs", 2490 ) 2491 metadata: Optional[Dict[str, Any]] = Field( 2492 None, 2493 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2494 ) 2495 description: Optional[str] = Field( 2496 None, 2497 description="A description of the connector. It will be presented on the Source documentation page.", 2498 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2501class DeclarativeSource(BaseModel): 2502 class Config: 2503 extra = Extra.forbid 2504 2505 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2506 ..., 2507 description="An API source that extracts data according to its declarative components.", 2508 title="DeclarativeSource", 2509 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2512class SelectiveAuthenticator(BaseModel): 2513 class Config: 2514 extra = Extra.allow 2515 2516 type: Literal["SelectiveAuthenticator"] 2517 authenticator_selection_path: List[str] = Field( 2518 ..., 2519 description="Path of the field in config with selected authenticator name", 2520 examples=[["auth"], ["auth", "type"]], 2521 title="Authenticator Selection Path", 2522 ) 2523 authenticators: Dict[ 2524 str, 2525 Union[ 2526 ApiKeyAuthenticator, 2527 BasicHttpAuthenticator, 2528 BearerAuthenticator, 2529 OAuthAuthenticator, 2530 JwtAuthenticator, 2531 SessionTokenAuthenticator, 2532 LegacySessionTokenAuthenticator, 2533 CustomAuthenticator, 2534 NoAuth, 2535 ], 2536 ] = Field( 2537 ..., 2538 description="Authenticators to select from.", 2539 examples=[ 2540 { 2541 "authenticators": { 2542 "token": "#/definitions/ApiKeyAuthenticator", 2543 "oauth": "#/definitions/OAuthAuthenticator", 2544 "jwt": "#/definitions/JwtAuthenticator", 2545 } 2546 } 2547 ], 2548 title="Authenticators", 2549 ) 2550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2553class ConditionalStreams(BaseModel): 2554 type: Literal["ConditionalStreams"] 2555 condition: str = Field( 2556 ..., 2557 description="Condition that will be evaluated to determine if a set of streams should be available.", 2558 examples=["{{ config['is_sandbox'] }}"], 2559 title="Condition", 2560 ) 2561 streams: List[DeclarativeStream] = Field( 2562 ..., 2563 description="Streams that will be used during an operation based on the condition.", 2564 title="Streams", 2565 ) 2566 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2569class FileUploader(BaseModel): 2570 type: Literal["FileUploader"] 2571 requester: Union[HttpRequester, CustomRequester] = Field( 2572 ..., 2573 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2574 ) 2575 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2576 ..., 2577 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2578 ) 2579 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2580 None, 2581 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2582 ) 2583 filename_extractor: Optional[str] = Field( 2584 None, 2585 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2586 examples=[ 2587 "{{ record.id }}/{{ record.file_name }}/", 2588 "{{ record.id }}_{{ record.file_name }}/", 2589 ], 2590 ) 2591 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2594class DeclarativeStream(BaseModel): 2595 class Config: 2596 extra = Extra.allow 2597 2598 type: Literal["DeclarativeStream"] 2599 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2600 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2601 ..., 2602 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2603 title="Retriever", 2604 ) 2605 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2606 None, 2607 description="Component used to fetch data incrementally based on a time field in the data.", 2608 title="Incremental Sync", 2609 ) 2610 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2611 schema_loader: Optional[ 2612 Union[ 2613 InlineSchemaLoader, 2614 DynamicSchemaLoader, 2615 JsonFileSchemaLoader, 2616 List[ 2617 Union[ 2618 InlineSchemaLoader, 2619 DynamicSchemaLoader, 2620 JsonFileSchemaLoader, 2621 CustomSchemaLoader, 2622 ] 2623 ], 2624 CustomSchemaLoader, 2625 ] 2626 ] = Field( 2627 None, 2628 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2629 title="Schema Loader", 2630 ) 2631 transformations: Optional[ 2632 List[ 2633 Union[ 2634 AddFields, 2635 RemoveFields, 2636 KeysToLower, 2637 KeysToSnakeCase, 2638 FlattenFields, 2639 DpathFlattenFields, 2640 KeysReplace, 2641 CustomTransformation, 2642 ] 2643 ] 2644 ] = Field( 2645 None, 2646 description="A list of transformations to be applied to each output record.", 2647 title="Transformations", 2648 ) 2649 state_migrations: Optional[ 2650 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2651 ] = Field( 2652 [], 2653 description="Array of state migrations to be applied on the input state", 2654 title="State Migrations", 2655 ) 2656 file_uploader: Optional[FileUploader] = Field( 2657 None, 2658 description="(experimental) Describes how to fetch a file", 2659 title="File Uploader", 2660 ) 2661 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2664class SessionTokenAuthenticator(BaseModel): 2665 type: Literal["SessionTokenAuthenticator"] 2666 login_requester: HttpRequester = Field( 2667 ..., 2668 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2669 examples=[ 2670 { 2671 "type": "HttpRequester", 2672 "url_base": "https://my_api.com", 2673 "path": "/login", 2674 "authenticator": { 2675 "type": "BasicHttpAuthenticator", 2676 "username": "{{ config.username }}", 2677 "password": "{{ config.password }}", 2678 }, 2679 } 2680 ], 2681 title="Login Requester", 2682 ) 2683 session_token_path: List[str] = Field( 2684 ..., 2685 description="The path in the response body returned from the login requester to the session token.", 2686 examples=[["access_token"], ["result", "token"]], 2687 title="Session Token Path", 2688 ) 2689 expiration_duration: Optional[str] = Field( 2690 None, 2691 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2692 examples=["PT1H", "P1D"], 2693 title="Expiration Duration", 2694 ) 2695 request_authentication: Union[ 2696 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2697 ] = Field( 2698 ..., 2699 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2700 title="Data Request Authentication", 2701 ) 2702 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2703 None, description="Component used to decode the response.", title="Decoder" 2704 ) 2705 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2708class HttpRequester(BaseModelWithDeprecations): 2709 type: Literal["HttpRequester"] 2710 url_base: Optional[str] = Field( 2711 None, 2712 deprecated=True, 2713 deprecation_message="Use `url` field instead.", 2714 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2715 examples=[ 2716 "https://connect.squareup.com/v2", 2717 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2718 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2719 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2720 ], 2721 title="API Base URL", 2722 ) 2723 url: Optional[str] = Field( 2724 None, 2725 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2726 examples=[ 2727 "https://connect.squareup.com/v2", 2728 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2729 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2730 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2731 ], 2732 title="API Endpoint URL", 2733 ) 2734 path: Optional[str] = Field( 2735 None, 2736 deprecated=True, 2737 deprecation_message="Use `url` field instead.", 2738 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2739 examples=[ 2740 "/products", 2741 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2742 "/trades/{{ config['symbol_id'] }}/history", 2743 ], 2744 title="URL Path", 2745 ) 2746 http_method: Optional[HttpMethod] = Field( 2747 HttpMethod.GET, 2748 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2749 examples=["GET", "POST"], 2750 title="HTTP Method", 2751 ) 2752 authenticator: Optional[ 2753 Union[ 2754 ApiKeyAuthenticator, 2755 BasicHttpAuthenticator, 2756 BearerAuthenticator, 2757 OAuthAuthenticator, 2758 JwtAuthenticator, 2759 SessionTokenAuthenticator, 2760 SelectiveAuthenticator, 2761 CustomAuthenticator, 2762 NoAuth, 2763 LegacySessionTokenAuthenticator, 2764 ] 2765 ] = Field( 2766 None, 2767 description="Authentication method to use for requests sent to the API.", 2768 title="Authenticator", 2769 ) 2770 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2771 None, 2772 deprecated=True, 2773 deprecation_message="Use `query_properties` field instead.", 2774 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2775 title="Fetch Properties from Endpoint", 2776 ) 2777 query_properties: Optional[QueryProperties] = Field( 2778 None, 2779 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2780 title="Query Properties", 2781 ) 2782 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2783 None, 2784 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2785 examples=[ 2786 {"unit": "day"}, 2787 { 2788 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2789 }, 2790 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2791 {"sort_by[asc]": "updated_at"}, 2792 ], 2793 title="Query Parameters", 2794 ) 2795 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2796 None, 2797 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2798 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2799 title="Request Headers", 2800 ) 2801 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2802 None, 2803 deprecated=True, 2804 deprecation_message="Use `request_body` field instead.", 2805 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2806 examples=[ 2807 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2808 ], 2809 title="Request Body Payload (Non-JSON)", 2810 ) 2811 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2812 None, 2813 deprecated=True, 2814 deprecation_message="Use `request_body` field instead.", 2815 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2816 examples=[ 2817 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2818 {"key": "{{ config['value'] }}"}, 2819 {"sort": {"field": "updated_at", "order": "ascending"}}, 2820 ], 2821 title="Request Body JSON Payload", 2822 ) 2823 request_body: Optional[ 2824 Union[ 2825 RequestBodyPlainText, 2826 RequestBodyUrlEncodedForm, 2827 RequestBodyJsonObject, 2828 RequestBodyGraphQL, 2829 ] 2830 ] = Field( 2831 None, 2832 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2833 title="Request Body", 2834 ) 2835 error_handler: Optional[ 2836 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2837 ] = Field( 2838 None, 2839 description="Error handler component that defines how to handle errors.", 2840 title="Error Handler", 2841 ) 2842 use_cache: Optional[bool] = Field( 2843 False, 2844 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2845 title="Use Cache", 2846 ) 2847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2850class DynamicSchemaLoader(BaseModel): 2851 type: Literal["DynamicSchemaLoader"] 2852 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2853 ..., 2854 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2855 title="Retriever", 2856 ) 2857 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2858 None, 2859 description="Responsible for filtering fields to be added to json schema.", 2860 title="Schema Filter", 2861 ) 2862 schema_transformations: Optional[ 2863 List[ 2864 Union[ 2865 AddFields, 2866 RemoveFields, 2867 KeysToLower, 2868 KeysToSnakeCase, 2869 FlattenFields, 2870 DpathFlattenFields, 2871 KeysReplace, 2872 CustomTransformation, 2873 ] 2874 ] 2875 ] = Field( 2876 None, 2877 description="A list of transformations to be applied to the schema.", 2878 title="Schema Transformations", 2879 ) 2880 schema_type_identifier: SchemaTypeIdentifier 2881 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2884class ParentStreamConfig(BaseModel): 2885 type: Literal["ParentStreamConfig"] 2886 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2887 ..., description="Reference to the parent stream.", title="Parent Stream" 2888 ) 2889 parent_key: str = Field( 2890 ..., 2891 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2892 examples=["id", "{{ config['parent_record_id'] }}"], 2893 title="Parent Key", 2894 ) 2895 partition_field: str = Field( 2896 ..., 2897 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2898 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2899 title="Current Parent Key Value Identifier", 2900 ) 2901 request_option: Optional[RequestOption] = Field( 2902 None, 2903 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2904 title="Request Option", 2905 ) 2906 incremental_dependency: Optional[bool] = Field( 2907 False, 2908 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2909 title="Incremental Dependency", 2910 ) 2911 lazy_read_pointer: Optional[List[str]] = Field( 2912 [], 2913 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2914 title="Lazy Read Pointer", 2915 ) 2916 extra_fields: Optional[List[List[str]]] = Field( 2917 None, 2918 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2919 title="Extra Fields", 2920 ) 2921 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2924class PropertiesFromEndpoint(BaseModel): 2925 type: Literal["PropertiesFromEndpoint"] 2926 property_field_path: List[str] = Field( 2927 ..., 2928 description="Describes the path to the field that should be extracted", 2929 examples=[["name"]], 2930 ) 2931 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2932 ..., 2933 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2934 ) 2935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2938class QueryProperties(BaseModel): 2939 type: Literal["QueryProperties"] 2940 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2941 ..., 2942 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2943 title="Property List", 2944 ) 2945 always_include_properties: Optional[List[str]] = Field( 2946 None, 2947 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2948 title="Always Include Properties", 2949 ) 2950 property_chunking: Optional[PropertyChunking] = Field( 2951 None, 2952 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2953 title="Property Chunking", 2954 ) 2955 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2956 None, 2957 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2958 title="Property Selector", 2959 ) 2960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2963class StateDelegatingStream(BaseModel): 2964 type: Literal["StateDelegatingStream"] 2965 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2966 full_refresh_stream: DeclarativeStream = Field( 2967 ..., 2968 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2969 title="Full Refresh Stream", 2970 ) 2971 incremental_stream: DeclarativeStream = Field( 2972 ..., 2973 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2974 title="Incremental Stream", 2975 ) 2976 api_retention_period: Optional[str] = Field( 2977 None, 2978 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2979 examples=["P30D", "P90D", "P1Y"], 2980 title="API Retention Period", 2981 ) 2982 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2985class SimpleRetriever(BaseModel): 2986 type: Literal["SimpleRetriever"] 2987 requester: Union[HttpRequester, CustomRequester] = Field( 2988 ..., 2989 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2990 ) 2991 decoder: Optional[ 2992 Union[ 2993 JsonDecoder, 2994 XmlDecoder, 2995 CsvDecoder, 2996 JsonlDecoder, 2997 GzipDecoder, 2998 IterableDecoder, 2999 ZipfileDecoder, 3000 CustomDecoder, 3001 ] 3002 ] = Field( 3003 None, 3004 description="Component decoding the response so records can be extracted.", 3005 title="HTTP Response Format", 3006 ) 3007 record_selector: RecordSelector = Field( 3008 ..., 3009 description="Component that describes how to extract records from a HTTP response.", 3010 ) 3011 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3012 None, 3013 description="Paginator component that describes how to navigate through the API's pages.", 3014 ) 3015 pagination_reset: Optional[PaginationReset] = Field( 3016 None, 3017 description="Describes what triggers pagination reset and how to handle it.", 3018 ) 3019 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3020 False, 3021 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3022 ) 3023 partition_router: Optional[ 3024 Union[ 3025 SubstreamPartitionRouter, 3026 ListPartitionRouter, 3027 GroupingPartitionRouter, 3028 CustomPartitionRouter, 3029 List[ 3030 Union[ 3031 SubstreamPartitionRouter, 3032 ListPartitionRouter, 3033 GroupingPartitionRouter, 3034 CustomPartitionRouter, 3035 ] 3036 ], 3037 ] 3038 ] = Field( 3039 None, 3040 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3041 title="Partition Router", 3042 ) 3043 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3046class AsyncRetriever(BaseModel): 3047 type: Literal["AsyncRetriever"] 3048 record_selector: RecordSelector = Field( 3049 ..., 3050 description="Component that describes how to extract records from a HTTP response.", 3051 ) 3052 status_mapping: AsyncJobStatusMap = Field( 3053 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3054 ) 3055 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3056 ..., description="Responsible for fetching the actual status of the async job." 3057 ) 3058 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3059 None, 3060 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3061 ) 3062 download_extractor: Optional[ 3063 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3064 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3065 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3066 ..., 3067 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3068 ) 3069 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3070 ..., 3071 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3072 ) 3073 polling_job_timeout: Optional[Union[int, str]] = Field( 3074 None, 3075 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3076 ) 3077 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3078 None, 3079 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3080 ) 3081 download_requester: Union[HttpRequester, CustomRequester] = Field( 3082 ..., 3083 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3084 ) 3085 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3086 None, 3087 description="Paginator component that describes how to navigate through the API's pages during download.", 3088 ) 3089 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3090 None, 3091 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3092 ) 3093 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3094 None, 3095 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3096 ) 3097 partition_router: Optional[ 3098 Union[ 3099 ListPartitionRouter, 3100 SubstreamPartitionRouter, 3101 GroupingPartitionRouter, 3102 CustomPartitionRouter, 3103 List[ 3104 Union[ 3105 ListPartitionRouter, 3106 SubstreamPartitionRouter, 3107 GroupingPartitionRouter, 3108 CustomPartitionRouter, 3109 ] 3110 ], 3111 ] 3112 ] = Field( 3113 [], 3114 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3115 title="Partition Router", 3116 ) 3117 decoder: Optional[ 3118 Union[ 3119 CsvDecoder, 3120 GzipDecoder, 3121 JsonDecoder, 3122 JsonlDecoder, 3123 IterableDecoder, 3124 XmlDecoder, 3125 ZipfileDecoder, 3126 CustomDecoder, 3127 ] 3128 ] = Field( 3129 None, 3130 description="Component decoding the response so records can be extracted.", 3131 title="HTTP Response Format", 3132 ) 3133 download_decoder: Optional[ 3134 Union[ 3135 CsvDecoder, 3136 GzipDecoder, 3137 JsonDecoder, 3138 JsonlDecoder, 3139 IterableDecoder, 3140 XmlDecoder, 3141 ZipfileDecoder, 3142 CustomDecoder, 3143 ] 3144 ] = Field( 3145 None, 3146 description="Component decoding the download response so records can be extracted.", 3147 title="Download HTTP Response Format", 3148 ) 3149 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3156class StreamGroup(BaseModel): 3157 streams: List[str] = Field( 3158 ..., 3159 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3160 title="Streams", 3161 ) 3162 action: BlockSimultaneousSyncsAction = Field( 3163 ..., 3164 description="The action to apply to streams in this group.", 3165 title="Action", 3166 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3169class SubstreamPartitionRouter(BaseModel): 3170 type: Literal["SubstreamPartitionRouter"] 3171 parent_stream_configs: List[ParentStreamConfig] = Field( 3172 ..., 3173 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3174 title="Parent Stream Configs", 3175 ) 3176 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3179class GroupingPartitionRouter(BaseModel): 3180 type: Literal["GroupingPartitionRouter"] 3181 group_size: int = Field( 3182 ..., 3183 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3184 examples=[10, 50], 3185 title="Group Size", 3186 ) 3187 underlying_partition_router: Union[ 3188 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3189 ] = Field( 3190 ..., 3191 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3192 title="Underlying Partition Router", 3193 ) 3194 deduplicate: Optional[bool] = Field( 3195 True, 3196 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3197 title="Deduplicate Partitions", 3198 ) 3199 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3202class HttpComponentsResolver(BaseModel): 3203 type: Literal["HttpComponentsResolver"] 3204 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3205 ..., 3206 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3207 title="Retriever", 3208 ) 3209 components_mapping: List[ComponentMappingDefinition] 3210 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3213class DynamicDeclarativeStream(BaseModel): 3214 type: Literal["DynamicDeclarativeStream"] 3215 name: Optional[str] = Field( 3216 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3217 ) 3218 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3219 ..., description="Reference to the stream template.", title="Stream Template" 3220 ) 3221 components_resolver: Union[ 3222 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3223 ] = Field( 3224 ..., 3225 description="Component resolve and populates stream templates with components values.", 3226 title="Components Resolver", 3227 ) 3228 use_parent_parameters: Optional[bool] = Field( 3229 True, 3230 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3231 title="Use Parent Parameters", 3232 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]