airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 jitter_range_in_seconds: Optional[float] = Field( 108 None, 109 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 110 examples=[15], 111 ge=0, 112 title="Jitter Range", 113 ) 114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 115 116 117class CursorPagination(BaseModel): 118 type: Literal["CursorPagination"] 119 cursor_value: str = Field( 120 ..., 121 description="Value of the cursor defining the next page to fetch.", 122 examples=[ 123 "{{ headers.link.next.cursor }}", 124 "{{ last_record['key'] }}", 125 "{{ response['nextPage'] }}", 126 ], 127 title="Cursor Value", 128 ) 129 page_size: Optional[Union[int, str]] = Field( 130 None, 131 description="The number of records to include in each pages.", 132 examples=[100, "{{ config['page_size'] }}"], 133 title="Page Size", 134 ) 135 stop_condition: Optional[str] = Field( 136 None, 137 description="Template string evaluating when to stop paginating.", 138 examples=[ 139 "{{ response.data.has_more is false }}", 140 "{{ 'next' not in headers['link'] }}", 141 ], 142 title="Stop Condition", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomAuthenticator(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomAuthenticator"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomBackoffStrategy(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomBackoffStrategy"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomBackoffStrategy"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomErrorHandler(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomErrorHandler"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomErrorHandler"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomPaginationStrategy(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomPaginationStrategy"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomPaginationStrategy"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordExtractor(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordExtractor"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomRecordExtractor"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRecordFilter(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRecordFilter"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomCustomRecordFilter"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRequester(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRequester"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRecordExtractor"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomRetriever(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomRetriever"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomRetriever"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomPartitionRouter(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomPartitionRouter"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomPartitionRouter"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaLoader(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaLoader"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 281 examples=["source_railz.components.MyCustomSchemaLoader"], 282 title="Class Name", 283 ) 284 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 285 286 287class CustomSchemaNormalization(BaseModel): 288 class Config: 289 extra = Extra.allow 290 291 type: Literal["CustomSchemaNormalization"] 292 class_name: str = Field( 293 ..., 294 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 295 examples=[ 296 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 297 ], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomStateMigration(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomStateMigration"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomStateMigration"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class CustomTransformation(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Literal["CustomTransformation"] 322 class_name: str = Field( 323 ..., 324 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 325 examples=["source_railz.components.MyCustomTransformation"], 326 title="Class Name", 327 ) 328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 329 330 331class LegacyToPerPartitionStateMigration(BaseModel): 332 class Config: 333 extra = Extra.allow 334 335 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 336 337 338class Clamping(BaseModel): 339 target: str = Field( 340 ..., 341 description="The period of time that datetime windows will be clamped by", 342 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 343 title="Target", 344 ) 345 target_details: Optional[Dict[str, Any]] = None 346 347 348class Algorithm(Enum): 349 HS256 = "HS256" 350 HS384 = "HS384" 351 HS512 = "HS512" 352 ES256 = "ES256" 353 ES256K = "ES256K" 354 ES384 = "ES384" 355 ES512 = "ES512" 356 RS256 = "RS256" 357 RS384 = "RS384" 358 RS512 = "RS512" 359 PS256 = "PS256" 360 PS384 = "PS384" 361 PS512 = "PS512" 362 EdDSA = "EdDSA" 363 364 365class JwtHeaders(BaseModel): 366 class Config: 367 extra = Extra.forbid 368 369 kid: Optional[str] = Field( 370 None, 371 description="Private key ID for user account.", 372 examples=["{{ config['kid'] }}"], 373 title="Key Identifier", 374 ) 375 typ: Optional[str] = Field( 376 "JWT", 377 description="The media type of the complete JWT.", 378 examples=["JWT"], 379 title="Type", 380 ) 381 cty: Optional[str] = Field( 382 None, 383 description="Content type of JWT header.", 384 examples=["JWT"], 385 title="Content Type", 386 ) 387 388 389class JwtPayload(BaseModel): 390 class Config: 391 extra = Extra.forbid 392 393 iss: Optional[str] = Field( 394 None, 395 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 396 examples=["{{ config['iss'] }}"], 397 title="Issuer", 398 ) 399 sub: Optional[str] = Field( 400 None, 401 description="The subject of the JWT. Commonly defined by the API.", 402 title="Subject", 403 ) 404 aud: Optional[str] = Field( 405 None, 406 description="The recipient that the JWT is intended for. Commonly defined by the API.", 407 examples=["appstoreconnect-v1"], 408 title="Audience", 409 ) 410 411 412class RefreshTokenUpdater(BaseModel): 413 refresh_token_name: Optional[str] = Field( 414 "refresh_token", 415 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 416 examples=["refresh_token"], 417 title="Refresh Token Property Name", 418 ) 419 access_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "access_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "access_token"], ["access_token"]], 423 title="Config Path To Access Token", 424 ) 425 refresh_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "refresh_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "refresh_token"], ["refresh_token"]], 429 title="Config Path To Refresh Token", 430 ) 431 token_expiry_date_config_path: Optional[List[str]] = Field( 432 ["credentials", "token_expiry_date"], 433 description="Config path to the expiry date. Make sure actually exists in the config.", 434 examples=[["credentials", "token_expiry_date"]], 435 title="Config Path To Expiry Date", 436 ) 437 refresh_token_error_status_codes: Optional[List[int]] = Field( 438 [], 439 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 440 examples=[[400, 500]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 442 ) 443 refresh_token_error_key: Optional[str] = Field( 444 "", 445 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 446 examples=["error"], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 448 ) 449 refresh_token_error_values: Optional[List[str]] = Field( 450 [], 451 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 452 examples=[["invalid_grant", "invalid_permissions"]], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 454 ) 455 456 457class Rate(BaseModel): 458 class Config: 459 extra = Extra.allow 460 461 limit: Union[int, str] = Field( 462 ..., 463 description="The maximum number of calls allowed within the interval.", 464 title="Limit", 465 ) 466 interval: str = Field( 467 ..., 468 description="The time interval for the rate limit.", 469 examples=["PT1H", "P1D"], 470 title="Interval", 471 ) 472 473 474class HttpRequestRegexMatcher(BaseModel): 475 class Config: 476 extra = Extra.allow 477 478 method: Optional[str] = Field( 479 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 480 ) 481 url_base: Optional[str] = Field( 482 None, 483 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 484 title="URL Base", 485 ) 486 url_path_pattern: Optional[str] = Field( 487 None, 488 description="A regular expression pattern to match the URL path.", 489 title="URL Path Pattern", 490 ) 491 params: Optional[Dict[str, Any]] = Field( 492 None, description="The query parameters to match.", title="Parameters" 493 ) 494 headers: Optional[Dict[str, Any]] = Field( 495 None, description="The headers to match.", title="Headers" 496 ) 497 weight: Optional[Union[int, str]] = Field( 498 None, 499 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 500 title="Weight", 501 ) 502 503 504class ResponseToFileExtractor(BaseModel): 505 type: Literal["ResponseToFileExtractor"] 506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 507 508 509class OnNoRecords(Enum): 510 skip = "skip" 511 emit_parent = "emit_parent" 512 513 514class ExponentialBackoffStrategy(BaseModel): 515 type: Literal["ExponentialBackoffStrategy"] 516 factor: Optional[Union[float, str]] = Field( 517 5, 518 description="Multiplicative constant applied on each retry.", 519 examples=[5, 5.5, "10"], 520 title="Factor", 521 ) 522 jitter_range_in_seconds: Optional[float] = Field( 523 None, 524 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 525 examples=[2], 526 ge=0, 527 title="Jitter Range", 528 ) 529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 530 531 532class GroupByKeyMergeStrategy(BaseModel): 533 type: Literal["GroupByKeyMergeStrategy"] 534 key: Union[str, List[str]] = Field( 535 ..., 536 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 537 examples=["id", ["parent_id", "end_date"]], 538 title="Key", 539 ) 540 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 541 542 543class SessionTokenRequestBearerAuthenticator(BaseModel): 544 type: Literal["Bearer"] 545 546 547class HttpMethod(Enum): 548 GET = "GET" 549 POST = "POST" 550 551 552class Action(Enum): 553 SUCCESS = "SUCCESS" 554 FAIL = "FAIL" 555 RETRY = "RETRY" 556 IGNORE = "IGNORE" 557 RESET_PAGINATION = "RESET_PAGINATION" 558 RATE_LIMITED = "RATE_LIMITED" 559 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 560 561 562class FailureType(Enum): 563 system_error = "system_error" 564 config_error = "config_error" 565 transient_error = "transient_error" 566 567 568class HttpResponseFilter(BaseModel): 569 type: Literal["HttpResponseFilter"] 570 action: Optional[Action] = Field( 571 None, 572 description="Action to execute if a response matches the filter.", 573 examples=[ 574 "SUCCESS", 575 "FAIL", 576 "RETRY", 577 "IGNORE", 578 "RESET_PAGINATION", 579 "RATE_LIMITED", 580 "REFRESH_TOKEN_THEN_RETRY", 581 ], 582 title="Action", 583 ) 584 failure_type: Optional[FailureType] = Field( 585 None, 586 description="Failure type of traced exception if a response matches the filter.", 587 examples=["system_error", "config_error", "transient_error"], 588 title="Failure Type", 589 ) 590 error_message: Optional[str] = Field( 591 None, 592 description="Error Message to display if the response matches the filter.", 593 title="Error Message", 594 ) 595 error_message_contains: Optional[str] = Field( 596 None, 597 description="Match the response if its error message contains the substring.", 598 example=["This API operation is not enabled for this site"], 599 title="Error Message Substring", 600 ) 601 http_codes: Optional[List[int]] = Field( 602 None, 603 description="Match the response if its HTTP code is included in this list.", 604 examples=[[420, 429], [500]], 605 title="HTTP Codes", 606 unique_items=True, 607 ) 608 predicate: Optional[str] = Field( 609 None, 610 description="Match the response if the predicate evaluates to true.", 611 examples=[ 612 "{{ 'Too much requests' in response }}", 613 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 614 ], 615 title="Predicate", 616 ) 617 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 618 619 620class ComplexFieldType(BaseModel): 621 field_type: str 622 items: Optional[Union[str, ComplexFieldType]] = None 623 624 625class TypesMap(BaseModel): 626 target_type: Union[str, List[str], ComplexFieldType] 627 current_type: Union[str, List[str]] 628 condition: Optional[str] = None 629 630 631class SchemaTypeIdentifier(BaseModel): 632 type: Optional[Literal["SchemaTypeIdentifier"]] = None 633 schema_pointer: Optional[List[str]] = Field( 634 [], 635 description="List of nested fields defining the schema field path to extract. Defaults to [].", 636 title="Schema Path", 637 ) 638 key_pointer: List[str] = Field( 639 ..., 640 description="List of potentially nested fields describing the full path of the field key to extract.", 641 title="Key Path", 642 ) 643 type_pointer: Optional[List[str]] = Field( 644 None, 645 description="List of potentially nested fields describing the full path of the field type to extract.", 646 title="Type Path", 647 ) 648 types_mapping: Optional[List[TypesMap]] = None 649 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 650 651 652class InlineSchemaLoader(BaseModel): 653 type: Literal["InlineSchemaLoader"] 654 schema_: Optional[Dict[str, Any]] = Field( 655 None, 656 alias="schema", 657 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 658 title="Schema", 659 ) 660 661 662class JsonFileSchemaLoader(BaseModel): 663 type: Literal["JsonFileSchemaLoader"] 664 file_path: Optional[str] = Field( 665 None, 666 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 667 example=["./schemas/users.json"], 668 title="File Path", 669 ) 670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 671 672 673class JsonDecoder(BaseModel): 674 type: Literal["JsonDecoder"] 675 676 677class JsonlDecoder(BaseModel): 678 type: Literal["JsonlDecoder"] 679 680 681class KeysToLower(BaseModel): 682 type: Literal["KeysToLower"] 683 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 684 685 686class KeysToSnakeCase(BaseModel): 687 type: Literal["KeysToSnakeCase"] 688 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 689 690 691class FlattenFields(BaseModel): 692 type: Literal["FlattenFields"] 693 flatten_lists: Optional[bool] = Field( 694 True, 695 description="Whether to flatten lists or leave it as is. Default is True.", 696 title="Flatten Lists", 697 ) 698 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 699 700 701class KeyTransformation(BaseModel): 702 type: Literal["KeyTransformation"] 703 prefix: Optional[str] = Field( 704 None, 705 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 706 examples=["flattened_"], 707 title="Key Prefix", 708 ) 709 suffix: Optional[str] = Field( 710 None, 711 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 712 examples=["_flattened"], 713 title="Key Suffix", 714 ) 715 716 717class DpathFlattenFields(BaseModel): 718 type: Literal["DpathFlattenFields"] 719 field_path: List[str] = Field( 720 ..., 721 description="A path to field that needs to be flattened.", 722 examples=[["data"], ["data", "*", "field"]], 723 title="Field Path", 724 ) 725 delete_origin_value: Optional[bool] = Field( 726 None, 727 description="Whether to delete the origin value or keep it. Default is False.", 728 title="Delete Origin Value", 729 ) 730 replace_record: Optional[bool] = Field( 731 None, 732 description="Whether to replace the origin record or not. Default is False.", 733 title="Replace Origin Record", 734 ) 735 key_transformation: Optional[KeyTransformation] = Field( 736 None, 737 description="Transformation for object keys. If not provided, original key will be used.", 738 title="Key transformation", 739 ) 740 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 741 742 743class KeysReplace(BaseModel): 744 type: Literal["KeysReplace"] 745 old: str = Field( 746 ..., 747 description="Old value to replace.", 748 examples=[ 749 " ", 750 "{{ record.id }}", 751 "{{ config['id'] }}", 752 "{{ stream_slice['id'] }}", 753 ], 754 title="Old value", 755 ) 756 new: str = Field( 757 ..., 758 description="New value to set.", 759 examples=[ 760 "_", 761 "{{ record.id }}", 762 "{{ config['id'] }}", 763 "{{ stream_slice['id'] }}", 764 ], 765 title="New value", 766 ) 767 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 768 769 770class IterableDecoder(BaseModel): 771 type: Literal["IterableDecoder"] 772 773 774class XmlDecoder(BaseModel): 775 type: Literal["XmlDecoder"] 776 777 778class CustomDecoder(BaseModel): 779 class Config: 780 extra = Extra.allow 781 782 type: Literal["CustomDecoder"] 783 class_name: str = Field( 784 ..., 785 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 786 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 787 title="Class Name", 788 ) 789 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 790 791 792class MinMaxDatetime(BaseModel): 793 type: Literal["MinMaxDatetime"] 794 datetime: str = Field( 795 ..., 796 description="Datetime value.", 797 examples=[ 798 "2021-01-01", 799 "2021-01-01T00:00:00Z", 800 "{{ config['start_time'] }}", 801 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 802 ], 803 title="Datetime", 804 ) 805 datetime_format: Optional[str] = Field( 806 "", 807 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 808 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 809 title="Datetime Format", 810 ) 811 max_datetime: Optional[str] = Field( 812 None, 813 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 814 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 815 title="Max Datetime", 816 ) 817 min_datetime: Optional[str] = Field( 818 None, 819 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 820 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 821 title="Min Datetime", 822 ) 823 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 824 825 826class NoAuth(BaseModel): 827 type: Literal["NoAuth"] 828 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 829 830 831class NoPagination(BaseModel): 832 type: Literal["NoPagination"] 833 834 835class State(BaseModel): 836 class Config: 837 extra = Extra.allow 838 839 min: int 840 max: int 841 842 843class OAuthScope(BaseModel): 844 class Config: 845 extra = Extra.allow 846 847 scope: str = Field( 848 ..., 849 description="The OAuth scope string to request from the provider.", 850 ) 851 852 853class OauthConnectorInputSpecification(BaseModel): 854 class Config: 855 extra = Extra.allow 856 857 consent_url: str = Field( 858 ..., 859 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 860 examples=[ 861 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 862 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 863 ], 864 title="Consent URL", 865 ) 866 scope: Optional[str] = Field( 867 None, 868 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 869 examples=["user:read user:read_orders workspaces:read"], 870 title="Scopes", 871 ) 872 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 873 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 874 # The CDK schema defines the manifest contract; the platform reads these fields 875 # during the OAuth consent flow to build the authorization URL. 876 scopes: Optional[List[OAuthScope]] = Field( 877 None, 878 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 879 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 880 title="Scopes", 881 ) 882 optional_scopes: Optional[List[OAuthScope]] = Field( 883 None, 884 description="Optional OAuth scope objects that may or may not be granted.", 885 examples=[[{"scope": "admin:read"}]], 886 title="Optional Scopes", 887 ) 888 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 889 ScopesJoinStrategy.space, 890 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 891 title="Scopes Join Strategy", 892 ) 893 access_token_url: str = Field( 894 ..., 895 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 896 examples=[ 897 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 898 ], 899 title="Access Token URL", 900 ) 901 access_token_headers: Optional[Dict[str, Any]] = Field( 902 None, 903 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 904 examples=[ 905 { 906 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 907 } 908 ], 909 title="Access Token Headers", 910 ) 911 access_token_params: Optional[Dict[str, Any]] = Field( 912 None, 913 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 914 examples=[ 915 { 916 "{{ auth_code_key }}": "{{ auth_code_value }}", 917 "{{ client_id_key }}": "{{ client_id_value }}", 918 "{{ client_secret_key }}": "{{ client_secret_value }}", 919 } 920 ], 921 title="Access Token Query Params (Json Encoded)", 922 ) 923 extract_output: Optional[List[str]] = Field( 924 None, 925 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 926 examples=[["access_token", "refresh_token", "other_field"]], 927 title="Extract Output", 928 ) 929 state: Optional[State] = Field( 930 None, 931 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 932 examples=[{"min": 7, "max": 128}], 933 title="Configurable State Query Param", 934 ) 935 client_id_key: Optional[str] = Field( 936 None, 937 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 938 examples=["my_custom_client_id_key_name"], 939 title="Client ID Key Override", 940 ) 941 client_secret_key: Optional[str] = Field( 942 None, 943 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 944 examples=["my_custom_client_secret_key_name"], 945 title="Client Secret Key Override", 946 ) 947 scope_key: Optional[str] = Field( 948 None, 949 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 950 examples=["my_custom_scope_key_key_name"], 951 title="Scopes Key Override", 952 ) 953 state_key: Optional[str] = Field( 954 None, 955 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 956 examples=["my_custom_state_key_key_name"], 957 title="State Key Override", 958 ) 959 auth_code_key: Optional[str] = Field( 960 None, 961 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 962 examples=["my_custom_auth_code_key_name"], 963 title="Auth Code Key Override", 964 ) 965 redirect_uri_key: Optional[str] = Field( 966 None, 967 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 968 examples=["my_custom_redirect_uri_key_name"], 969 title="Redirect URI Key Override", 970 ) 971 972 973class OAuthConfigSpecification(BaseModel): 974 class Config: 975 extra = Extra.allow 976 977 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 978 None, 979 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 980 examples=[ 981 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 982 { 983 "app_id": { 984 "type": "string", 985 "path_in_connector_config": ["info", "app_id"], 986 } 987 }, 988 ], 989 title="OAuth user input", 990 ) 991 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 992 None, 993 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 994 title="DeclarativeOAuth Connector Specification", 995 ) 996 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 997 None, 998 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 999 examples=[ 1000 { 1001 "refresh_token": { 1002 "type": "string,", 1003 "path_in_connector_config": ["credentials", "refresh_token"], 1004 } 1005 } 1006 ], 1007 title="OAuth output specification", 1008 ) 1009 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1010 None, 1011 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1012 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1013 title="OAuth input specification", 1014 ) 1015 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1016 None, 1017 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1018 examples=[ 1019 { 1020 "client_id": { 1021 "type": "string,", 1022 "path_in_connector_config": ["credentials", "client_id"], 1023 }, 1024 "client_secret": { 1025 "type": "string,", 1026 "path_in_connector_config": ["credentials", "client_secret"], 1027 }, 1028 } 1029 ], 1030 title="OAuth server output specification", 1031 ) 1032 1033 1034class OffsetIncrement(BaseModel): 1035 type: Literal["OffsetIncrement"] 1036 page_size: Optional[Union[int, str]] = Field( 1037 None, 1038 description="The number of records to include in each pages.", 1039 examples=[100, "{{ config['page_size'] }}"], 1040 title="Limit", 1041 ) 1042 inject_on_first_request: Optional[bool] = Field( 1043 False, 1044 description="Using the `offset` with value `0` during the first request", 1045 title="Inject Offset on First Request", 1046 ) 1047 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1048 1049 1050class PageIncrement(BaseModel): 1051 type: Literal["PageIncrement"] 1052 page_size: Optional[Union[int, str]] = Field( 1053 None, 1054 description="The number of records to include in each pages.", 1055 examples=[100, "100", "{{ config['page_size'] }}"], 1056 title="Page Size", 1057 ) 1058 start_from_page: Optional[int] = Field( 1059 0, 1060 description="Index of the first page to request.", 1061 examples=[0, 1], 1062 title="Start From Page", 1063 ) 1064 inject_on_first_request: Optional[bool] = Field( 1065 False, 1066 description="Using the `page number` with value defined by `start_from_page` during the first request", 1067 title="Inject Page Number on First Request", 1068 ) 1069 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1070 1071 1072class PrimaryKey(BaseModel): 1073 __root__: Union[str, List[str], List[List[str]]] = Field( 1074 ..., 1075 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1076 examples=["id", ["code", "type"]], 1077 title="Primary Key", 1078 ) 1079 1080 1081class PropertyLimitType(Enum): 1082 characters = "characters" 1083 property_count = "property_count" 1084 1085 1086class PropertyChunking(BaseModel): 1087 type: Literal["PropertyChunking"] 1088 property_limit_type: PropertyLimitType = Field( 1089 ..., 1090 description="The type used to determine the maximum number of properties per chunk", 1091 title="Property Limit Type", 1092 ) 1093 property_limit: Optional[int] = Field( 1094 None, 1095 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1096 title="Property Limit", 1097 ) 1098 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1099 None, 1100 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1101 title="Record Merge Strategy", 1102 ) 1103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1104 1105 1106class RecordFilter(BaseModel): 1107 type: Literal["RecordFilter"] 1108 condition: Optional[str] = Field( 1109 "", 1110 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1111 examples=[ 1112 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1113 "{{ record.status in ['active', 'expired'] }}", 1114 ], 1115 title="Condition", 1116 ) 1117 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1118 1119 1120class SchemaNormalization(Enum): 1121 Default = "Default" 1122 None_ = "None" 1123 1124 1125class RemoveFields(BaseModel): 1126 type: Literal["RemoveFields"] 1127 condition: Optional[str] = Field( 1128 "", 1129 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1130 examples=[ 1131 "{{ property|string == '' }}", 1132 "{{ property is integer }}", 1133 "{{ property|length > 5 }}", 1134 "{{ property == 'some_string_to_match' }}", 1135 ], 1136 ) 1137 field_pointers: List[List[str]] = Field( 1138 ..., 1139 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1140 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1141 title="Field Paths", 1142 ) 1143 1144 1145class RequestPath(BaseModel): 1146 type: Literal["RequestPath"] 1147 1148 1149class InjectInto(Enum): 1150 request_parameter = "request_parameter" 1151 header = "header" 1152 body_data = "body_data" 1153 body_json = "body_json" 1154 1155 1156class RequestOption(BaseModel): 1157 type: Literal["RequestOption"] 1158 inject_into: InjectInto = Field( 1159 ..., 1160 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1161 examples=["request_parameter", "header", "body_data", "body_json"], 1162 title="Inject Into", 1163 ) 1164 field_name: Optional[str] = Field( 1165 None, 1166 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1167 examples=["segment_id"], 1168 title="Field Name", 1169 ) 1170 field_path: Optional[List[str]] = Field( 1171 None, 1172 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1173 examples=[["data", "viewer", "id"]], 1174 title="Field Path", 1175 ) 1176 1177 1178class Schemas(BaseModel): 1179 pass 1180 1181 class Config: 1182 extra = Extra.allow 1183 1184 1185class LegacySessionTokenAuthenticator(BaseModel): 1186 type: Literal["LegacySessionTokenAuthenticator"] 1187 header: str = Field( 1188 ..., 1189 description="The name of the session token header that will be injected in the request", 1190 examples=["X-Session"], 1191 title="Session Request Header", 1192 ) 1193 login_url: str = Field( 1194 ..., 1195 description="Path of the login URL (do not include the base URL)", 1196 examples=["session"], 1197 title="Login Path", 1198 ) 1199 session_token: Optional[str] = Field( 1200 None, 1201 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1202 example=["{{ config['session_token'] }}"], 1203 title="Session Token", 1204 ) 1205 session_token_response_key: str = Field( 1206 ..., 1207 description="Name of the key of the session token to be extracted from the response", 1208 examples=["id"], 1209 title="Response Token Response Key", 1210 ) 1211 username: Optional[str] = Field( 1212 None, 1213 description="Username used to authenticate and obtain a session token", 1214 examples=[" {{ config['username'] }}"], 1215 title="Username", 1216 ) 1217 password: Optional[str] = Field( 1218 "", 1219 description="Password used to authenticate and obtain a session token", 1220 examples=["{{ config['password'] }}", ""], 1221 title="Password", 1222 ) 1223 validate_session_url: str = Field( 1224 ..., 1225 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1226 examples=["user/current"], 1227 title="Validate Session Path", 1228 ) 1229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1230 1231 1232class Action1(Enum): 1233 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1234 RESET = "RESET" 1235 1236 1237class PaginationResetLimits(BaseModel): 1238 type: Literal["PaginationResetLimits"] 1239 number_of_records: Optional[int] = None 1240 1241 1242class CsvDecoder(BaseModel): 1243 type: Literal["CsvDecoder"] 1244 encoding: Optional[str] = "utf-8" 1245 delimiter: Optional[str] = "," 1246 set_values_to_none: Optional[List[str]] = None 1247 1248 1249class AsyncJobStatusMap(BaseModel): 1250 type: Optional[Literal["AsyncJobStatusMap"]] = None 1251 running: List[str] 1252 completed: List[str] 1253 failed: List[str] 1254 timeout: List[str] 1255 skipped: Optional[List[str]] = None 1256 1257 1258class ValueType(Enum): 1259 string = "string" 1260 number = "number" 1261 integer = "integer" 1262 boolean = "boolean" 1263 1264 1265class WaitTimeFromHeader(BaseModel): 1266 type: Literal["WaitTimeFromHeader"] 1267 header: str = Field( 1268 ..., 1269 description="The name of the response header defining how long to wait before retrying.", 1270 examples=["Retry-After"], 1271 title="Response Header Name", 1272 ) 1273 regex: Optional[str] = Field( 1274 None, 1275 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1276 examples=["([-+]?\\d+)"], 1277 title="Extraction Regex", 1278 ) 1279 max_waiting_time_in_seconds: Optional[float] = Field( 1280 None, 1281 description="Given the value extracted from the header is greater than this value, stop the stream.", 1282 examples=[3600], 1283 title="Max Waiting Time in Seconds", 1284 ) 1285 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1286 1287 1288class WaitUntilTimeFromHeader(BaseModel): 1289 type: Literal["WaitUntilTimeFromHeader"] 1290 header: str = Field( 1291 ..., 1292 description="The name of the response header defining how long to wait before retrying.", 1293 examples=["wait_time"], 1294 title="Response Header", 1295 ) 1296 min_wait: Optional[Union[float, str]] = Field( 1297 None, 1298 description="Minimum time to wait before retrying.", 1299 examples=[10, "60"], 1300 title="Minimum Wait Time", 1301 ) 1302 regex: Optional[str] = Field( 1303 None, 1304 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1305 examples=["([-+]?\\d+)"], 1306 title="Extraction Regex", 1307 ) 1308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1309 1310 1311class ComponentMappingDefinition(BaseModel): 1312 type: Literal["ComponentMappingDefinition"] 1313 field_path: List[str] = Field( 1314 ..., 1315 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1316 examples=[ 1317 ["name"], 1318 ["retriever", "requester", "url"], 1319 ["retriever", "requester", "{{ components_values.field }}"], 1320 ["*", "**", "name"], 1321 ], 1322 title="Field Path", 1323 ) 1324 value: str = Field( 1325 ..., 1326 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1327 examples=[ 1328 "{{ components_values['updates'] }}", 1329 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1330 "{{ config['segment_id'] }}", 1331 "{{ stream_slice['parent_id'] }}", 1332 "{{ stream_slice['extra_fields']['name'] }}", 1333 ], 1334 title="Value", 1335 ) 1336 value_type: Optional[ValueType] = Field( 1337 None, 1338 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1339 title="Value Type", 1340 ) 1341 create_or_update: Optional[bool] = Field( 1342 False, 1343 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1344 title="Create or Update", 1345 ) 1346 condition: Optional[str] = Field( 1347 None, 1348 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1349 examples=[ 1350 "{{ components_values.get('cursor_field', None) }}", 1351 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1352 ], 1353 title="Condition", 1354 ) 1355 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1356 1357 1358class StreamConfig(BaseModel): 1359 type: Literal["StreamConfig"] 1360 configs_pointer: List[str] = Field( 1361 ..., 1362 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1363 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1364 title="Configs Pointer", 1365 ) 1366 default_values: Optional[List[Dict[str, Any]]] = Field( 1367 None, 1368 description="A list of default values, each matching the structure expected from the parsed component value.", 1369 title="Default Values", 1370 ) 1371 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1372 1373 1374class ConfigComponentsResolver(BaseModel): 1375 type: Literal["ConfigComponentsResolver"] 1376 stream_config: Union[List[StreamConfig], StreamConfig] 1377 components_mapping: List[ComponentMappingDefinition] 1378 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1379 1380 1381class StreamParametersDefinition(BaseModel): 1382 type: Literal["StreamParametersDefinition"] 1383 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1384 ..., 1385 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1386 examples=[ 1387 [ 1388 { 1389 "name": "test stream", 1390 "$parameters": {"entity": "test entity"}, 1391 "primary_key": "test key", 1392 } 1393 ] 1394 ], 1395 title="Stream Parameters", 1396 ) 1397 1398 1399class ParametrizedComponentsResolver(BaseModel): 1400 type: Literal["ParametrizedComponentsResolver"] 1401 stream_parameters: StreamParametersDefinition 1402 components_mapping: List[ComponentMappingDefinition] 1403 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1404 1405 1406class RequestBodyPlainText(BaseModel): 1407 type: Literal["RequestBodyPlainText"] 1408 value: str 1409 1410 1411class RequestBodyUrlEncodedForm(BaseModel): 1412 type: Literal["RequestBodyUrlEncodedForm"] 1413 value: Dict[str, str] 1414 1415 1416class RequestBodyJsonObject(BaseModel): 1417 type: Literal["RequestBodyJsonObject"] 1418 value: Dict[str, Any] 1419 1420 1421class RequestBodyGraphQlQuery(BaseModel): 1422 class Config: 1423 extra = Extra.allow 1424 1425 query: str = Field(..., description="The GraphQL query to be executed") 1426 1427 1428class ValidateAdheresToSchema(BaseModel): 1429 type: Literal["ValidateAdheresToSchema"] 1430 base_schema: Union[str, Dict[str, Any]] = Field( 1431 ..., 1432 description="The base JSON schema against which the user-provided schema will be validated.", 1433 examples=[ 1434 "{{ config['report_validation_schema'] }}", 1435 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1436 { 1437 "$schema": "http://json-schema.org/draft-07/schema#", 1438 "title": "Person", 1439 "type": "object", 1440 "properties": { 1441 "name": {"type": "string", "description": "The person's name"}, 1442 "age": { 1443 "type": "integer", 1444 "minimum": 0, 1445 "description": "The person's age", 1446 }, 1447 }, 1448 "required": ["name", "age"], 1449 }, 1450 ], 1451 title="Base JSON Schema", 1452 ) 1453 1454 1455class CustomValidationStrategy(BaseModel): 1456 class Config: 1457 extra = Extra.allow 1458 1459 type: Literal["CustomValidationStrategy"] 1460 class_name: str = Field( 1461 ..., 1462 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1463 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1464 title="Class Name", 1465 ) 1466 1467 1468class ConfigRemapField(BaseModel): 1469 type: Literal["ConfigRemapField"] 1470 map: Union[Dict[str, Any], str] = Field( 1471 ..., 1472 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1473 examples=[ 1474 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1475 "{{ config['status_mapping'] }}", 1476 ], 1477 title="Value Mapping", 1478 ) 1479 field_path: List[str] = Field( 1480 ..., 1481 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1482 examples=[ 1483 ["status"], 1484 ["data", "status"], 1485 ["data", "{{ config.name }}", "status"], 1486 ["data", "*", "status"], 1487 ], 1488 title="Field Path", 1489 ) 1490 1491 1492class ConfigRemoveFields(BaseModel): 1493 type: Literal["ConfigRemoveFields"] 1494 field_pointers: List[List[str]] = Field( 1495 ..., 1496 description="A list of field pointers to be removed from the config.", 1497 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1498 title="Field Pointers", 1499 ) 1500 condition: Optional[str] = Field( 1501 "", 1502 description="Fields will be removed if expression is evaluated to True.", 1503 examples=[ 1504 "{{ config['environemnt'] == 'sandbox' }}", 1505 "{{ property is integer }}", 1506 "{{ property|length > 5 }}", 1507 "{{ property == 'some_string_to_match' }}", 1508 ], 1509 ) 1510 1511 1512class CustomConfigTransformation(BaseModel): 1513 type: Literal["CustomConfigTransformation"] 1514 class_name: str = Field( 1515 ..., 1516 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1517 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1518 ) 1519 parameters: Optional[Dict[str, Any]] = Field( 1520 None, 1521 alias="$parameters", 1522 description="Additional parameters to be passed to the custom config transformation.", 1523 ) 1524 1525 1526class AddedFieldDefinition(BaseModel): 1527 type: Literal["AddedFieldDefinition"] 1528 path: List[str] = Field( 1529 ..., 1530 description="List of strings defining the path where to add the value on the record.", 1531 examples=[["segment_id"], ["metadata", "segment_id"]], 1532 title="Path", 1533 ) 1534 value: str = Field( 1535 ..., 1536 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1537 examples=[ 1538 "{{ record['updates'] }}", 1539 "{{ record['MetaData']['LastUpdatedTime'] }}", 1540 "{{ stream_partition['segment_id'] }}", 1541 ], 1542 title="Value", 1543 ) 1544 value_type: Optional[ValueType] = Field( 1545 None, 1546 description="Type of the value. If not specified, the type will be inferred from the value.", 1547 title="Value Type", 1548 ) 1549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1550 1551 1552class AddFields(BaseModel): 1553 type: Literal["AddFields"] 1554 fields: List[AddedFieldDefinition] = Field( 1555 ..., 1556 description="List of transformations (path and corresponding value) that will be added to the record.", 1557 title="Fields", 1558 ) 1559 condition: Optional[str] = Field( 1560 "", 1561 description="Fields will be added if expression is evaluated to True.", 1562 examples=[ 1563 "{{ property|string == '' }}", 1564 "{{ property is integer }}", 1565 "{{ property|length > 5 }}", 1566 "{{ property == 'some_string_to_match' }}", 1567 ], 1568 ) 1569 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1570 1571 1572class ApiKeyAuthenticator(BaseModel): 1573 type: Literal["ApiKeyAuthenticator"] 1574 api_token: Optional[str] = Field( 1575 None, 1576 description="The API key to inject in the request. Fill it in the user inputs.", 1577 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1578 title="API Key", 1579 ) 1580 header: Optional[str] = Field( 1581 None, 1582 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1583 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1584 title="Header Name", 1585 ) 1586 inject_into: Optional[RequestOption] = Field( 1587 None, 1588 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1589 examples=[ 1590 {"inject_into": "header", "field_name": "Authorization"}, 1591 {"inject_into": "request_parameter", "field_name": "authKey"}, 1592 ], 1593 title="Inject API Key Into Outgoing HTTP Request", 1594 ) 1595 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1596 1597 1598class AuthFlow(BaseModel): 1599 auth_flow_type: Optional[AuthFlowType] = Field( 1600 None, description="The type of auth to use", title="Auth flow type" 1601 ) 1602 predicate_key: Optional[List[str]] = Field( 1603 None, 1604 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1605 examples=[["credentials", "auth_type"]], 1606 title="Predicate key", 1607 ) 1608 predicate_value: Optional[str] = Field( 1609 None, 1610 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1611 examples=["Oauth"], 1612 title="Predicate value", 1613 ) 1614 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1615 1616 1617class CheckStream(BaseModel): 1618 type: Literal["CheckStream"] 1619 stream_names: Optional[List[str]] = Field( 1620 None, 1621 description="Names of the streams to try reading from when running a check operation.", 1622 examples=[["users"], ["users", "contacts"]], 1623 title="Stream Names", 1624 ) 1625 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1626 1627 1628class IncrementingCountCursor(BaseModel): 1629 type: Literal["IncrementingCountCursor"] 1630 cursor_field: str = Field( 1631 ..., 1632 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1633 examples=["created_at", "{{ config['record_cursor'] }}"], 1634 title="Cursor Field", 1635 ) 1636 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1637 None, 1638 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1639 title="Allow Catalog Defined Cursor Field", 1640 ) 1641 start_value: Optional[Union[str, int]] = Field( 1642 None, 1643 description="The value that determines the earliest record that should be synced.", 1644 examples=[0, "{{ config['start_value'] }}"], 1645 title="Start Value", 1646 ) 1647 start_value_option: Optional[RequestOption] = Field( 1648 None, 1649 description="Optionally configures how the start value will be sent in requests to the source API.", 1650 title="Inject Start Value Into Outgoing HTTP Request", 1651 ) 1652 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1653 1654 1655class DatetimeBasedCursor(BaseModel): 1656 type: Literal["DatetimeBasedCursor"] 1657 clamping: Optional[Clamping] = Field( 1658 None, 1659 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1660 title="Date Range Clamping", 1661 ) 1662 cursor_field: str = Field( 1663 ..., 1664 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1665 examples=["created_at", "{{ config['record_cursor'] }}"], 1666 title="Cursor Field", 1667 ) 1668 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1669 None, 1670 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1671 title="Allow Catalog Defined Cursor Field", 1672 ) 1673 cursor_datetime_formats: Optional[List[str]] = Field( 1674 None, 1675 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1676 examples=[ 1677 "%Y-%m-%d", 1678 "%Y-%m-%d %H:%M:%S", 1679 "%Y-%m-%dT%H:%M:%S", 1680 "%Y-%m-%dT%H:%M:%SZ", 1681 "%Y-%m-%dT%H:%M:%S%z", 1682 "%Y-%m-%dT%H:%M:%S.%fZ", 1683 "%Y-%m-%dT%H:%M:%S.%f%z", 1684 "%Y-%m-%d %H:%M:%S.%f+00:00", 1685 "%s", 1686 "%ms", 1687 ], 1688 title="Cursor Datetime Formats", 1689 ) 1690 start_datetime: Union[MinMaxDatetime, str] = Field( 1691 ..., 1692 description="The datetime that determines the earliest record that should be synced.", 1693 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1694 title="Start Datetime", 1695 ) 1696 start_time_option: Optional[RequestOption] = Field( 1697 None, 1698 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1699 title="Inject Start Time Into Outgoing HTTP Request", 1700 ) 1701 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1702 None, 1703 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1704 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1705 title="End Datetime", 1706 ) 1707 end_time_option: Optional[RequestOption] = Field( 1708 None, 1709 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1710 title="Inject End Time Into Outgoing HTTP Request", 1711 ) 1712 datetime_format: str = Field( 1713 ..., 1714 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1715 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1716 title="Outgoing Datetime Format", 1717 ) 1718 cursor_granularity: Optional[str] = Field( 1719 None, 1720 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1721 examples=["PT1S"], 1722 title="Cursor Granularity", 1723 ) 1724 is_data_feed: Optional[bool] = Field( 1725 None, 1726 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1727 title="Data Feed API", 1728 ) 1729 is_client_side_incremental: Optional[bool] = Field( 1730 None, 1731 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1732 title="Client-side Incremental Filtering", 1733 ) 1734 is_compare_strictly: Optional[bool] = Field( 1735 False, 1736 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1737 title="Strict Start-End Time Comparison", 1738 ) 1739 global_substream_cursor: Optional[bool] = Field( 1740 False, 1741 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1742 title="Global Substream Cursor", 1743 ) 1744 lookback_window: Optional[str] = Field( 1745 None, 1746 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1747 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1748 title="Lookback Window", 1749 ) 1750 partition_field_end: Optional[str] = Field( 1751 None, 1752 description="Name of the partition start time field.", 1753 examples=["ending_time"], 1754 title="Partition Field End", 1755 ) 1756 partition_field_start: Optional[str] = Field( 1757 None, 1758 description="Name of the partition end time field.", 1759 examples=["starting_time"], 1760 title="Partition Field Start", 1761 ) 1762 step: Optional[str] = Field( 1763 None, 1764 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1765 examples=["P1W", "{{ config['step_increment'] }}"], 1766 title="Step", 1767 ) 1768 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1769 1770 1771class JwtAuthenticator(BaseModel): 1772 type: Literal["JwtAuthenticator"] 1773 secret_key: str = Field( 1774 ..., 1775 description="Secret used to sign the JSON web token.", 1776 examples=["{{ config['secret_key'] }}"], 1777 title="Secret Key", 1778 ) 1779 base64_encode_secret_key: Optional[bool] = Field( 1780 False, 1781 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1782 title="Base64-encode Secret Key", 1783 ) 1784 algorithm: Algorithm = Field( 1785 ..., 1786 description="Algorithm used to sign the JSON web token.", 1787 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1788 title="Algorithm", 1789 ) 1790 token_duration: Optional[int] = Field( 1791 1200, 1792 description="The amount of time in seconds a JWT token can be valid after being issued.", 1793 examples=[1200, 3600], 1794 title="Token Duration", 1795 ) 1796 header_prefix: Optional[str] = Field( 1797 None, 1798 description="The prefix to be used within the Authentication header.", 1799 examples=["Bearer", "Basic"], 1800 title="Header Prefix", 1801 ) 1802 jwt_headers: Optional[JwtHeaders] = Field( 1803 None, 1804 description="JWT headers used when signing JSON web token.", 1805 title="JWT Headers", 1806 ) 1807 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1808 None, 1809 description="Additional headers to be included with the JWT headers object.", 1810 title="Additional JWT Headers", 1811 ) 1812 jwt_payload: Optional[JwtPayload] = Field( 1813 None, 1814 description="JWT Payload used when signing JSON web token.", 1815 title="JWT Payload", 1816 ) 1817 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1818 None, 1819 description="Additional properties to be added to the JWT payload.", 1820 title="Additional JWT Payload Properties", 1821 ) 1822 passphrase: Optional[str] = Field( 1823 None, 1824 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1825 examples=["{{ config['passphrase'] }}"], 1826 title="Passphrase", 1827 ) 1828 request_option: Optional[RequestOption] = Field( 1829 None, 1830 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1831 title="Request Option", 1832 ) 1833 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1834 1835 1836class OAuthAuthenticator(BaseModel): 1837 type: Literal["OAuthAuthenticator"] 1838 client_id_name: Optional[str] = Field( 1839 "client_id", 1840 description="The name of the property to use to refresh the `access_token`.", 1841 examples=["custom_app_id"], 1842 title="Client ID Property Name", 1843 ) 1844 client_id: Optional[str] = Field( 1845 None, 1846 description="The OAuth client ID. Fill it in the user inputs.", 1847 examples=[ 1848 "{{ config['client_id'] }}", 1849 "{{ config['credentials']['client_id }}", 1850 ], 1851 title="Client ID", 1852 ) 1853 client_secret_name: Optional[str] = Field( 1854 "client_secret", 1855 description="The name of the property to use to refresh the `access_token`.", 1856 examples=["custom_app_secret"], 1857 title="Client Secret Property Name", 1858 ) 1859 client_secret: Optional[str] = Field( 1860 None, 1861 description="The OAuth client secret. Fill it in the user inputs.", 1862 examples=[ 1863 "{{ config['client_secret'] }}", 1864 "{{ config['credentials']['client_secret }}", 1865 ], 1866 title="Client Secret", 1867 ) 1868 refresh_token_name: Optional[str] = Field( 1869 "refresh_token", 1870 description="The name of the property to use to refresh the `access_token`.", 1871 examples=["custom_app_refresh_value"], 1872 title="Refresh Token Property Name", 1873 ) 1874 refresh_token: Optional[str] = Field( 1875 None, 1876 description="Credential artifact used to get a new access token.", 1877 examples=[ 1878 "{{ config['refresh_token'] }}", 1879 "{{ config['credentials]['refresh_token'] }}", 1880 ], 1881 title="Refresh Token", 1882 ) 1883 token_refresh_endpoint: Optional[str] = Field( 1884 None, 1885 description="The full URL to call to obtain a new access token.", 1886 examples=["https://connect.squareup.com/oauth2/token"], 1887 title="Token Refresh Endpoint", 1888 ) 1889 access_token_name: Optional[str] = Field( 1890 "access_token", 1891 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1892 examples=["access_token"], 1893 title="Access Token Property Name", 1894 ) 1895 access_token_value: Optional[str] = Field( 1896 None, 1897 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1898 examples=["secret_access_token_value"], 1899 title="Access Token Value", 1900 ) 1901 expires_in_name: Optional[str] = Field( 1902 "expires_in", 1903 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1904 examples=["expires_in"], 1905 title="Token Expiry Property Name", 1906 ) 1907 grant_type_name: Optional[str] = Field( 1908 "grant_type", 1909 description="The name of the property to use to refresh the `access_token`.", 1910 examples=["custom_grant_type"], 1911 title="Grant Type Property Name", 1912 ) 1913 grant_type: Optional[str] = Field( 1914 "refresh_token", 1915 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1916 examples=["refresh_token", "client_credentials"], 1917 title="Grant Type", 1918 ) 1919 refresh_request_body: Optional[Dict[str, Any]] = Field( 1920 None, 1921 description="Body of the request sent to get a new access token.", 1922 examples=[ 1923 { 1924 "applicationId": "{{ config['application_id'] }}", 1925 "applicationSecret": "{{ config['application_secret'] }}", 1926 "token": "{{ config['token'] }}", 1927 } 1928 ], 1929 title="Refresh Request Body", 1930 ) 1931 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1932 None, 1933 description="Headers of the request sent to get a new access token.", 1934 examples=[ 1935 { 1936 "Authorization": "<AUTH_TOKEN>", 1937 "Content-Type": "application/x-www-form-urlencoded", 1938 } 1939 ], 1940 title="Refresh Request Headers", 1941 ) 1942 scopes: Optional[List[str]] = Field( 1943 None, 1944 description="List of scopes that should be granted to the access token.", 1945 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1946 title="Scopes", 1947 ) 1948 token_expiry_date: Optional[str] = Field( 1949 None, 1950 description="The access token expiry date.", 1951 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1952 title="Token Expiry Date", 1953 ) 1954 token_expiry_date_format: Optional[str] = Field( 1955 None, 1956 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1957 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1958 title="Token Expiry Date Format", 1959 ) 1960 refresh_token_error_status_codes: Optional[List[int]] = Field( 1961 None, 1962 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1963 examples=[[400, 500]], 1964 title="Refresh Token Error Status Codes", 1965 ) 1966 refresh_token_error_key: Optional[str] = Field( 1967 None, 1968 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1969 examples=["error"], 1970 title="Refresh Token Error Key", 1971 ) 1972 refresh_token_error_values: Optional[List[str]] = Field( 1973 None, 1974 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1975 examples=[["invalid_grant", "invalid_permissions"]], 1976 title="Refresh Token Error Values", 1977 ) 1978 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1979 None, 1980 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1981 title="Refresh Token Updater", 1982 ) 1983 profile_assertion: Optional[JwtAuthenticator] = Field( 1984 None, 1985 description="The authenticator being used to authenticate the client authenticator.", 1986 title="Profile Assertion", 1987 ) 1988 use_profile_assertion: Optional[bool] = Field( 1989 False, 1990 description="Enable using profile assertion as a flow for OAuth authorization.", 1991 title="Use Profile Assertion", 1992 ) 1993 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1994 1995 1996class FixedWindowCallRatePolicy(BaseModel): 1997 class Config: 1998 extra = Extra.allow 1999 2000 type: Literal["FixedWindowCallRatePolicy"] 2001 period: str = Field( 2002 ..., description="The time interval for the rate limit window.", title="Period" 2003 ) 2004 call_limit: int = Field( 2005 ..., 2006 description="The maximum number of calls allowed within the period.", 2007 title="Call Limit", 2008 ) 2009 matchers: List[HttpRequestRegexMatcher] = Field( 2010 ..., 2011 description="List of matchers that define which requests this policy applies to.", 2012 title="Matchers", 2013 ) 2014 2015 2016class MovingWindowCallRatePolicy(BaseModel): 2017 class Config: 2018 extra = Extra.allow 2019 2020 type: Literal["MovingWindowCallRatePolicy"] 2021 rates: List[Rate] = Field( 2022 ..., 2023 description="List of rates that define the call limits for different time intervals.", 2024 title="Rates", 2025 ) 2026 matchers: List[HttpRequestRegexMatcher] = Field( 2027 ..., 2028 description="List of matchers that define which requests this policy applies to.", 2029 title="Matchers", 2030 ) 2031 2032 2033class UnlimitedCallRatePolicy(BaseModel): 2034 class Config: 2035 extra = Extra.allow 2036 2037 type: Literal["UnlimitedCallRatePolicy"] 2038 matchers: List[HttpRequestRegexMatcher] = Field( 2039 ..., 2040 description="List of matchers that define which requests this policy applies to.", 2041 title="Matchers", 2042 ) 2043 2044 2045class DefaultErrorHandler(BaseModel): 2046 type: Literal["DefaultErrorHandler"] 2047 backoff_strategies: Optional[ 2048 List[ 2049 Union[ 2050 ConstantBackoffStrategy, 2051 ExponentialBackoffStrategy, 2052 WaitTimeFromHeader, 2053 WaitUntilTimeFromHeader, 2054 CustomBackoffStrategy, 2055 ] 2056 ] 2057 ] = Field( 2058 None, 2059 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2060 title="Backoff Strategies", 2061 ) 2062 max_retries: Optional[Union[int, str]] = Field( 2063 5, 2064 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2065 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2066 title="Max Retry Count", 2067 ) 2068 response_filters: Optional[List[HttpResponseFilter]] = Field( 2069 None, 2070 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2071 title="Response Filters", 2072 ) 2073 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2074 2075 2076class DefaultPaginator(BaseModel): 2077 type: Literal["DefaultPaginator"] 2078 pagination_strategy: Union[ 2079 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2080 ] = Field( 2081 ..., 2082 description="Strategy defining how records are paginated.", 2083 title="Pagination Strategy", 2084 ) 2085 page_size_option: Optional[RequestOption] = Field( 2086 None, title="Inject Page Size Into Outgoing HTTP Request" 2087 ) 2088 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2089 None, 2090 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2091 title="Inject Page Token Into Outgoing HTTP Request", 2092 ) 2093 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2094 2095 2096class RecordExpander(BaseModel): 2097 type: Literal["RecordExpander"] 2098 expand_records_from_field: List[str] = Field( 2099 ..., 2100 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2101 examples=[ 2102 ["lines", "data"], 2103 ["items"], 2104 ["nested", "array"], 2105 ["sections", "*", "items"], 2106 ], 2107 title="Expand Records From Field", 2108 ) 2109 remain_original_record: Optional[bool] = Field( 2110 False, 2111 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2112 title="Remain Original Record", 2113 ) 2114 on_no_records: Optional[OnNoRecords] = Field( 2115 OnNoRecords.skip, 2116 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2117 title="On No Records", 2118 ) 2119 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2120 2121 2122class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2123 type: Literal["ApiKey"] 2124 inject_into: RequestOption = Field( 2125 ..., 2126 description="Configure how the API Key will be sent in requests to the source API.", 2127 examples=[ 2128 {"inject_into": "header", "field_name": "Authorization"}, 2129 {"inject_into": "request_parameter", "field_name": "authKey"}, 2130 ], 2131 title="Inject API Key Into Outgoing HTTP Request", 2132 ) 2133 api_token: Optional[str] = Field( 2134 "{{ session_token }}", 2135 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2136 examples=[ 2137 "{{ session_token }}", 2138 "Token {{ session_token }}", 2139 "Bearer {{ session_token }}", 2140 ], 2141 title="API Token Template", 2142 ) 2143 2144 2145class JsonSchemaPropertySelector(BaseModel): 2146 type: Literal["JsonSchemaPropertySelector"] 2147 transformations: Optional[ 2148 List[ 2149 Union[ 2150 AddFields, 2151 RemoveFields, 2152 KeysToLower, 2153 KeysToSnakeCase, 2154 FlattenFields, 2155 DpathFlattenFields, 2156 KeysReplace, 2157 CustomTransformation, 2158 ] 2159 ] 2160 ] = Field( 2161 None, 2162 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2163 title="Transformations", 2164 ) 2165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2166 2167 2168class ListPartitionRouter(BaseModel): 2169 type: Literal["ListPartitionRouter"] 2170 cursor_field: str = Field( 2171 ..., 2172 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2173 examples=["section", "{{ config['section_key'] }}"], 2174 title="Current Partition Value Identifier", 2175 ) 2176 values: Union[str, List[str]] = Field( 2177 ..., 2178 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2179 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2180 title="Partition Values", 2181 ) 2182 request_option: Optional[RequestOption] = Field( 2183 None, 2184 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2185 title="Inject Partition Value Into Outgoing HTTP Request", 2186 ) 2187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2188 2189 2190class PaginationReset(BaseModel): 2191 type: Literal["PaginationReset"] 2192 action: Action1 2193 limits: Optional[PaginationResetLimits] = None 2194 2195 2196class GzipDecoder(BaseModel): 2197 type: Literal["GzipDecoder"] 2198 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2199 2200 2201class RequestBodyGraphQL(BaseModel): 2202 type: Literal["RequestBodyGraphQL"] 2203 value: RequestBodyGraphQlQuery 2204 2205 2206class DpathValidator(BaseModel): 2207 type: Literal["DpathValidator"] 2208 field_path: List[str] = Field( 2209 ..., 2210 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2211 examples=[ 2212 ["data"], 2213 ["data", "records"], 2214 ["data", "{{ parameters.name }}"], 2215 ["data", "*", "record"], 2216 ], 2217 title="Field Path", 2218 ) 2219 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2220 ..., 2221 description="The condition that the specified config value will be evaluated against", 2222 title="Validation Strategy", 2223 ) 2224 2225 2226class PredicateValidator(BaseModel): 2227 type: Literal["PredicateValidator"] 2228 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2229 ..., 2230 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2231 examples=[ 2232 "test-value", 2233 "{{ config['api_version'] }}", 2234 "{{ config['tenant_id'] }}", 2235 123, 2236 ], 2237 title="Value", 2238 ) 2239 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2240 ..., 2241 description="The validation strategy to apply to the value.", 2242 title="Validation Strategy", 2243 ) 2244 2245 2246class ConfigAddFields(BaseModel): 2247 type: Literal["ConfigAddFields"] 2248 fields: List[AddedFieldDefinition] = Field( 2249 ..., 2250 description="A list of transformations (path and corresponding value) that will be added to the config.", 2251 title="Fields", 2252 ) 2253 condition: Optional[str] = Field( 2254 "", 2255 description="Fields will be added if expression is evaluated to True.", 2256 examples=[ 2257 "{{ config['environemnt'] == 'sandbox' }}", 2258 "{{ property is integer }}", 2259 "{{ property|length > 5 }}", 2260 "{{ property == 'some_string_to_match' }}", 2261 ], 2262 ) 2263 2264 2265class CompositeErrorHandler(BaseModel): 2266 type: Literal["CompositeErrorHandler"] 2267 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2268 Field( 2269 ..., 2270 description="List of error handlers to iterate on to determine how to handle a failed response.", 2271 title="Error Handlers", 2272 ) 2273 ) 2274 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2275 2276 2277class HTTPAPIBudget(BaseModel): 2278 class Config: 2279 extra = Extra.allow 2280 2281 type: Literal["HTTPAPIBudget"] 2282 policies: List[ 2283 Union[ 2284 FixedWindowCallRatePolicy, 2285 MovingWindowCallRatePolicy, 2286 UnlimitedCallRatePolicy, 2287 ] 2288 ] = Field( 2289 ..., 2290 description="List of call rate policies that define how many calls are allowed.", 2291 title="Policies", 2292 ) 2293 ratelimit_reset_header: Optional[str] = Field( 2294 "ratelimit-reset", 2295 description="The HTTP response header name that indicates when the rate limit resets.", 2296 title="Rate Limit Reset Header", 2297 ) 2298 ratelimit_remaining_header: Optional[str] = Field( 2299 "ratelimit-remaining", 2300 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2301 title="Rate Limit Remaining Header", 2302 ) 2303 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2304 [429], 2305 description="List of HTTP status codes that indicate a rate limit has been hit.", 2306 title="Status Codes for Rate Limit Hit", 2307 ) 2308 2309 2310class DpathExtractor(BaseModel): 2311 type: Literal["DpathExtractor"] 2312 field_path: List[str] = Field( 2313 ..., 2314 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2315 examples=[ 2316 ["data"], 2317 ["data", "records"], 2318 ["data", "{{ parameters.name }}"], 2319 ["data", "*", "record"], 2320 ], 2321 title="Field Path", 2322 ) 2323 record_expander: Optional[RecordExpander] = Field( 2324 None, 2325 description="Optional component to expand records by extracting items from nested array fields.", 2326 title="Record Expander", 2327 ) 2328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2329 2330 2331class ZipfileDecoder(BaseModel): 2332 class Config: 2333 extra = Extra.allow 2334 2335 type: Literal["ZipfileDecoder"] 2336 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2337 ..., 2338 description="Parser to parse the decompressed data from the zipfile(s).", 2339 title="Parser", 2340 ) 2341 2342 2343class RecordSelector(BaseModel): 2344 type: Literal["RecordSelector"] 2345 extractor: Union[DpathExtractor, CustomRecordExtractor] 2346 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2347 None, 2348 description="Responsible for filtering records to be emitted by the Source.", 2349 title="Record Filter", 2350 ) 2351 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2352 None, 2353 description="Responsible for normalization according to the schema.", 2354 title="Schema Normalization", 2355 ) 2356 transform_before_filtering: Optional[bool] = Field( 2357 None, 2358 description="If true, transformation will be applied before record filtering.", 2359 title="Transform Before Filtering", 2360 ) 2361 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2362 2363 2364class ConfigMigration(BaseModel): 2365 type: Literal["ConfigMigration"] 2366 description: Optional[str] = Field( 2367 None, description="The description/purpose of the config migration." 2368 ) 2369 transformations: List[ 2370 Union[ 2371 ConfigRemapField, 2372 ConfigAddFields, 2373 ConfigRemoveFields, 2374 CustomConfigTransformation, 2375 ] 2376 ] = Field( 2377 ..., 2378 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2379 title="Transformations", 2380 ) 2381 2382 2383class ConfigNormalizationRules(BaseModel): 2384 class Config: 2385 extra = Extra.forbid 2386 2387 type: Literal["ConfigNormalizationRules"] 2388 config_migrations: Optional[List[ConfigMigration]] = Field( 2389 [], 2390 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2391 title="Config Migrations", 2392 ) 2393 transformations: Optional[ 2394 List[ 2395 Union[ 2396 ConfigRemapField, 2397 ConfigAddFields, 2398 ConfigRemoveFields, 2399 CustomConfigTransformation, 2400 ] 2401 ] 2402 ] = Field( 2403 [], 2404 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2405 title="Transformations", 2406 ) 2407 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2408 [], 2409 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2410 title="Validations", 2411 ) 2412 2413 2414class Spec(BaseModel): 2415 type: Literal["Spec"] 2416 connection_specification: Dict[str, Any] = Field( 2417 ..., 2418 description="A connection specification describing how a the connector can be configured.", 2419 title="Connection Specification", 2420 ) 2421 documentation_url: Optional[str] = Field( 2422 None, 2423 description="URL of the connector's documentation page.", 2424 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2425 title="Documentation URL", 2426 ) 2427 advanced_auth: Optional[AuthFlow] = Field( 2428 None, 2429 description="Advanced specification for configuring the authentication flow.", 2430 title="Advanced Auth", 2431 ) 2432 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2433 None, title="Config Normalization Rules" 2434 ) 2435 2436 2437class DeclarativeSource1(BaseModel): 2438 class Config: 2439 extra = Extra.forbid 2440 2441 type: Literal["DeclarativeSource"] 2442 check: Union[CheckStream, CheckDynamicStream] 2443 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2444 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2445 version: str = Field( 2446 ..., 2447 description="The version of the Airbyte CDK used to build and test the source.", 2448 ) 2449 schemas: Optional[Schemas] = None 2450 definitions: Optional[Dict[str, Any]] = None 2451 spec: Optional[Spec] = None 2452 concurrency_level: Optional[ConcurrencyLevel] = None 2453 api_budget: Optional[HTTPAPIBudget] = None 2454 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2455 None, 2456 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2457 title="Stream Groups", 2458 ) 2459 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2460 None, 2461 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2462 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2463 title="Maximum Concurrent Asynchronous Jobs", 2464 ) 2465 metadata: Optional[Dict[str, Any]] = Field( 2466 None, 2467 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2468 ) 2469 description: Optional[str] = Field( 2470 None, 2471 description="A description of the connector. It will be presented on the Source documentation page.", 2472 ) 2473 2474 2475class DeclarativeSource2(BaseModel): 2476 class Config: 2477 extra = Extra.forbid 2478 2479 type: Literal["DeclarativeSource"] 2480 check: Union[CheckStream, CheckDynamicStream] 2481 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2482 None 2483 ) 2484 dynamic_streams: List[DynamicDeclarativeStream] 2485 version: str = Field( 2486 ..., 2487 description="The version of the Airbyte CDK used to build and test the source.", 2488 ) 2489 schemas: Optional[Schemas] = None 2490 definitions: Optional[Dict[str, Any]] = None 2491 spec: Optional[Spec] = None 2492 concurrency_level: Optional[ConcurrencyLevel] = None 2493 api_budget: Optional[HTTPAPIBudget] = None 2494 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2495 None, 2496 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2497 title="Stream Groups", 2498 ) 2499 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2500 None, 2501 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2502 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2503 title="Maximum Concurrent Asynchronous Jobs", 2504 ) 2505 metadata: Optional[Dict[str, Any]] = Field( 2506 None, 2507 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2508 ) 2509 description: Optional[str] = Field( 2510 None, 2511 description="A description of the connector. It will be presented on the Source documentation page.", 2512 ) 2513 2514 2515class DeclarativeSource(BaseModel): 2516 class Config: 2517 extra = Extra.forbid 2518 2519 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2520 ..., 2521 description="An API source that extracts data according to its declarative components.", 2522 title="DeclarativeSource", 2523 ) 2524 2525 2526class SelectiveAuthenticator(BaseModel): 2527 class Config: 2528 extra = Extra.allow 2529 2530 type: Literal["SelectiveAuthenticator"] 2531 authenticator_selection_path: List[str] = Field( 2532 ..., 2533 description="Path of the field in config with selected authenticator name", 2534 examples=[["auth"], ["auth", "type"]], 2535 title="Authenticator Selection Path", 2536 ) 2537 authenticators: Dict[ 2538 str, 2539 Union[ 2540 ApiKeyAuthenticator, 2541 BasicHttpAuthenticator, 2542 BearerAuthenticator, 2543 OAuthAuthenticator, 2544 JwtAuthenticator, 2545 SessionTokenAuthenticator, 2546 LegacySessionTokenAuthenticator, 2547 CustomAuthenticator, 2548 NoAuth, 2549 ], 2550 ] = Field( 2551 ..., 2552 description="Authenticators to select from.", 2553 examples=[ 2554 { 2555 "authenticators": { 2556 "token": "#/definitions/ApiKeyAuthenticator", 2557 "oauth": "#/definitions/OAuthAuthenticator", 2558 "jwt": "#/definitions/JwtAuthenticator", 2559 } 2560 } 2561 ], 2562 title="Authenticators", 2563 ) 2564 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2565 2566 2567class ConditionalStreams(BaseModel): 2568 type: Literal["ConditionalStreams"] 2569 condition: str = Field( 2570 ..., 2571 description="Condition that will be evaluated to determine if a set of streams should be available.", 2572 examples=["{{ config['is_sandbox'] }}"], 2573 title="Condition", 2574 ) 2575 streams: List[DeclarativeStream] = Field( 2576 ..., 2577 description="Streams that will be used during an operation based on the condition.", 2578 title="Streams", 2579 ) 2580 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2581 2582 2583class FileUploader(BaseModel): 2584 type: Literal["FileUploader"] 2585 requester: Union[HttpRequester, CustomRequester] = Field( 2586 ..., 2587 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2588 ) 2589 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2590 ..., 2591 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2592 ) 2593 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2594 None, 2595 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2596 ) 2597 filename_extractor: Optional[str] = Field( 2598 None, 2599 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2600 examples=[ 2601 "{{ record.id }}/{{ record.file_name }}/", 2602 "{{ record.id }}_{{ record.file_name }}/", 2603 ], 2604 ) 2605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2606 2607 2608class DeclarativeStream(BaseModel): 2609 class Config: 2610 extra = Extra.allow 2611 2612 type: Literal["DeclarativeStream"] 2613 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2614 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2615 ..., 2616 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2617 title="Retriever", 2618 ) 2619 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2620 None, 2621 description="Component used to fetch data incrementally based on a time field in the data.", 2622 title="Incremental Sync", 2623 ) 2624 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2625 schema_loader: Optional[ 2626 Union[ 2627 InlineSchemaLoader, 2628 DynamicSchemaLoader, 2629 JsonFileSchemaLoader, 2630 List[ 2631 Union[ 2632 InlineSchemaLoader, 2633 DynamicSchemaLoader, 2634 JsonFileSchemaLoader, 2635 CustomSchemaLoader, 2636 ] 2637 ], 2638 CustomSchemaLoader, 2639 ] 2640 ] = Field( 2641 None, 2642 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2643 title="Schema Loader", 2644 ) 2645 transformations: Optional[ 2646 List[ 2647 Union[ 2648 AddFields, 2649 RemoveFields, 2650 KeysToLower, 2651 KeysToSnakeCase, 2652 FlattenFields, 2653 DpathFlattenFields, 2654 KeysReplace, 2655 CustomTransformation, 2656 ] 2657 ] 2658 ] = Field( 2659 None, 2660 description="A list of transformations to be applied to each output record.", 2661 title="Transformations", 2662 ) 2663 state_migrations: Optional[ 2664 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2665 ] = Field( 2666 [], 2667 description="Array of state migrations to be applied on the input state", 2668 title="State Migrations", 2669 ) 2670 file_uploader: Optional[FileUploader] = Field( 2671 None, 2672 description="(experimental) Describes how to fetch a file", 2673 title="File Uploader", 2674 ) 2675 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2676 2677 2678class SessionTokenAuthenticator(BaseModel): 2679 type: Literal["SessionTokenAuthenticator"] 2680 login_requester: HttpRequester = Field( 2681 ..., 2682 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2683 examples=[ 2684 { 2685 "type": "HttpRequester", 2686 "url_base": "https://my_api.com", 2687 "path": "/login", 2688 "authenticator": { 2689 "type": "BasicHttpAuthenticator", 2690 "username": "{{ config.username }}", 2691 "password": "{{ config.password }}", 2692 }, 2693 } 2694 ], 2695 title="Login Requester", 2696 ) 2697 session_token_path: List[str] = Field( 2698 ..., 2699 description="The path in the response body returned from the login requester to the session token.", 2700 examples=[["access_token"], ["result", "token"]], 2701 title="Session Token Path", 2702 ) 2703 expiration_duration: Optional[str] = Field( 2704 None, 2705 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2706 examples=["PT1H", "P1D"], 2707 title="Expiration Duration", 2708 ) 2709 request_authentication: Union[ 2710 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2711 ] = Field( 2712 ..., 2713 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2714 title="Data Request Authentication", 2715 ) 2716 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2717 None, description="Component used to decode the response.", title="Decoder" 2718 ) 2719 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2720 2721 2722class HttpRequester(BaseModelWithDeprecations): 2723 type: Literal["HttpRequester"] 2724 url_base: Optional[str] = Field( 2725 None, 2726 deprecated=True, 2727 deprecation_message="Use `url` field instead.", 2728 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2729 examples=[ 2730 "https://connect.squareup.com/v2", 2731 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2732 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2733 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2734 ], 2735 title="API Base URL", 2736 ) 2737 url: Optional[str] = Field( 2738 None, 2739 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2740 examples=[ 2741 "https://connect.squareup.com/v2", 2742 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2743 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2744 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2745 ], 2746 title="API Endpoint URL", 2747 ) 2748 path: Optional[str] = Field( 2749 None, 2750 deprecated=True, 2751 deprecation_message="Use `url` field instead.", 2752 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2753 examples=[ 2754 "/products", 2755 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2756 "/trades/{{ config['symbol_id'] }}/history", 2757 ], 2758 title="URL Path", 2759 ) 2760 http_method: Optional[HttpMethod] = Field( 2761 HttpMethod.GET, 2762 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2763 examples=["GET", "POST"], 2764 title="HTTP Method", 2765 ) 2766 authenticator: Optional[ 2767 Union[ 2768 ApiKeyAuthenticator, 2769 BasicHttpAuthenticator, 2770 BearerAuthenticator, 2771 OAuthAuthenticator, 2772 JwtAuthenticator, 2773 SessionTokenAuthenticator, 2774 SelectiveAuthenticator, 2775 CustomAuthenticator, 2776 NoAuth, 2777 LegacySessionTokenAuthenticator, 2778 ] 2779 ] = Field( 2780 None, 2781 description="Authentication method to use for requests sent to the API.", 2782 title="Authenticator", 2783 ) 2784 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2785 None, 2786 deprecated=True, 2787 deprecation_message="Use `query_properties` field instead.", 2788 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2789 title="Fetch Properties from Endpoint", 2790 ) 2791 query_properties: Optional[QueryProperties] = Field( 2792 None, 2793 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2794 title="Query Properties", 2795 ) 2796 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2797 None, 2798 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2799 examples=[ 2800 {"unit": "day"}, 2801 { 2802 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2803 }, 2804 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2805 {"sort_by[asc]": "updated_at"}, 2806 ], 2807 title="Query Parameters", 2808 ) 2809 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2810 None, 2811 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2812 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2813 title="Request Headers", 2814 ) 2815 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2816 None, 2817 deprecated=True, 2818 deprecation_message="Use `request_body` field instead.", 2819 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2820 examples=[ 2821 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2822 ], 2823 title="Request Body Payload (Non-JSON)", 2824 ) 2825 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2826 None, 2827 deprecated=True, 2828 deprecation_message="Use `request_body` field instead.", 2829 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2830 examples=[ 2831 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2832 {"key": "{{ config['value'] }}"}, 2833 {"sort": {"field": "updated_at", "order": "ascending"}}, 2834 ], 2835 title="Request Body JSON Payload", 2836 ) 2837 request_body: Optional[ 2838 Union[ 2839 RequestBodyPlainText, 2840 RequestBodyUrlEncodedForm, 2841 RequestBodyJsonObject, 2842 RequestBodyGraphQL, 2843 ] 2844 ] = Field( 2845 None, 2846 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2847 title="Request Body", 2848 ) 2849 error_handler: Optional[ 2850 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2851 ] = Field( 2852 None, 2853 description="Error handler component that defines how to handle errors.", 2854 title="Error Handler", 2855 ) 2856 use_cache: Optional[bool] = Field( 2857 False, 2858 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2859 title="Use Cache", 2860 ) 2861 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2862 2863 2864class DynamicSchemaLoader(BaseModel): 2865 type: Literal["DynamicSchemaLoader"] 2866 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2867 ..., 2868 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2869 title="Retriever", 2870 ) 2871 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2872 None, 2873 description="Responsible for filtering fields to be added to json schema.", 2874 title="Schema Filter", 2875 ) 2876 schema_transformations: Optional[ 2877 List[ 2878 Union[ 2879 AddFields, 2880 RemoveFields, 2881 KeysToLower, 2882 KeysToSnakeCase, 2883 FlattenFields, 2884 DpathFlattenFields, 2885 KeysReplace, 2886 CustomTransformation, 2887 ] 2888 ] 2889 ] = Field( 2890 None, 2891 description="A list of transformations to be applied to the schema.", 2892 title="Schema Transformations", 2893 ) 2894 schema_type_identifier: SchemaTypeIdentifier 2895 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2896 2897 2898class ParentStreamConfig(BaseModel): 2899 type: Literal["ParentStreamConfig"] 2900 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2901 ..., description="Reference to the parent stream.", title="Parent Stream" 2902 ) 2903 parent_key: str = Field( 2904 ..., 2905 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2906 examples=["id", "{{ config['parent_record_id'] }}"], 2907 title="Parent Key", 2908 ) 2909 partition_field: str = Field( 2910 ..., 2911 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2912 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2913 title="Current Parent Key Value Identifier", 2914 ) 2915 request_option: Optional[RequestOption] = Field( 2916 None, 2917 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2918 title="Request Option", 2919 ) 2920 incremental_dependency: Optional[bool] = Field( 2921 False, 2922 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2923 title="Incremental Dependency", 2924 ) 2925 lazy_read_pointer: Optional[List[str]] = Field( 2926 [], 2927 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2928 title="Lazy Read Pointer", 2929 ) 2930 extra_fields: Optional[List[List[str]]] = Field( 2931 None, 2932 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2933 title="Extra Fields", 2934 ) 2935 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2936 2937 2938class PropertiesFromEndpoint(BaseModel): 2939 type: Literal["PropertiesFromEndpoint"] 2940 property_field_path: List[str] = Field( 2941 ..., 2942 description="Describes the path to the field that should be extracted", 2943 examples=[["name"]], 2944 ) 2945 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2946 ..., 2947 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2948 ) 2949 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2950 2951 2952class QueryProperties(BaseModel): 2953 type: Literal["QueryProperties"] 2954 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2955 ..., 2956 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2957 title="Property List", 2958 ) 2959 always_include_properties: Optional[List[str]] = Field( 2960 None, 2961 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2962 title="Always Include Properties", 2963 ) 2964 property_chunking: Optional[PropertyChunking] = Field( 2965 None, 2966 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2967 title="Property Chunking", 2968 ) 2969 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2970 None, 2971 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2972 title="Property Selector", 2973 ) 2974 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2975 2976 2977class StateDelegatingStream(BaseModel): 2978 type: Literal["StateDelegatingStream"] 2979 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2980 full_refresh_stream: DeclarativeStream = Field( 2981 ..., 2982 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2983 title="Full Refresh Stream", 2984 ) 2985 incremental_stream: DeclarativeStream = Field( 2986 ..., 2987 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2988 title="Incremental Stream", 2989 ) 2990 api_retention_period: Optional[str] = Field( 2991 None, 2992 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2993 examples=["P30D", "P90D", "P1Y"], 2994 title="API Retention Period", 2995 ) 2996 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2997 2998 2999class SimpleRetriever(BaseModel): 3000 type: Literal["SimpleRetriever"] 3001 requester: Union[HttpRequester, CustomRequester] = Field( 3002 ..., 3003 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3004 ) 3005 decoder: Optional[ 3006 Union[ 3007 JsonDecoder, 3008 XmlDecoder, 3009 CsvDecoder, 3010 JsonlDecoder, 3011 GzipDecoder, 3012 IterableDecoder, 3013 ZipfileDecoder, 3014 CustomDecoder, 3015 ] 3016 ] = Field( 3017 None, 3018 description="Component decoding the response so records can be extracted.", 3019 title="HTTP Response Format", 3020 ) 3021 record_selector: RecordSelector = Field( 3022 ..., 3023 description="Component that describes how to extract records from a HTTP response.", 3024 ) 3025 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3026 None, 3027 description="Paginator component that describes how to navigate through the API's pages.", 3028 ) 3029 pagination_reset: Optional[PaginationReset] = Field( 3030 None, 3031 description="Describes what triggers pagination reset and how to handle it.", 3032 ) 3033 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3034 False, 3035 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3036 ) 3037 partition_router: Optional[ 3038 Union[ 3039 SubstreamPartitionRouter, 3040 ListPartitionRouter, 3041 GroupingPartitionRouter, 3042 CustomPartitionRouter, 3043 List[ 3044 Union[ 3045 SubstreamPartitionRouter, 3046 ListPartitionRouter, 3047 GroupingPartitionRouter, 3048 CustomPartitionRouter, 3049 ] 3050 ], 3051 ] 3052 ] = Field( 3053 None, 3054 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3055 title="Partition Router", 3056 ) 3057 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3058 3059 3060class AsyncRetriever(BaseModel): 3061 type: Literal["AsyncRetriever"] 3062 record_selector: RecordSelector = Field( 3063 ..., 3064 description="Component that describes how to extract records from a HTTP response.", 3065 ) 3066 status_mapping: AsyncJobStatusMap = Field( 3067 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3068 ) 3069 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3070 ..., description="Responsible for fetching the actual status of the async job." 3071 ) 3072 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3073 None, 3074 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3075 ) 3076 download_extractor: Optional[ 3077 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3078 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3079 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3080 ..., 3081 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3082 ) 3083 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3084 ..., 3085 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3086 ) 3087 polling_job_timeout: Optional[Union[int, str]] = Field( 3088 None, 3089 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3090 ) 3091 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3092 None, 3093 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3094 ge=1, 3095 ) 3096 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3097 None, 3098 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3099 ) 3100 download_requester: Union[HttpRequester, CustomRequester] = Field( 3101 ..., 3102 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3103 ) 3104 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3105 None, 3106 description="Paginator component that describes how to navigate through the API's pages during download.", 3107 ) 3108 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3109 None, 3110 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3111 ) 3112 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3113 None, 3114 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3115 ) 3116 partition_router: Optional[ 3117 Union[ 3118 ListPartitionRouter, 3119 SubstreamPartitionRouter, 3120 GroupingPartitionRouter, 3121 CustomPartitionRouter, 3122 List[ 3123 Union[ 3124 ListPartitionRouter, 3125 SubstreamPartitionRouter, 3126 GroupingPartitionRouter, 3127 CustomPartitionRouter, 3128 ] 3129 ], 3130 ] 3131 ] = Field( 3132 [], 3133 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3134 title="Partition Router", 3135 ) 3136 decoder: Optional[ 3137 Union[ 3138 CsvDecoder, 3139 GzipDecoder, 3140 JsonDecoder, 3141 JsonlDecoder, 3142 IterableDecoder, 3143 XmlDecoder, 3144 ZipfileDecoder, 3145 CustomDecoder, 3146 ] 3147 ] = Field( 3148 None, 3149 description="Component decoding the response so records can be extracted.", 3150 title="HTTP Response Format", 3151 ) 3152 download_decoder: Optional[ 3153 Union[ 3154 CsvDecoder, 3155 GzipDecoder, 3156 JsonDecoder, 3157 JsonlDecoder, 3158 IterableDecoder, 3159 XmlDecoder, 3160 ZipfileDecoder, 3161 CustomDecoder, 3162 ] 3163 ] = Field( 3164 None, 3165 description="Component decoding the download response so records can be extracted.", 3166 title="Download HTTP Response Format", 3167 ) 3168 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3169 3170 3171class BlockSimultaneousSyncsAction(BaseModel): 3172 type: Literal["BlockSimultaneousSyncsAction"] 3173 3174 3175class StreamGroup(BaseModel): 3176 streams: List[str] = Field( 3177 ..., 3178 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3179 title="Streams", 3180 ) 3181 action: BlockSimultaneousSyncsAction = Field( 3182 ..., 3183 description="The action to apply to streams in this group.", 3184 title="Action", 3185 ) 3186 3187 3188class SubstreamPartitionRouter(BaseModel): 3189 type: Literal["SubstreamPartitionRouter"] 3190 parent_stream_configs: List[ParentStreamConfig] = Field( 3191 ..., 3192 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3193 title="Parent Stream Configs", 3194 ) 3195 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3196 3197 3198class GroupingPartitionRouter(BaseModel): 3199 type: Literal["GroupingPartitionRouter"] 3200 group_size: int = Field( 3201 ..., 3202 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3203 examples=[10, 50], 3204 title="Group Size", 3205 ) 3206 underlying_partition_router: Union[ 3207 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3208 ] = Field( 3209 ..., 3210 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3211 title="Underlying Partition Router", 3212 ) 3213 deduplicate: Optional[bool] = Field( 3214 True, 3215 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3216 title="Deduplicate Partitions", 3217 ) 3218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3219 3220 3221class HttpComponentsResolver(BaseModel): 3222 type: Literal["HttpComponentsResolver"] 3223 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3224 ..., 3225 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3226 title="Retriever", 3227 ) 3228 components_mapping: List[ComponentMappingDefinition] 3229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3230 3231 3232class DynamicDeclarativeStream(BaseModel): 3233 type: Literal["DynamicDeclarativeStream"] 3234 name: Optional[str] = Field( 3235 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3236 ) 3237 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3238 ..., description="Reference to the stream template.", title="Stream Template" 3239 ) 3240 components_resolver: Union[ 3241 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3242 ] = Field( 3243 ..., 3244 description="Component resolve and populates stream templates with components values.", 3245 title="Components Resolver", 3246 ) 3247 use_parent_parameters: Optional[bool] = Field( 3248 True, 3249 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3250 title="Use Parent Parameters", 3251 ) 3252 3253 3254ComplexFieldType.update_forward_refs() 3255GzipDecoder.update_forward_refs() 3256CompositeErrorHandler.update_forward_refs() 3257DeclarativeSource1.update_forward_refs() 3258DeclarativeSource2.update_forward_refs() 3259SelectiveAuthenticator.update_forward_refs() 3260ConditionalStreams.update_forward_refs() 3261FileUploader.update_forward_refs() 3262DeclarativeStream.update_forward_refs() 3263SessionTokenAuthenticator.update_forward_refs() 3264HttpRequester.update_forward_refs() 3265DynamicSchemaLoader.update_forward_refs() 3266ParentStreamConfig.update_forward_refs() 3267PropertiesFromEndpoint.update_forward_refs() 3268SimpleRetriever.update_forward_refs() 3269AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 jitter_range_in_seconds: Optional[float] = Field( 109 None, 110 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 111 examples=[15], 112 ge=0, 113 title="Jitter Range", 114 ) 115 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
118class CursorPagination(BaseModel): 119 type: Literal["CursorPagination"] 120 cursor_value: str = Field( 121 ..., 122 description="Value of the cursor defining the next page to fetch.", 123 examples=[ 124 "{{ headers.link.next.cursor }}", 125 "{{ last_record['key'] }}", 126 "{{ response['nextPage'] }}", 127 ], 128 title="Cursor Value", 129 ) 130 page_size: Optional[Union[int, str]] = Field( 131 None, 132 description="The number of records to include in each pages.", 133 examples=[100, "{{ config['page_size'] }}"], 134 title="Page Size", 135 ) 136 stop_condition: Optional[str] = Field( 137 None, 138 description="Template string evaluating when to stop paginating.", 139 examples=[ 140 "{{ response.data.has_more is false }}", 141 "{{ 'next' not in headers['link'] }}", 142 ], 143 title="Stop Condition", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
148class CustomAuthenticator(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomAuthenticator"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
162class CustomBackoffStrategy(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomBackoffStrategy"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomBackoffStrategy"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
176class CustomErrorHandler(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomErrorHandler"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomErrorHandler"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
190class CustomPaginationStrategy(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomPaginationStrategy"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomPaginationStrategy"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
204class CustomRecordExtractor(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordExtractor"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomRecordExtractor"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
218class CustomRecordFilter(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRecordFilter"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomCustomRecordFilter"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
232class CustomRequester(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRequester"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRecordExtractor"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
246class CustomRetriever(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomRetriever"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomRetriever"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
260class CustomPartitionRouter(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomPartitionRouter"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomPartitionRouter"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
274class CustomSchemaLoader(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaLoader"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 282 examples=["source_railz.components.MyCustomSchemaLoader"], 283 title="Class Name", 284 ) 285 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
288class CustomSchemaNormalization(BaseModel): 289 class Config: 290 extra = Extra.allow 291 292 type: Literal["CustomSchemaNormalization"] 293 class_name: str = Field( 294 ..., 295 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 296 examples=[ 297 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 298 ], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
304class CustomStateMigration(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomStateMigration"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomStateMigration"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
318class CustomTransformation(BaseModel): 319 class Config: 320 extra = Extra.allow 321 322 type: Literal["CustomTransformation"] 323 class_name: str = Field( 324 ..., 325 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 326 examples=["source_railz.components.MyCustomTransformation"], 327 title="Class Name", 328 ) 329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
349class Algorithm(Enum): 350 HS256 = "HS256" 351 HS384 = "HS384" 352 HS512 = "HS512" 353 ES256 = "ES256" 354 ES256K = "ES256K" 355 ES384 = "ES384" 356 ES512 = "ES512" 357 RS256 = "RS256" 358 RS384 = "RS384" 359 RS512 = "RS512" 360 PS256 = "PS256" 361 PS384 = "PS384" 362 PS512 = "PS512" 363 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
366class JwtHeaders(BaseModel): 367 class Config: 368 extra = Extra.forbid 369 370 kid: Optional[str] = Field( 371 None, 372 description="Private key ID for user account.", 373 examples=["{{ config['kid'] }}"], 374 title="Key Identifier", 375 ) 376 typ: Optional[str] = Field( 377 "JWT", 378 description="The media type of the complete JWT.", 379 examples=["JWT"], 380 title="Type", 381 ) 382 cty: Optional[str] = Field( 383 None, 384 description="Content type of JWT header.", 385 examples=["JWT"], 386 title="Content Type", 387 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
390class JwtPayload(BaseModel): 391 class Config: 392 extra = Extra.forbid 393 394 iss: Optional[str] = Field( 395 None, 396 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 397 examples=["{{ config['iss'] }}"], 398 title="Issuer", 399 ) 400 sub: Optional[str] = Field( 401 None, 402 description="The subject of the JWT. Commonly defined by the API.", 403 title="Subject", 404 ) 405 aud: Optional[str] = Field( 406 None, 407 description="The recipient that the JWT is intended for. Commonly defined by the API.", 408 examples=["appstoreconnect-v1"], 409 title="Audience", 410 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
413class RefreshTokenUpdater(BaseModel): 414 refresh_token_name: Optional[str] = Field( 415 "refresh_token", 416 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 417 examples=["refresh_token"], 418 title="Refresh Token Property Name", 419 ) 420 access_token_config_path: Optional[List[str]] = Field( 421 ["credentials", "access_token"], 422 description="Config path to the access token. Make sure the field actually exists in the config.", 423 examples=[["credentials", "access_token"], ["access_token"]], 424 title="Config Path To Access Token", 425 ) 426 refresh_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "refresh_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "refresh_token"], ["refresh_token"]], 430 title="Config Path To Refresh Token", 431 ) 432 token_expiry_date_config_path: Optional[List[str]] = Field( 433 ["credentials", "token_expiry_date"], 434 description="Config path to the expiry date. Make sure actually exists in the config.", 435 examples=[["credentials", "token_expiry_date"]], 436 title="Config Path To Expiry Date", 437 ) 438 refresh_token_error_status_codes: Optional[List[int]] = Field( 439 [], 440 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 441 examples=[[400, 500]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 443 ) 444 refresh_token_error_key: Optional[str] = Field( 445 "", 446 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 447 examples=["error"], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 449 ) 450 refresh_token_error_values: Optional[List[str]] = Field( 451 [], 452 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 453 examples=[["invalid_grant", "invalid_permissions"]], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 455 )
class
Rate(pydantic.v1.main.BaseModel):
458class Rate(BaseModel): 459 class Config: 460 extra = Extra.allow 461 462 limit: Union[int, str] = Field( 463 ..., 464 description="The maximum number of calls allowed within the interval.", 465 title="Limit", 466 ) 467 interval: str = Field( 468 ..., 469 description="The time interval for the rate limit.", 470 examples=["PT1H", "P1D"], 471 title="Interval", 472 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
475class HttpRequestRegexMatcher(BaseModel): 476 class Config: 477 extra = Extra.allow 478 479 method: Optional[str] = Field( 480 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 481 ) 482 url_base: Optional[str] = Field( 483 None, 484 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 485 title="URL Base", 486 ) 487 url_path_pattern: Optional[str] = Field( 488 None, 489 description="A regular expression pattern to match the URL path.", 490 title="URL Path Pattern", 491 ) 492 params: Optional[Dict[str, Any]] = Field( 493 None, description="The query parameters to match.", title="Parameters" 494 ) 495 headers: Optional[Dict[str, Any]] = Field( 496 None, description="The headers to match.", title="Headers" 497 ) 498 weight: Optional[Union[int, str]] = Field( 499 None, 500 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 501 title="Weight", 502 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
OnNoRecords(enum.Enum):
An enumeration.
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
515class ExponentialBackoffStrategy(BaseModel): 516 type: Literal["ExponentialBackoffStrategy"] 517 factor: Optional[Union[float, str]] = Field( 518 5, 519 description="Multiplicative constant applied on each retry.", 520 examples=[5, 5.5, "10"], 521 title="Factor", 522 ) 523 jitter_range_in_seconds: Optional[float] = Field( 524 None, 525 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 526 examples=[2], 527 ge=0, 528 title="Jitter Range", 529 ) 530 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
533class GroupByKeyMergeStrategy(BaseModel): 534 type: Literal["GroupByKeyMergeStrategy"] 535 key: Union[str, List[str]] = Field( 536 ..., 537 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 538 examples=["id", ["parent_id", "end_date"]], 539 title="Key", 540 ) 541 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
553class Action(Enum): 554 SUCCESS = "SUCCESS" 555 FAIL = "FAIL" 556 RETRY = "RETRY" 557 IGNORE = "IGNORE" 558 RESET_PAGINATION = "RESET_PAGINATION" 559 RATE_LIMITED = "RATE_LIMITED" 560 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
563class FailureType(Enum): 564 system_error = "system_error" 565 config_error = "config_error" 566 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
569class HttpResponseFilter(BaseModel): 570 type: Literal["HttpResponseFilter"] 571 action: Optional[Action] = Field( 572 None, 573 description="Action to execute if a response matches the filter.", 574 examples=[ 575 "SUCCESS", 576 "FAIL", 577 "RETRY", 578 "IGNORE", 579 "RESET_PAGINATION", 580 "RATE_LIMITED", 581 "REFRESH_TOKEN_THEN_RETRY", 582 ], 583 title="Action", 584 ) 585 failure_type: Optional[FailureType] = Field( 586 None, 587 description="Failure type of traced exception if a response matches the filter.", 588 examples=["system_error", "config_error", "transient_error"], 589 title="Failure Type", 590 ) 591 error_message: Optional[str] = Field( 592 None, 593 description="Error Message to display if the response matches the filter.", 594 title="Error Message", 595 ) 596 error_message_contains: Optional[str] = Field( 597 None, 598 description="Match the response if its error message contains the substring.", 599 example=["This API operation is not enabled for this site"], 600 title="Error Message Substring", 601 ) 602 http_codes: Optional[List[int]] = Field( 603 None, 604 description="Match the response if its HTTP code is included in this list.", 605 examples=[[420, 429], [500]], 606 title="HTTP Codes", 607 unique_items=True, 608 ) 609 predicate: Optional[str] = Field( 610 None, 611 description="Match the response if the predicate evaluates to true.", 612 examples=[ 613 "{{ 'Too much requests' in response }}", 614 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 615 ], 616 title="Predicate", 617 ) 618 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
621class ComplexFieldType(BaseModel): 622 field_type: str 623 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
626class TypesMap(BaseModel): 627 target_type: Union[str, List[str], ComplexFieldType] 628 current_type: Union[str, List[str]] 629 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
632class SchemaTypeIdentifier(BaseModel): 633 type: Optional[Literal["SchemaTypeIdentifier"]] = None 634 schema_pointer: Optional[List[str]] = Field( 635 [], 636 description="List of nested fields defining the schema field path to extract. Defaults to [].", 637 title="Schema Path", 638 ) 639 key_pointer: List[str] = Field( 640 ..., 641 description="List of potentially nested fields describing the full path of the field key to extract.", 642 title="Key Path", 643 ) 644 type_pointer: Optional[List[str]] = Field( 645 None, 646 description="List of potentially nested fields describing the full path of the field type to extract.", 647 title="Type Path", 648 ) 649 types_mapping: Optional[List[TypesMap]] = None 650 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
653class InlineSchemaLoader(BaseModel): 654 type: Literal["InlineSchemaLoader"] 655 schema_: Optional[Dict[str, Any]] = Field( 656 None, 657 alias="schema", 658 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 659 title="Schema", 660 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
663class JsonFileSchemaLoader(BaseModel): 664 type: Literal["JsonFileSchemaLoader"] 665 file_path: Optional[str] = Field( 666 None, 667 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 668 example=["./schemas/users.json"], 669 title="File Path", 670 ) 671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
692class FlattenFields(BaseModel): 693 type: Literal["FlattenFields"] 694 flatten_lists: Optional[bool] = Field( 695 True, 696 description="Whether to flatten lists or leave it as is. Default is True.", 697 title="Flatten Lists", 698 ) 699 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
702class KeyTransformation(BaseModel): 703 type: Literal["KeyTransformation"] 704 prefix: Optional[str] = Field( 705 None, 706 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 707 examples=["flattened_"], 708 title="Key Prefix", 709 ) 710 suffix: Optional[str] = Field( 711 None, 712 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 713 examples=["_flattened"], 714 title="Key Suffix", 715 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
718class DpathFlattenFields(BaseModel): 719 type: Literal["DpathFlattenFields"] 720 field_path: List[str] = Field( 721 ..., 722 description="A path to field that needs to be flattened.", 723 examples=[["data"], ["data", "*", "field"]], 724 title="Field Path", 725 ) 726 delete_origin_value: Optional[bool] = Field( 727 None, 728 description="Whether to delete the origin value or keep it. Default is False.", 729 title="Delete Origin Value", 730 ) 731 replace_record: Optional[bool] = Field( 732 None, 733 description="Whether to replace the origin record or not. Default is False.", 734 title="Replace Origin Record", 735 ) 736 key_transformation: Optional[KeyTransformation] = Field( 737 None, 738 description="Transformation for object keys. If not provided, original key will be used.", 739 title="Key transformation", 740 ) 741 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
744class KeysReplace(BaseModel): 745 type: Literal["KeysReplace"] 746 old: str = Field( 747 ..., 748 description="Old value to replace.", 749 examples=[ 750 " ", 751 "{{ record.id }}", 752 "{{ config['id'] }}", 753 "{{ stream_slice['id'] }}", 754 ], 755 title="Old value", 756 ) 757 new: str = Field( 758 ..., 759 description="New value to set.", 760 examples=[ 761 "_", 762 "{{ record.id }}", 763 "{{ config['id'] }}", 764 "{{ stream_slice['id'] }}", 765 ], 766 title="New value", 767 ) 768 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
779class CustomDecoder(BaseModel): 780 class Config: 781 extra = Extra.allow 782 783 type: Literal["CustomDecoder"] 784 class_name: str = Field( 785 ..., 786 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 787 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 788 title="Class Name", 789 ) 790 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
793class MinMaxDatetime(BaseModel): 794 type: Literal["MinMaxDatetime"] 795 datetime: str = Field( 796 ..., 797 description="Datetime value.", 798 examples=[ 799 "2021-01-01", 800 "2021-01-01T00:00:00Z", 801 "{{ config['start_time'] }}", 802 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 803 ], 804 title="Datetime", 805 ) 806 datetime_format: Optional[str] = Field( 807 "", 808 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 809 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 810 title="Datetime Format", 811 ) 812 max_datetime: Optional[str] = Field( 813 None, 814 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 815 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 816 title="Max Datetime", 817 ) 818 min_datetime: Optional[str] = Field( 819 None, 820 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 821 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 822 title="Min Datetime", 823 ) 824 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
854class OauthConnectorInputSpecification(BaseModel): 855 class Config: 856 extra = Extra.allow 857 858 consent_url: str = Field( 859 ..., 860 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 861 examples=[ 862 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 863 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 864 ], 865 title="Consent URL", 866 ) 867 scope: Optional[str] = Field( 868 None, 869 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 870 examples=["user:read user:read_orders workspaces:read"], 871 title="Scopes", 872 ) 873 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 874 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 875 # The CDK schema defines the manifest contract; the platform reads these fields 876 # during the OAuth consent flow to build the authorization URL. 877 scopes: Optional[List[OAuthScope]] = Field( 878 None, 879 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 880 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 881 title="Scopes", 882 ) 883 optional_scopes: Optional[List[OAuthScope]] = Field( 884 None, 885 description="Optional OAuth scope objects that may or may not be granted.", 886 examples=[[{"scope": "admin:read"}]], 887 title="Optional Scopes", 888 ) 889 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 890 ScopesJoinStrategy.space, 891 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 892 title="Scopes Join Strategy", 893 ) 894 access_token_url: str = Field( 895 ..., 896 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 897 examples=[ 898 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 899 ], 900 title="Access Token URL", 901 ) 902 access_token_headers: Optional[Dict[str, Any]] = Field( 903 None, 904 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 905 examples=[ 906 { 907 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 908 } 909 ], 910 title="Access Token Headers", 911 ) 912 access_token_params: Optional[Dict[str, Any]] = Field( 913 None, 914 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 915 examples=[ 916 { 917 "{{ auth_code_key }}": "{{ auth_code_value }}", 918 "{{ client_id_key }}": "{{ client_id_value }}", 919 "{{ client_secret_key }}": "{{ client_secret_value }}", 920 } 921 ], 922 title="Access Token Query Params (Json Encoded)", 923 ) 924 extract_output: Optional[List[str]] = Field( 925 None, 926 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 927 examples=[["access_token", "refresh_token", "other_field"]], 928 title="Extract Output", 929 ) 930 state: Optional[State] = Field( 931 None, 932 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 933 examples=[{"min": 7, "max": 128}], 934 title="Configurable State Query Param", 935 ) 936 client_id_key: Optional[str] = Field( 937 None, 938 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 939 examples=["my_custom_client_id_key_name"], 940 title="Client ID Key Override", 941 ) 942 client_secret_key: Optional[str] = Field( 943 None, 944 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 945 examples=["my_custom_client_secret_key_name"], 946 title="Client Secret Key Override", 947 ) 948 scope_key: Optional[str] = Field( 949 None, 950 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 951 examples=["my_custom_scope_key_key_name"], 952 title="Scopes Key Override", 953 ) 954 state_key: Optional[str] = Field( 955 None, 956 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 957 examples=["my_custom_state_key_key_name"], 958 title="State Key Override", 959 ) 960 auth_code_key: Optional[str] = Field( 961 None, 962 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 963 examples=["my_custom_auth_code_key_name"], 964 title="Auth Code Key Override", 965 ) 966 redirect_uri_key: Optional[str] = Field( 967 None, 968 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 969 examples=["my_custom_redirect_uri_key_name"], 970 title="Redirect URI Key Override", 971 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
974class OAuthConfigSpecification(BaseModel): 975 class Config: 976 extra = Extra.allow 977 978 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 979 None, 980 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 981 examples=[ 982 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 983 { 984 "app_id": { 985 "type": "string", 986 "path_in_connector_config": ["info", "app_id"], 987 } 988 }, 989 ], 990 title="OAuth user input", 991 ) 992 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 993 None, 994 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 995 title="DeclarativeOAuth Connector Specification", 996 ) 997 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 998 None, 999 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1000 examples=[ 1001 { 1002 "refresh_token": { 1003 "type": "string,", 1004 "path_in_connector_config": ["credentials", "refresh_token"], 1005 } 1006 } 1007 ], 1008 title="OAuth output specification", 1009 ) 1010 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1011 None, 1012 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1013 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1014 title="OAuth input specification", 1015 ) 1016 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1017 None, 1018 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1019 examples=[ 1020 { 1021 "client_id": { 1022 "type": "string,", 1023 "path_in_connector_config": ["credentials", "client_id"], 1024 }, 1025 "client_secret": { 1026 "type": "string,", 1027 "path_in_connector_config": ["credentials", "client_secret"], 1028 }, 1029 } 1030 ], 1031 title="OAuth server output specification", 1032 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1035class OffsetIncrement(BaseModel): 1036 type: Literal["OffsetIncrement"] 1037 page_size: Optional[Union[int, str]] = Field( 1038 None, 1039 description="The number of records to include in each pages.", 1040 examples=[100, "{{ config['page_size'] }}"], 1041 title="Limit", 1042 ) 1043 inject_on_first_request: Optional[bool] = Field( 1044 False, 1045 description="Using the `offset` with value `0` during the first request", 1046 title="Inject Offset on First Request", 1047 ) 1048 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1051class PageIncrement(BaseModel): 1052 type: Literal["PageIncrement"] 1053 page_size: Optional[Union[int, str]] = Field( 1054 None, 1055 description="The number of records to include in each pages.", 1056 examples=[100, "100", "{{ config['page_size'] }}"], 1057 title="Page Size", 1058 ) 1059 start_from_page: Optional[int] = Field( 1060 0, 1061 description="Index of the first page to request.", 1062 examples=[0, 1], 1063 title="Start From Page", 1064 ) 1065 inject_on_first_request: Optional[bool] = Field( 1066 False, 1067 description="Using the `page number` with value defined by `start_from_page` during the first request", 1068 title="Inject Page Number on First Request", 1069 ) 1070 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1073class PrimaryKey(BaseModel): 1074 __root__: Union[str, List[str], List[List[str]]] = Field( 1075 ..., 1076 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1077 examples=["id", ["code", "type"]], 1078 title="Primary Key", 1079 )
class
PropertyLimitType(enum.Enum):
1082class PropertyLimitType(Enum): 1083 characters = "characters" 1084 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1087class PropertyChunking(BaseModel): 1088 type: Literal["PropertyChunking"] 1089 property_limit_type: PropertyLimitType = Field( 1090 ..., 1091 description="The type used to determine the maximum number of properties per chunk", 1092 title="Property Limit Type", 1093 ) 1094 property_limit: Optional[int] = Field( 1095 None, 1096 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1097 title="Property Limit", 1098 ) 1099 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1100 None, 1101 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1102 title="Record Merge Strategy", 1103 ) 1104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1107class RecordFilter(BaseModel): 1108 type: Literal["RecordFilter"] 1109 condition: Optional[str] = Field( 1110 "", 1111 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1112 examples=[ 1113 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1114 "{{ record.status in ['active', 'expired'] }}", 1115 ], 1116 title="Condition", 1117 ) 1118 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1126class RemoveFields(BaseModel): 1127 type: Literal["RemoveFields"] 1128 condition: Optional[str] = Field( 1129 "", 1130 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1131 examples=[ 1132 "{{ property|string == '' }}", 1133 "{{ property is integer }}", 1134 "{{ property|length > 5 }}", 1135 "{{ property == 'some_string_to_match' }}", 1136 ], 1137 ) 1138 field_pointers: List[List[str]] = Field( 1139 ..., 1140 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1141 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1142 title="Field Paths", 1143 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1150class InjectInto(Enum): 1151 request_parameter = "request_parameter" 1152 header = "header" 1153 body_data = "body_data" 1154 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1157class RequestOption(BaseModel): 1158 type: Literal["RequestOption"] 1159 inject_into: InjectInto = Field( 1160 ..., 1161 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1162 examples=["request_parameter", "header", "body_data", "body_json"], 1163 title="Inject Into", 1164 ) 1165 field_name: Optional[str] = Field( 1166 None, 1167 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1168 examples=["segment_id"], 1169 title="Field Name", 1170 ) 1171 field_path: Optional[List[str]] = Field( 1172 None, 1173 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1174 examples=[["data", "viewer", "id"]], 1175 title="Field Path", 1176 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1186class LegacySessionTokenAuthenticator(BaseModel): 1187 type: Literal["LegacySessionTokenAuthenticator"] 1188 header: str = Field( 1189 ..., 1190 description="The name of the session token header that will be injected in the request", 1191 examples=["X-Session"], 1192 title="Session Request Header", 1193 ) 1194 login_url: str = Field( 1195 ..., 1196 description="Path of the login URL (do not include the base URL)", 1197 examples=["session"], 1198 title="Login Path", 1199 ) 1200 session_token: Optional[str] = Field( 1201 None, 1202 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1203 example=["{{ config['session_token'] }}"], 1204 title="Session Token", 1205 ) 1206 session_token_response_key: str = Field( 1207 ..., 1208 description="Name of the key of the session token to be extracted from the response", 1209 examples=["id"], 1210 title="Response Token Response Key", 1211 ) 1212 username: Optional[str] = Field( 1213 None, 1214 description="Username used to authenticate and obtain a session token", 1215 examples=[" {{ config['username'] }}"], 1216 title="Username", 1217 ) 1218 password: Optional[str] = Field( 1219 "", 1220 description="Password used to authenticate and obtain a session token", 1221 examples=["{{ config['password'] }}", ""], 1222 title="Password", 1223 ) 1224 validate_session_url: str = Field( 1225 ..., 1226 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1227 examples=["user/current"], 1228 title="Validate Session Path", 1229 ) 1230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1259class ValueType(Enum): 1260 string = "string" 1261 number = "number" 1262 integer = "integer" 1263 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1266class WaitTimeFromHeader(BaseModel): 1267 type: Literal["WaitTimeFromHeader"] 1268 header: str = Field( 1269 ..., 1270 description="The name of the response header defining how long to wait before retrying.", 1271 examples=["Retry-After"], 1272 title="Response Header Name", 1273 ) 1274 regex: Optional[str] = Field( 1275 None, 1276 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1277 examples=["([-+]?\\d+)"], 1278 title="Extraction Regex", 1279 ) 1280 max_waiting_time_in_seconds: Optional[float] = Field( 1281 None, 1282 description="Given the value extracted from the header is greater than this value, stop the stream.", 1283 examples=[3600], 1284 title="Max Waiting Time in Seconds", 1285 ) 1286 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1289class WaitUntilTimeFromHeader(BaseModel): 1290 type: Literal["WaitUntilTimeFromHeader"] 1291 header: str = Field( 1292 ..., 1293 description="The name of the response header defining how long to wait before retrying.", 1294 examples=["wait_time"], 1295 title="Response Header", 1296 ) 1297 min_wait: Optional[Union[float, str]] = Field( 1298 None, 1299 description="Minimum time to wait before retrying.", 1300 examples=[10, "60"], 1301 title="Minimum Wait Time", 1302 ) 1303 regex: Optional[str] = Field( 1304 None, 1305 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1306 examples=["([-+]?\\d+)"], 1307 title="Extraction Regex", 1308 ) 1309 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1312class ComponentMappingDefinition(BaseModel): 1313 type: Literal["ComponentMappingDefinition"] 1314 field_path: List[str] = Field( 1315 ..., 1316 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1317 examples=[ 1318 ["name"], 1319 ["retriever", "requester", "url"], 1320 ["retriever", "requester", "{{ components_values.field }}"], 1321 ["*", "**", "name"], 1322 ], 1323 title="Field Path", 1324 ) 1325 value: str = Field( 1326 ..., 1327 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1328 examples=[ 1329 "{{ components_values['updates'] }}", 1330 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1331 "{{ config['segment_id'] }}", 1332 "{{ stream_slice['parent_id'] }}", 1333 "{{ stream_slice['extra_fields']['name'] }}", 1334 ], 1335 title="Value", 1336 ) 1337 value_type: Optional[ValueType] = Field( 1338 None, 1339 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1340 title="Value Type", 1341 ) 1342 create_or_update: Optional[bool] = Field( 1343 False, 1344 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1345 title="Create or Update", 1346 ) 1347 condition: Optional[str] = Field( 1348 None, 1349 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1350 examples=[ 1351 "{{ components_values.get('cursor_field', None) }}", 1352 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1353 ], 1354 title="Condition", 1355 ) 1356 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1359class StreamConfig(BaseModel): 1360 type: Literal["StreamConfig"] 1361 configs_pointer: List[str] = Field( 1362 ..., 1363 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1364 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1365 title="Configs Pointer", 1366 ) 1367 default_values: Optional[List[Dict[str, Any]]] = Field( 1368 None, 1369 description="A list of default values, each matching the structure expected from the parsed component value.", 1370 title="Default Values", 1371 ) 1372 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1375class ConfigComponentsResolver(BaseModel): 1376 type: Literal["ConfigComponentsResolver"] 1377 stream_config: Union[List[StreamConfig], StreamConfig] 1378 components_mapping: List[ComponentMappingDefinition] 1379 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1382class StreamParametersDefinition(BaseModel): 1383 type: Literal["StreamParametersDefinition"] 1384 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1385 ..., 1386 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1387 examples=[ 1388 [ 1389 { 1390 "name": "test stream", 1391 "$parameters": {"entity": "test entity"}, 1392 "primary_key": "test key", 1393 } 1394 ] 1395 ], 1396 title="Stream Parameters", 1397 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1400class ParametrizedComponentsResolver(BaseModel): 1401 type: Literal["ParametrizedComponentsResolver"] 1402 stream_parameters: StreamParametersDefinition 1403 components_mapping: List[ComponentMappingDefinition] 1404 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1429class ValidateAdheresToSchema(BaseModel): 1430 type: Literal["ValidateAdheresToSchema"] 1431 base_schema: Union[str, Dict[str, Any]] = Field( 1432 ..., 1433 description="The base JSON schema against which the user-provided schema will be validated.", 1434 examples=[ 1435 "{{ config['report_validation_schema'] }}", 1436 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1437 { 1438 "$schema": "http://json-schema.org/draft-07/schema#", 1439 "title": "Person", 1440 "type": "object", 1441 "properties": { 1442 "name": {"type": "string", "description": "The person's name"}, 1443 "age": { 1444 "type": "integer", 1445 "minimum": 0, 1446 "description": "The person's age", 1447 }, 1448 }, 1449 "required": ["name", "age"], 1450 }, 1451 ], 1452 title="Base JSON Schema", 1453 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1456class CustomValidationStrategy(BaseModel): 1457 class Config: 1458 extra = Extra.allow 1459 1460 type: Literal["CustomValidationStrategy"] 1461 class_name: str = Field( 1462 ..., 1463 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1464 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1465 title="Class Name", 1466 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1469class ConfigRemapField(BaseModel): 1470 type: Literal["ConfigRemapField"] 1471 map: Union[Dict[str, Any], str] = Field( 1472 ..., 1473 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1474 examples=[ 1475 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1476 "{{ config['status_mapping'] }}", 1477 ], 1478 title="Value Mapping", 1479 ) 1480 field_path: List[str] = Field( 1481 ..., 1482 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1483 examples=[ 1484 ["status"], 1485 ["data", "status"], 1486 ["data", "{{ config.name }}", "status"], 1487 ["data", "*", "status"], 1488 ], 1489 title="Field Path", 1490 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1493class ConfigRemoveFields(BaseModel): 1494 type: Literal["ConfigRemoveFields"] 1495 field_pointers: List[List[str]] = Field( 1496 ..., 1497 description="A list of field pointers to be removed from the config.", 1498 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1499 title="Field Pointers", 1500 ) 1501 condition: Optional[str] = Field( 1502 "", 1503 description="Fields will be removed if expression is evaluated to True.", 1504 examples=[ 1505 "{{ config['environemnt'] == 'sandbox' }}", 1506 "{{ property is integer }}", 1507 "{{ property|length > 5 }}", 1508 "{{ property == 'some_string_to_match' }}", 1509 ], 1510 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1513class CustomConfigTransformation(BaseModel): 1514 type: Literal["CustomConfigTransformation"] 1515 class_name: str = Field( 1516 ..., 1517 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1518 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1519 ) 1520 parameters: Optional[Dict[str, Any]] = Field( 1521 None, 1522 alias="$parameters", 1523 description="Additional parameters to be passed to the custom config transformation.", 1524 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1527class AddedFieldDefinition(BaseModel): 1528 type: Literal["AddedFieldDefinition"] 1529 path: List[str] = Field( 1530 ..., 1531 description="List of strings defining the path where to add the value on the record.", 1532 examples=[["segment_id"], ["metadata", "segment_id"]], 1533 title="Path", 1534 ) 1535 value: str = Field( 1536 ..., 1537 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1538 examples=[ 1539 "{{ record['updates'] }}", 1540 "{{ record['MetaData']['LastUpdatedTime'] }}", 1541 "{{ stream_partition['segment_id'] }}", 1542 ], 1543 title="Value", 1544 ) 1545 value_type: Optional[ValueType] = Field( 1546 None, 1547 description="Type of the value. If not specified, the type will be inferred from the value.", 1548 title="Value Type", 1549 ) 1550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1553class AddFields(BaseModel): 1554 type: Literal["AddFields"] 1555 fields: List[AddedFieldDefinition] = Field( 1556 ..., 1557 description="List of transformations (path and corresponding value) that will be added to the record.", 1558 title="Fields", 1559 ) 1560 condition: Optional[str] = Field( 1561 "", 1562 description="Fields will be added if expression is evaluated to True.", 1563 examples=[ 1564 "{{ property|string == '' }}", 1565 "{{ property is integer }}", 1566 "{{ property|length > 5 }}", 1567 "{{ property == 'some_string_to_match' }}", 1568 ], 1569 ) 1570 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1573class ApiKeyAuthenticator(BaseModel): 1574 type: Literal["ApiKeyAuthenticator"] 1575 api_token: Optional[str] = Field( 1576 None, 1577 description="The API key to inject in the request. Fill it in the user inputs.", 1578 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1579 title="API Key", 1580 ) 1581 header: Optional[str] = Field( 1582 None, 1583 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1584 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1585 title="Header Name", 1586 ) 1587 inject_into: Optional[RequestOption] = Field( 1588 None, 1589 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1590 examples=[ 1591 {"inject_into": "header", "field_name": "Authorization"}, 1592 {"inject_into": "request_parameter", "field_name": "authKey"}, 1593 ], 1594 title="Inject API Key Into Outgoing HTTP Request", 1595 ) 1596 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1599class AuthFlow(BaseModel): 1600 auth_flow_type: Optional[AuthFlowType] = Field( 1601 None, description="The type of auth to use", title="Auth flow type" 1602 ) 1603 predicate_key: Optional[List[str]] = Field( 1604 None, 1605 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1606 examples=[["credentials", "auth_type"]], 1607 title="Predicate key", 1608 ) 1609 predicate_value: Optional[str] = Field( 1610 None, 1611 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1612 examples=["Oauth"], 1613 title="Predicate value", 1614 ) 1615 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1618class CheckStream(BaseModel): 1619 type: Literal["CheckStream"] 1620 stream_names: Optional[List[str]] = Field( 1621 None, 1622 description="Names of the streams to try reading from when running a check operation.", 1623 examples=[["users"], ["users", "contacts"]], 1624 title="Stream Names", 1625 ) 1626 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1629class IncrementingCountCursor(BaseModel): 1630 type: Literal["IncrementingCountCursor"] 1631 cursor_field: str = Field( 1632 ..., 1633 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1634 examples=["created_at", "{{ config['record_cursor'] }}"], 1635 title="Cursor Field", 1636 ) 1637 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1638 None, 1639 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1640 title="Allow Catalog Defined Cursor Field", 1641 ) 1642 start_value: Optional[Union[str, int]] = Field( 1643 None, 1644 description="The value that determines the earliest record that should be synced.", 1645 examples=[0, "{{ config['start_value'] }}"], 1646 title="Start Value", 1647 ) 1648 start_value_option: Optional[RequestOption] = Field( 1649 None, 1650 description="Optionally configures how the start value will be sent in requests to the source API.", 1651 title="Inject Start Value Into Outgoing HTTP Request", 1652 ) 1653 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1656class DatetimeBasedCursor(BaseModel): 1657 type: Literal["DatetimeBasedCursor"] 1658 clamping: Optional[Clamping] = Field( 1659 None, 1660 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1661 title="Date Range Clamping", 1662 ) 1663 cursor_field: str = Field( 1664 ..., 1665 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1666 examples=["created_at", "{{ config['record_cursor'] }}"], 1667 title="Cursor Field", 1668 ) 1669 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1670 None, 1671 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1672 title="Allow Catalog Defined Cursor Field", 1673 ) 1674 cursor_datetime_formats: Optional[List[str]] = Field( 1675 None, 1676 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1677 examples=[ 1678 "%Y-%m-%d", 1679 "%Y-%m-%d %H:%M:%S", 1680 "%Y-%m-%dT%H:%M:%S", 1681 "%Y-%m-%dT%H:%M:%SZ", 1682 "%Y-%m-%dT%H:%M:%S%z", 1683 "%Y-%m-%dT%H:%M:%S.%fZ", 1684 "%Y-%m-%dT%H:%M:%S.%f%z", 1685 "%Y-%m-%d %H:%M:%S.%f+00:00", 1686 "%s", 1687 "%ms", 1688 ], 1689 title="Cursor Datetime Formats", 1690 ) 1691 start_datetime: Union[MinMaxDatetime, str] = Field( 1692 ..., 1693 description="The datetime that determines the earliest record that should be synced.", 1694 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1695 title="Start Datetime", 1696 ) 1697 start_time_option: Optional[RequestOption] = Field( 1698 None, 1699 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1700 title="Inject Start Time Into Outgoing HTTP Request", 1701 ) 1702 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1703 None, 1704 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1705 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1706 title="End Datetime", 1707 ) 1708 end_time_option: Optional[RequestOption] = Field( 1709 None, 1710 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1711 title="Inject End Time Into Outgoing HTTP Request", 1712 ) 1713 datetime_format: str = Field( 1714 ..., 1715 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1716 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1717 title="Outgoing Datetime Format", 1718 ) 1719 cursor_granularity: Optional[str] = Field( 1720 None, 1721 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1722 examples=["PT1S"], 1723 title="Cursor Granularity", 1724 ) 1725 is_data_feed: Optional[bool] = Field( 1726 None, 1727 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1728 title="Data Feed API", 1729 ) 1730 is_client_side_incremental: Optional[bool] = Field( 1731 None, 1732 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1733 title="Client-side Incremental Filtering", 1734 ) 1735 is_compare_strictly: Optional[bool] = Field( 1736 False, 1737 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1738 title="Strict Start-End Time Comparison", 1739 ) 1740 global_substream_cursor: Optional[bool] = Field( 1741 False, 1742 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1743 title="Global Substream Cursor", 1744 ) 1745 lookback_window: Optional[str] = Field( 1746 None, 1747 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1748 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1749 title="Lookback Window", 1750 ) 1751 partition_field_end: Optional[str] = Field( 1752 None, 1753 description="Name of the partition start time field.", 1754 examples=["ending_time"], 1755 title="Partition Field End", 1756 ) 1757 partition_field_start: Optional[str] = Field( 1758 None, 1759 description="Name of the partition end time field.", 1760 examples=["starting_time"], 1761 title="Partition Field Start", 1762 ) 1763 step: Optional[str] = Field( 1764 None, 1765 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1766 examples=["P1W", "{{ config['step_increment'] }}"], 1767 title="Step", 1768 ) 1769 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1772class JwtAuthenticator(BaseModel): 1773 type: Literal["JwtAuthenticator"] 1774 secret_key: str = Field( 1775 ..., 1776 description="Secret used to sign the JSON web token.", 1777 examples=["{{ config['secret_key'] }}"], 1778 title="Secret Key", 1779 ) 1780 base64_encode_secret_key: Optional[bool] = Field( 1781 False, 1782 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1783 title="Base64-encode Secret Key", 1784 ) 1785 algorithm: Algorithm = Field( 1786 ..., 1787 description="Algorithm used to sign the JSON web token.", 1788 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1789 title="Algorithm", 1790 ) 1791 token_duration: Optional[int] = Field( 1792 1200, 1793 description="The amount of time in seconds a JWT token can be valid after being issued.", 1794 examples=[1200, 3600], 1795 title="Token Duration", 1796 ) 1797 header_prefix: Optional[str] = Field( 1798 None, 1799 description="The prefix to be used within the Authentication header.", 1800 examples=["Bearer", "Basic"], 1801 title="Header Prefix", 1802 ) 1803 jwt_headers: Optional[JwtHeaders] = Field( 1804 None, 1805 description="JWT headers used when signing JSON web token.", 1806 title="JWT Headers", 1807 ) 1808 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1809 None, 1810 description="Additional headers to be included with the JWT headers object.", 1811 title="Additional JWT Headers", 1812 ) 1813 jwt_payload: Optional[JwtPayload] = Field( 1814 None, 1815 description="JWT Payload used when signing JSON web token.", 1816 title="JWT Payload", 1817 ) 1818 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1819 None, 1820 description="Additional properties to be added to the JWT payload.", 1821 title="Additional JWT Payload Properties", 1822 ) 1823 passphrase: Optional[str] = Field( 1824 None, 1825 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1826 examples=["{{ config['passphrase'] }}"], 1827 title="Passphrase", 1828 ) 1829 request_option: Optional[RequestOption] = Field( 1830 None, 1831 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1832 title="Request Option", 1833 ) 1834 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1837class OAuthAuthenticator(BaseModel): 1838 type: Literal["OAuthAuthenticator"] 1839 client_id_name: Optional[str] = Field( 1840 "client_id", 1841 description="The name of the property to use to refresh the `access_token`.", 1842 examples=["custom_app_id"], 1843 title="Client ID Property Name", 1844 ) 1845 client_id: Optional[str] = Field( 1846 None, 1847 description="The OAuth client ID. Fill it in the user inputs.", 1848 examples=[ 1849 "{{ config['client_id'] }}", 1850 "{{ config['credentials']['client_id }}", 1851 ], 1852 title="Client ID", 1853 ) 1854 client_secret_name: Optional[str] = Field( 1855 "client_secret", 1856 description="The name of the property to use to refresh the `access_token`.", 1857 examples=["custom_app_secret"], 1858 title="Client Secret Property Name", 1859 ) 1860 client_secret: Optional[str] = Field( 1861 None, 1862 description="The OAuth client secret. Fill it in the user inputs.", 1863 examples=[ 1864 "{{ config['client_secret'] }}", 1865 "{{ config['credentials']['client_secret }}", 1866 ], 1867 title="Client Secret", 1868 ) 1869 refresh_token_name: Optional[str] = Field( 1870 "refresh_token", 1871 description="The name of the property to use to refresh the `access_token`.", 1872 examples=["custom_app_refresh_value"], 1873 title="Refresh Token Property Name", 1874 ) 1875 refresh_token: Optional[str] = Field( 1876 None, 1877 description="Credential artifact used to get a new access token.", 1878 examples=[ 1879 "{{ config['refresh_token'] }}", 1880 "{{ config['credentials]['refresh_token'] }}", 1881 ], 1882 title="Refresh Token", 1883 ) 1884 token_refresh_endpoint: Optional[str] = Field( 1885 None, 1886 description="The full URL to call to obtain a new access token.", 1887 examples=["https://connect.squareup.com/oauth2/token"], 1888 title="Token Refresh Endpoint", 1889 ) 1890 access_token_name: Optional[str] = Field( 1891 "access_token", 1892 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1893 examples=["access_token"], 1894 title="Access Token Property Name", 1895 ) 1896 access_token_value: Optional[str] = Field( 1897 None, 1898 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1899 examples=["secret_access_token_value"], 1900 title="Access Token Value", 1901 ) 1902 expires_in_name: Optional[str] = Field( 1903 "expires_in", 1904 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1905 examples=["expires_in"], 1906 title="Token Expiry Property Name", 1907 ) 1908 grant_type_name: Optional[str] = Field( 1909 "grant_type", 1910 description="The name of the property to use to refresh the `access_token`.", 1911 examples=["custom_grant_type"], 1912 title="Grant Type Property Name", 1913 ) 1914 grant_type: Optional[str] = Field( 1915 "refresh_token", 1916 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1917 examples=["refresh_token", "client_credentials"], 1918 title="Grant Type", 1919 ) 1920 refresh_request_body: Optional[Dict[str, Any]] = Field( 1921 None, 1922 description="Body of the request sent to get a new access token.", 1923 examples=[ 1924 { 1925 "applicationId": "{{ config['application_id'] }}", 1926 "applicationSecret": "{{ config['application_secret'] }}", 1927 "token": "{{ config['token'] }}", 1928 } 1929 ], 1930 title="Refresh Request Body", 1931 ) 1932 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1933 None, 1934 description="Headers of the request sent to get a new access token.", 1935 examples=[ 1936 { 1937 "Authorization": "<AUTH_TOKEN>", 1938 "Content-Type": "application/x-www-form-urlencoded", 1939 } 1940 ], 1941 title="Refresh Request Headers", 1942 ) 1943 scopes: Optional[List[str]] = Field( 1944 None, 1945 description="List of scopes that should be granted to the access token.", 1946 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1947 title="Scopes", 1948 ) 1949 token_expiry_date: Optional[str] = Field( 1950 None, 1951 description="The access token expiry date.", 1952 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1953 title="Token Expiry Date", 1954 ) 1955 token_expiry_date_format: Optional[str] = Field( 1956 None, 1957 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1958 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1959 title="Token Expiry Date Format", 1960 ) 1961 refresh_token_error_status_codes: Optional[List[int]] = Field( 1962 None, 1963 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1964 examples=[[400, 500]], 1965 title="Refresh Token Error Status Codes", 1966 ) 1967 refresh_token_error_key: Optional[str] = Field( 1968 None, 1969 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1970 examples=["error"], 1971 title="Refresh Token Error Key", 1972 ) 1973 refresh_token_error_values: Optional[List[str]] = Field( 1974 None, 1975 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1976 examples=[["invalid_grant", "invalid_permissions"]], 1977 title="Refresh Token Error Values", 1978 ) 1979 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1980 None, 1981 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1982 title="Refresh Token Updater", 1983 ) 1984 profile_assertion: Optional[JwtAuthenticator] = Field( 1985 None, 1986 description="The authenticator being used to authenticate the client authenticator.", 1987 title="Profile Assertion", 1988 ) 1989 use_profile_assertion: Optional[bool] = Field( 1990 False, 1991 description="Enable using profile assertion as a flow for OAuth authorization.", 1992 title="Use Profile Assertion", 1993 ) 1994 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1997class FixedWindowCallRatePolicy(BaseModel): 1998 class Config: 1999 extra = Extra.allow 2000 2001 type: Literal["FixedWindowCallRatePolicy"] 2002 period: str = Field( 2003 ..., description="The time interval for the rate limit window.", title="Period" 2004 ) 2005 call_limit: int = Field( 2006 ..., 2007 description="The maximum number of calls allowed within the period.", 2008 title="Call Limit", 2009 ) 2010 matchers: List[HttpRequestRegexMatcher] = Field( 2011 ..., 2012 description="List of matchers that define which requests this policy applies to.", 2013 title="Matchers", 2014 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2017class MovingWindowCallRatePolicy(BaseModel): 2018 class Config: 2019 extra = Extra.allow 2020 2021 type: Literal["MovingWindowCallRatePolicy"] 2022 rates: List[Rate] = Field( 2023 ..., 2024 description="List of rates that define the call limits for different time intervals.", 2025 title="Rates", 2026 ) 2027 matchers: List[HttpRequestRegexMatcher] = Field( 2028 ..., 2029 description="List of matchers that define which requests this policy applies to.", 2030 title="Matchers", 2031 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2034class UnlimitedCallRatePolicy(BaseModel): 2035 class Config: 2036 extra = Extra.allow 2037 2038 type: Literal["UnlimitedCallRatePolicy"] 2039 matchers: List[HttpRequestRegexMatcher] = Field( 2040 ..., 2041 description="List of matchers that define which requests this policy applies to.", 2042 title="Matchers", 2043 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2046class DefaultErrorHandler(BaseModel): 2047 type: Literal["DefaultErrorHandler"] 2048 backoff_strategies: Optional[ 2049 List[ 2050 Union[ 2051 ConstantBackoffStrategy, 2052 ExponentialBackoffStrategy, 2053 WaitTimeFromHeader, 2054 WaitUntilTimeFromHeader, 2055 CustomBackoffStrategy, 2056 ] 2057 ] 2058 ] = Field( 2059 None, 2060 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2061 title="Backoff Strategies", 2062 ) 2063 max_retries: Optional[Union[int, str]] = Field( 2064 5, 2065 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2066 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2067 title="Max Retry Count", 2068 ) 2069 response_filters: Optional[List[HttpResponseFilter]] = Field( 2070 None, 2071 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2072 title="Response Filters", 2073 ) 2074 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2077class DefaultPaginator(BaseModel): 2078 type: Literal["DefaultPaginator"] 2079 pagination_strategy: Union[ 2080 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2081 ] = Field( 2082 ..., 2083 description="Strategy defining how records are paginated.", 2084 title="Pagination Strategy", 2085 ) 2086 page_size_option: Optional[RequestOption] = Field( 2087 None, title="Inject Page Size Into Outgoing HTTP Request" 2088 ) 2089 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2090 None, 2091 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2092 title="Inject Page Token Into Outgoing HTTP Request", 2093 ) 2094 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2097class RecordExpander(BaseModel): 2098 type: Literal["RecordExpander"] 2099 expand_records_from_field: List[str] = Field( 2100 ..., 2101 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2102 examples=[ 2103 ["lines", "data"], 2104 ["items"], 2105 ["nested", "array"], 2106 ["sections", "*", "items"], 2107 ], 2108 title="Expand Records From Field", 2109 ) 2110 remain_original_record: Optional[bool] = Field( 2111 False, 2112 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2113 title="Remain Original Record", 2114 ) 2115 on_no_records: Optional[OnNoRecords] = Field( 2116 OnNoRecords.skip, 2117 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2118 title="On No Records", 2119 ) 2120 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2123class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2124 type: Literal["ApiKey"] 2125 inject_into: RequestOption = Field( 2126 ..., 2127 description="Configure how the API Key will be sent in requests to the source API.", 2128 examples=[ 2129 {"inject_into": "header", "field_name": "Authorization"}, 2130 {"inject_into": "request_parameter", "field_name": "authKey"}, 2131 ], 2132 title="Inject API Key Into Outgoing HTTP Request", 2133 ) 2134 api_token: Optional[str] = Field( 2135 "{{ session_token }}", 2136 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2137 examples=[ 2138 "{{ session_token }}", 2139 "Token {{ session_token }}", 2140 "Bearer {{ session_token }}", 2141 ], 2142 title="API Token Template", 2143 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2146class JsonSchemaPropertySelector(BaseModel): 2147 type: Literal["JsonSchemaPropertySelector"] 2148 transformations: Optional[ 2149 List[ 2150 Union[ 2151 AddFields, 2152 RemoveFields, 2153 KeysToLower, 2154 KeysToSnakeCase, 2155 FlattenFields, 2156 DpathFlattenFields, 2157 KeysReplace, 2158 CustomTransformation, 2159 ] 2160 ] 2161 ] = Field( 2162 None, 2163 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2164 title="Transformations", 2165 ) 2166 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2169class ListPartitionRouter(BaseModel): 2170 type: Literal["ListPartitionRouter"] 2171 cursor_field: str = Field( 2172 ..., 2173 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2174 examples=["section", "{{ config['section_key'] }}"], 2175 title="Current Partition Value Identifier", 2176 ) 2177 values: Union[str, List[str]] = Field( 2178 ..., 2179 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2180 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2181 title="Partition Values", 2182 ) 2183 request_option: Optional[RequestOption] = Field( 2184 None, 2185 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2186 title="Inject Partition Value Into Outgoing HTTP Request", 2187 ) 2188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2191class PaginationReset(BaseModel): 2192 type: Literal["PaginationReset"] 2193 action: Action1 2194 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2197class GzipDecoder(BaseModel): 2198 type: Literal["GzipDecoder"] 2199 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2202class RequestBodyGraphQL(BaseModel): 2203 type: Literal["RequestBodyGraphQL"] 2204 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2207class DpathValidator(BaseModel): 2208 type: Literal["DpathValidator"] 2209 field_path: List[str] = Field( 2210 ..., 2211 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2212 examples=[ 2213 ["data"], 2214 ["data", "records"], 2215 ["data", "{{ parameters.name }}"], 2216 ["data", "*", "record"], 2217 ], 2218 title="Field Path", 2219 ) 2220 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2221 ..., 2222 description="The condition that the specified config value will be evaluated against", 2223 title="Validation Strategy", 2224 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2227class PredicateValidator(BaseModel): 2228 type: Literal["PredicateValidator"] 2229 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2230 ..., 2231 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2232 examples=[ 2233 "test-value", 2234 "{{ config['api_version'] }}", 2235 "{{ config['tenant_id'] }}", 2236 123, 2237 ], 2238 title="Value", 2239 ) 2240 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2241 ..., 2242 description="The validation strategy to apply to the value.", 2243 title="Validation Strategy", 2244 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2247class ConfigAddFields(BaseModel): 2248 type: Literal["ConfigAddFields"] 2249 fields: List[AddedFieldDefinition] = Field( 2250 ..., 2251 description="A list of transformations (path and corresponding value) that will be added to the config.", 2252 title="Fields", 2253 ) 2254 condition: Optional[str] = Field( 2255 "", 2256 description="Fields will be added if expression is evaluated to True.", 2257 examples=[ 2258 "{{ config['environemnt'] == 'sandbox' }}", 2259 "{{ property is integer }}", 2260 "{{ property|length > 5 }}", 2261 "{{ property == 'some_string_to_match' }}", 2262 ], 2263 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2266class CompositeErrorHandler(BaseModel): 2267 type: Literal["CompositeErrorHandler"] 2268 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2269 Field( 2270 ..., 2271 description="List of error handlers to iterate on to determine how to handle a failed response.", 2272 title="Error Handlers", 2273 ) 2274 ) 2275 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2278class HTTPAPIBudget(BaseModel): 2279 class Config: 2280 extra = Extra.allow 2281 2282 type: Literal["HTTPAPIBudget"] 2283 policies: List[ 2284 Union[ 2285 FixedWindowCallRatePolicy, 2286 MovingWindowCallRatePolicy, 2287 UnlimitedCallRatePolicy, 2288 ] 2289 ] = Field( 2290 ..., 2291 description="List of call rate policies that define how many calls are allowed.", 2292 title="Policies", 2293 ) 2294 ratelimit_reset_header: Optional[str] = Field( 2295 "ratelimit-reset", 2296 description="The HTTP response header name that indicates when the rate limit resets.", 2297 title="Rate Limit Reset Header", 2298 ) 2299 ratelimit_remaining_header: Optional[str] = Field( 2300 "ratelimit-remaining", 2301 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2302 title="Rate Limit Remaining Header", 2303 ) 2304 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2305 [429], 2306 description="List of HTTP status codes that indicate a rate limit has been hit.", 2307 title="Status Codes for Rate Limit Hit", 2308 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2311class DpathExtractor(BaseModel): 2312 type: Literal["DpathExtractor"] 2313 field_path: List[str] = Field( 2314 ..., 2315 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2316 examples=[ 2317 ["data"], 2318 ["data", "records"], 2319 ["data", "{{ parameters.name }}"], 2320 ["data", "*", "record"], 2321 ], 2322 title="Field Path", 2323 ) 2324 record_expander: Optional[RecordExpander] = Field( 2325 None, 2326 description="Optional component to expand records by extracting items from nested array fields.", 2327 title="Record Expander", 2328 ) 2329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2332class ZipfileDecoder(BaseModel): 2333 class Config: 2334 extra = Extra.allow 2335 2336 type: Literal["ZipfileDecoder"] 2337 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2338 ..., 2339 description="Parser to parse the decompressed data from the zipfile(s).", 2340 title="Parser", 2341 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2344class RecordSelector(BaseModel): 2345 type: Literal["RecordSelector"] 2346 extractor: Union[DpathExtractor, CustomRecordExtractor] 2347 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2348 None, 2349 description="Responsible for filtering records to be emitted by the Source.", 2350 title="Record Filter", 2351 ) 2352 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2353 None, 2354 description="Responsible for normalization according to the schema.", 2355 title="Schema Normalization", 2356 ) 2357 transform_before_filtering: Optional[bool] = Field( 2358 None, 2359 description="If true, transformation will be applied before record filtering.", 2360 title="Transform Before Filtering", 2361 ) 2362 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2365class ConfigMigration(BaseModel): 2366 type: Literal["ConfigMigration"] 2367 description: Optional[str] = Field( 2368 None, description="The description/purpose of the config migration." 2369 ) 2370 transformations: List[ 2371 Union[ 2372 ConfigRemapField, 2373 ConfigAddFields, 2374 ConfigRemoveFields, 2375 CustomConfigTransformation, 2376 ] 2377 ] = Field( 2378 ..., 2379 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2380 title="Transformations", 2381 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2384class ConfigNormalizationRules(BaseModel): 2385 class Config: 2386 extra = Extra.forbid 2387 2388 type: Literal["ConfigNormalizationRules"] 2389 config_migrations: Optional[List[ConfigMigration]] = Field( 2390 [], 2391 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2392 title="Config Migrations", 2393 ) 2394 transformations: Optional[ 2395 List[ 2396 Union[ 2397 ConfigRemapField, 2398 ConfigAddFields, 2399 ConfigRemoveFields, 2400 CustomConfigTransformation, 2401 ] 2402 ] 2403 ] = Field( 2404 [], 2405 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2406 title="Transformations", 2407 ) 2408 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2409 [], 2410 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2411 title="Validations", 2412 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2415class Spec(BaseModel): 2416 type: Literal["Spec"] 2417 connection_specification: Dict[str, Any] = Field( 2418 ..., 2419 description="A connection specification describing how a the connector can be configured.", 2420 title="Connection Specification", 2421 ) 2422 documentation_url: Optional[str] = Field( 2423 None, 2424 description="URL of the connector's documentation page.", 2425 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2426 title="Documentation URL", 2427 ) 2428 advanced_auth: Optional[AuthFlow] = Field( 2429 None, 2430 description="Advanced specification for configuring the authentication flow.", 2431 title="Advanced Auth", 2432 ) 2433 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2434 None, title="Config Normalization Rules" 2435 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2438class DeclarativeSource1(BaseModel): 2439 class Config: 2440 extra = Extra.forbid 2441 2442 type: Literal["DeclarativeSource"] 2443 check: Union[CheckStream, CheckDynamicStream] 2444 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2445 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2446 version: str = Field( 2447 ..., 2448 description="The version of the Airbyte CDK used to build and test the source.", 2449 ) 2450 schemas: Optional[Schemas] = None 2451 definitions: Optional[Dict[str, Any]] = None 2452 spec: Optional[Spec] = None 2453 concurrency_level: Optional[ConcurrencyLevel] = None 2454 api_budget: Optional[HTTPAPIBudget] = None 2455 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2456 None, 2457 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2458 title="Stream Groups", 2459 ) 2460 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2461 None, 2462 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2463 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2464 title="Maximum Concurrent Asynchronous Jobs", 2465 ) 2466 metadata: Optional[Dict[str, Any]] = Field( 2467 None, 2468 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2469 ) 2470 description: Optional[str] = Field( 2471 None, 2472 description="A description of the connector. It will be presented on the Source documentation page.", 2473 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2476class DeclarativeSource2(BaseModel): 2477 class Config: 2478 extra = Extra.forbid 2479 2480 type: Literal["DeclarativeSource"] 2481 check: Union[CheckStream, CheckDynamicStream] 2482 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2483 None 2484 ) 2485 dynamic_streams: List[DynamicDeclarativeStream] 2486 version: str = Field( 2487 ..., 2488 description="The version of the Airbyte CDK used to build and test the source.", 2489 ) 2490 schemas: Optional[Schemas] = None 2491 definitions: Optional[Dict[str, Any]] = None 2492 spec: Optional[Spec] = None 2493 concurrency_level: Optional[ConcurrencyLevel] = None 2494 api_budget: Optional[HTTPAPIBudget] = None 2495 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2496 None, 2497 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2498 title="Stream Groups", 2499 ) 2500 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2501 None, 2502 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2503 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2504 title="Maximum Concurrent Asynchronous Jobs", 2505 ) 2506 metadata: Optional[Dict[str, Any]] = Field( 2507 None, 2508 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2509 ) 2510 description: Optional[str] = Field( 2511 None, 2512 description="A description of the connector. It will be presented on the Source documentation page.", 2513 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2516class DeclarativeSource(BaseModel): 2517 class Config: 2518 extra = Extra.forbid 2519 2520 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2521 ..., 2522 description="An API source that extracts data according to its declarative components.", 2523 title="DeclarativeSource", 2524 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2527class SelectiveAuthenticator(BaseModel): 2528 class Config: 2529 extra = Extra.allow 2530 2531 type: Literal["SelectiveAuthenticator"] 2532 authenticator_selection_path: List[str] = Field( 2533 ..., 2534 description="Path of the field in config with selected authenticator name", 2535 examples=[["auth"], ["auth", "type"]], 2536 title="Authenticator Selection Path", 2537 ) 2538 authenticators: Dict[ 2539 str, 2540 Union[ 2541 ApiKeyAuthenticator, 2542 BasicHttpAuthenticator, 2543 BearerAuthenticator, 2544 OAuthAuthenticator, 2545 JwtAuthenticator, 2546 SessionTokenAuthenticator, 2547 LegacySessionTokenAuthenticator, 2548 CustomAuthenticator, 2549 NoAuth, 2550 ], 2551 ] = Field( 2552 ..., 2553 description="Authenticators to select from.", 2554 examples=[ 2555 { 2556 "authenticators": { 2557 "token": "#/definitions/ApiKeyAuthenticator", 2558 "oauth": "#/definitions/OAuthAuthenticator", 2559 "jwt": "#/definitions/JwtAuthenticator", 2560 } 2561 } 2562 ], 2563 title="Authenticators", 2564 ) 2565 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2568class ConditionalStreams(BaseModel): 2569 type: Literal["ConditionalStreams"] 2570 condition: str = Field( 2571 ..., 2572 description="Condition that will be evaluated to determine if a set of streams should be available.", 2573 examples=["{{ config['is_sandbox'] }}"], 2574 title="Condition", 2575 ) 2576 streams: List[DeclarativeStream] = Field( 2577 ..., 2578 description="Streams that will be used during an operation based on the condition.", 2579 title="Streams", 2580 ) 2581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2584class FileUploader(BaseModel): 2585 type: Literal["FileUploader"] 2586 requester: Union[HttpRequester, CustomRequester] = Field( 2587 ..., 2588 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2589 ) 2590 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2591 ..., 2592 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2593 ) 2594 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2595 None, 2596 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2597 ) 2598 filename_extractor: Optional[str] = Field( 2599 None, 2600 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2601 examples=[ 2602 "{{ record.id }}/{{ record.file_name }}/", 2603 "{{ record.id }}_{{ record.file_name }}/", 2604 ], 2605 ) 2606 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2609class DeclarativeStream(BaseModel): 2610 class Config: 2611 extra = Extra.allow 2612 2613 type: Literal["DeclarativeStream"] 2614 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2615 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2616 ..., 2617 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2618 title="Retriever", 2619 ) 2620 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2621 None, 2622 description="Component used to fetch data incrementally based on a time field in the data.", 2623 title="Incremental Sync", 2624 ) 2625 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2626 schema_loader: Optional[ 2627 Union[ 2628 InlineSchemaLoader, 2629 DynamicSchemaLoader, 2630 JsonFileSchemaLoader, 2631 List[ 2632 Union[ 2633 InlineSchemaLoader, 2634 DynamicSchemaLoader, 2635 JsonFileSchemaLoader, 2636 CustomSchemaLoader, 2637 ] 2638 ], 2639 CustomSchemaLoader, 2640 ] 2641 ] = Field( 2642 None, 2643 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2644 title="Schema Loader", 2645 ) 2646 transformations: Optional[ 2647 List[ 2648 Union[ 2649 AddFields, 2650 RemoveFields, 2651 KeysToLower, 2652 KeysToSnakeCase, 2653 FlattenFields, 2654 DpathFlattenFields, 2655 KeysReplace, 2656 CustomTransformation, 2657 ] 2658 ] 2659 ] = Field( 2660 None, 2661 description="A list of transformations to be applied to each output record.", 2662 title="Transformations", 2663 ) 2664 state_migrations: Optional[ 2665 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2666 ] = Field( 2667 [], 2668 description="Array of state migrations to be applied on the input state", 2669 title="State Migrations", 2670 ) 2671 file_uploader: Optional[FileUploader] = Field( 2672 None, 2673 description="(experimental) Describes how to fetch a file", 2674 title="File Uploader", 2675 ) 2676 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2679class SessionTokenAuthenticator(BaseModel): 2680 type: Literal["SessionTokenAuthenticator"] 2681 login_requester: HttpRequester = Field( 2682 ..., 2683 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2684 examples=[ 2685 { 2686 "type": "HttpRequester", 2687 "url_base": "https://my_api.com", 2688 "path": "/login", 2689 "authenticator": { 2690 "type": "BasicHttpAuthenticator", 2691 "username": "{{ config.username }}", 2692 "password": "{{ config.password }}", 2693 }, 2694 } 2695 ], 2696 title="Login Requester", 2697 ) 2698 session_token_path: List[str] = Field( 2699 ..., 2700 description="The path in the response body returned from the login requester to the session token.", 2701 examples=[["access_token"], ["result", "token"]], 2702 title="Session Token Path", 2703 ) 2704 expiration_duration: Optional[str] = Field( 2705 None, 2706 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2707 examples=["PT1H", "P1D"], 2708 title="Expiration Duration", 2709 ) 2710 request_authentication: Union[ 2711 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2712 ] = Field( 2713 ..., 2714 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2715 title="Data Request Authentication", 2716 ) 2717 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2718 None, description="Component used to decode the response.", title="Decoder" 2719 ) 2720 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2723class HttpRequester(BaseModelWithDeprecations): 2724 type: Literal["HttpRequester"] 2725 url_base: Optional[str] = Field( 2726 None, 2727 deprecated=True, 2728 deprecation_message="Use `url` field instead.", 2729 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2730 examples=[ 2731 "https://connect.squareup.com/v2", 2732 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2733 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2734 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2735 ], 2736 title="API Base URL", 2737 ) 2738 url: Optional[str] = Field( 2739 None, 2740 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2741 examples=[ 2742 "https://connect.squareup.com/v2", 2743 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2744 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2745 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2746 ], 2747 title="API Endpoint URL", 2748 ) 2749 path: Optional[str] = Field( 2750 None, 2751 deprecated=True, 2752 deprecation_message="Use `url` field instead.", 2753 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2754 examples=[ 2755 "/products", 2756 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2757 "/trades/{{ config['symbol_id'] }}/history", 2758 ], 2759 title="URL Path", 2760 ) 2761 http_method: Optional[HttpMethod] = Field( 2762 HttpMethod.GET, 2763 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2764 examples=["GET", "POST"], 2765 title="HTTP Method", 2766 ) 2767 authenticator: Optional[ 2768 Union[ 2769 ApiKeyAuthenticator, 2770 BasicHttpAuthenticator, 2771 BearerAuthenticator, 2772 OAuthAuthenticator, 2773 JwtAuthenticator, 2774 SessionTokenAuthenticator, 2775 SelectiveAuthenticator, 2776 CustomAuthenticator, 2777 NoAuth, 2778 LegacySessionTokenAuthenticator, 2779 ] 2780 ] = Field( 2781 None, 2782 description="Authentication method to use for requests sent to the API.", 2783 title="Authenticator", 2784 ) 2785 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2786 None, 2787 deprecated=True, 2788 deprecation_message="Use `query_properties` field instead.", 2789 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2790 title="Fetch Properties from Endpoint", 2791 ) 2792 query_properties: Optional[QueryProperties] = Field( 2793 None, 2794 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2795 title="Query Properties", 2796 ) 2797 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2798 None, 2799 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2800 examples=[ 2801 {"unit": "day"}, 2802 { 2803 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2804 }, 2805 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2806 {"sort_by[asc]": "updated_at"}, 2807 ], 2808 title="Query Parameters", 2809 ) 2810 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2811 None, 2812 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2813 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2814 title="Request Headers", 2815 ) 2816 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2817 None, 2818 deprecated=True, 2819 deprecation_message="Use `request_body` field instead.", 2820 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2821 examples=[ 2822 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2823 ], 2824 title="Request Body Payload (Non-JSON)", 2825 ) 2826 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2827 None, 2828 deprecated=True, 2829 deprecation_message="Use `request_body` field instead.", 2830 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2831 examples=[ 2832 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2833 {"key": "{{ config['value'] }}"}, 2834 {"sort": {"field": "updated_at", "order": "ascending"}}, 2835 ], 2836 title="Request Body JSON Payload", 2837 ) 2838 request_body: Optional[ 2839 Union[ 2840 RequestBodyPlainText, 2841 RequestBodyUrlEncodedForm, 2842 RequestBodyJsonObject, 2843 RequestBodyGraphQL, 2844 ] 2845 ] = Field( 2846 None, 2847 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2848 title="Request Body", 2849 ) 2850 error_handler: Optional[ 2851 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2852 ] = Field( 2853 None, 2854 description="Error handler component that defines how to handle errors.", 2855 title="Error Handler", 2856 ) 2857 use_cache: Optional[bool] = Field( 2858 False, 2859 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2860 title="Use Cache", 2861 ) 2862 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2865class DynamicSchemaLoader(BaseModel): 2866 type: Literal["DynamicSchemaLoader"] 2867 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2868 ..., 2869 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2870 title="Retriever", 2871 ) 2872 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2873 None, 2874 description="Responsible for filtering fields to be added to json schema.", 2875 title="Schema Filter", 2876 ) 2877 schema_transformations: Optional[ 2878 List[ 2879 Union[ 2880 AddFields, 2881 RemoveFields, 2882 KeysToLower, 2883 KeysToSnakeCase, 2884 FlattenFields, 2885 DpathFlattenFields, 2886 KeysReplace, 2887 CustomTransformation, 2888 ] 2889 ] 2890 ] = Field( 2891 None, 2892 description="A list of transformations to be applied to the schema.", 2893 title="Schema Transformations", 2894 ) 2895 schema_type_identifier: SchemaTypeIdentifier 2896 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2899class ParentStreamConfig(BaseModel): 2900 type: Literal["ParentStreamConfig"] 2901 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2902 ..., description="Reference to the parent stream.", title="Parent Stream" 2903 ) 2904 parent_key: str = Field( 2905 ..., 2906 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2907 examples=["id", "{{ config['parent_record_id'] }}"], 2908 title="Parent Key", 2909 ) 2910 partition_field: str = Field( 2911 ..., 2912 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2913 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2914 title="Current Parent Key Value Identifier", 2915 ) 2916 request_option: Optional[RequestOption] = Field( 2917 None, 2918 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2919 title="Request Option", 2920 ) 2921 incremental_dependency: Optional[bool] = Field( 2922 False, 2923 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2924 title="Incremental Dependency", 2925 ) 2926 lazy_read_pointer: Optional[List[str]] = Field( 2927 [], 2928 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2929 title="Lazy Read Pointer", 2930 ) 2931 extra_fields: Optional[List[List[str]]] = Field( 2932 None, 2933 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2934 title="Extra Fields", 2935 ) 2936 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2939class PropertiesFromEndpoint(BaseModel): 2940 type: Literal["PropertiesFromEndpoint"] 2941 property_field_path: List[str] = Field( 2942 ..., 2943 description="Describes the path to the field that should be extracted", 2944 examples=[["name"]], 2945 ) 2946 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2947 ..., 2948 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2949 ) 2950 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2953class QueryProperties(BaseModel): 2954 type: Literal["QueryProperties"] 2955 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2956 ..., 2957 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2958 title="Property List", 2959 ) 2960 always_include_properties: Optional[List[str]] = Field( 2961 None, 2962 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2963 title="Always Include Properties", 2964 ) 2965 property_chunking: Optional[PropertyChunking] = Field( 2966 None, 2967 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2968 title="Property Chunking", 2969 ) 2970 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2971 None, 2972 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2973 title="Property Selector", 2974 ) 2975 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2978class StateDelegatingStream(BaseModel): 2979 type: Literal["StateDelegatingStream"] 2980 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2981 full_refresh_stream: DeclarativeStream = Field( 2982 ..., 2983 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2984 title="Full Refresh Stream", 2985 ) 2986 incremental_stream: DeclarativeStream = Field( 2987 ..., 2988 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2989 title="Incremental Stream", 2990 ) 2991 api_retention_period: Optional[str] = Field( 2992 None, 2993 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 2994 examples=["P30D", "P90D", "P1Y"], 2995 title="API Retention Period", 2996 ) 2997 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3000class SimpleRetriever(BaseModel): 3001 type: Literal["SimpleRetriever"] 3002 requester: Union[HttpRequester, CustomRequester] = Field( 3003 ..., 3004 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3005 ) 3006 decoder: Optional[ 3007 Union[ 3008 JsonDecoder, 3009 XmlDecoder, 3010 CsvDecoder, 3011 JsonlDecoder, 3012 GzipDecoder, 3013 IterableDecoder, 3014 ZipfileDecoder, 3015 CustomDecoder, 3016 ] 3017 ] = Field( 3018 None, 3019 description="Component decoding the response so records can be extracted.", 3020 title="HTTP Response Format", 3021 ) 3022 record_selector: RecordSelector = Field( 3023 ..., 3024 description="Component that describes how to extract records from a HTTP response.", 3025 ) 3026 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3027 None, 3028 description="Paginator component that describes how to navigate through the API's pages.", 3029 ) 3030 pagination_reset: Optional[PaginationReset] = Field( 3031 None, 3032 description="Describes what triggers pagination reset and how to handle it.", 3033 ) 3034 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3035 False, 3036 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3037 ) 3038 partition_router: Optional[ 3039 Union[ 3040 SubstreamPartitionRouter, 3041 ListPartitionRouter, 3042 GroupingPartitionRouter, 3043 CustomPartitionRouter, 3044 List[ 3045 Union[ 3046 SubstreamPartitionRouter, 3047 ListPartitionRouter, 3048 GroupingPartitionRouter, 3049 CustomPartitionRouter, 3050 ] 3051 ], 3052 ] 3053 ] = Field( 3054 None, 3055 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3056 title="Partition Router", 3057 ) 3058 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3061class AsyncRetriever(BaseModel): 3062 type: Literal["AsyncRetriever"] 3063 record_selector: RecordSelector = Field( 3064 ..., 3065 description="Component that describes how to extract records from a HTTP response.", 3066 ) 3067 status_mapping: AsyncJobStatusMap = Field( 3068 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3069 ) 3070 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3071 ..., description="Responsible for fetching the actual status of the async job." 3072 ) 3073 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3074 None, 3075 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3076 ) 3077 download_extractor: Optional[ 3078 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3079 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3080 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3081 ..., 3082 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3083 ) 3084 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3085 ..., 3086 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3087 ) 3088 polling_job_timeout: Optional[Union[int, str]] = Field( 3089 None, 3090 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3091 ) 3092 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3093 None, 3094 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3095 ge=1, 3096 ) 3097 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3098 None, 3099 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3100 ) 3101 download_requester: Union[HttpRequester, CustomRequester] = Field( 3102 ..., 3103 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3104 ) 3105 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3106 None, 3107 description="Paginator component that describes how to navigate through the API's pages during download.", 3108 ) 3109 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3110 None, 3111 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3112 ) 3113 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3114 None, 3115 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3116 ) 3117 partition_router: Optional[ 3118 Union[ 3119 ListPartitionRouter, 3120 SubstreamPartitionRouter, 3121 GroupingPartitionRouter, 3122 CustomPartitionRouter, 3123 List[ 3124 Union[ 3125 ListPartitionRouter, 3126 SubstreamPartitionRouter, 3127 GroupingPartitionRouter, 3128 CustomPartitionRouter, 3129 ] 3130 ], 3131 ] 3132 ] = Field( 3133 [], 3134 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3135 title="Partition Router", 3136 ) 3137 decoder: Optional[ 3138 Union[ 3139 CsvDecoder, 3140 GzipDecoder, 3141 JsonDecoder, 3142 JsonlDecoder, 3143 IterableDecoder, 3144 XmlDecoder, 3145 ZipfileDecoder, 3146 CustomDecoder, 3147 ] 3148 ] = Field( 3149 None, 3150 description="Component decoding the response so records can be extracted.", 3151 title="HTTP Response Format", 3152 ) 3153 download_decoder: Optional[ 3154 Union[ 3155 CsvDecoder, 3156 GzipDecoder, 3157 JsonDecoder, 3158 JsonlDecoder, 3159 IterableDecoder, 3160 XmlDecoder, 3161 ZipfileDecoder, 3162 CustomDecoder, 3163 ] 3164 ] = Field( 3165 None, 3166 description="Component decoding the download response so records can be extracted.", 3167 title="Download HTTP Response Format", 3168 ) 3169 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3176class StreamGroup(BaseModel): 3177 streams: List[str] = Field( 3178 ..., 3179 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3180 title="Streams", 3181 ) 3182 action: BlockSimultaneousSyncsAction = Field( 3183 ..., 3184 description="The action to apply to streams in this group.", 3185 title="Action", 3186 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3189class SubstreamPartitionRouter(BaseModel): 3190 type: Literal["SubstreamPartitionRouter"] 3191 parent_stream_configs: List[ParentStreamConfig] = Field( 3192 ..., 3193 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3194 title="Parent Stream Configs", 3195 ) 3196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3199class GroupingPartitionRouter(BaseModel): 3200 type: Literal["GroupingPartitionRouter"] 3201 group_size: int = Field( 3202 ..., 3203 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3204 examples=[10, 50], 3205 title="Group Size", 3206 ) 3207 underlying_partition_router: Union[ 3208 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3209 ] = Field( 3210 ..., 3211 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3212 title="Underlying Partition Router", 3213 ) 3214 deduplicate: Optional[bool] = Field( 3215 True, 3216 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3217 title="Deduplicate Partitions", 3218 ) 3219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3222class HttpComponentsResolver(BaseModel): 3223 type: Literal["HttpComponentsResolver"] 3224 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3225 ..., 3226 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3227 title="Retriever", 3228 ) 3229 components_mapping: List[ComponentMappingDefinition] 3230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3233class DynamicDeclarativeStream(BaseModel): 3234 type: Literal["DynamicDeclarativeStream"] 3235 name: Optional[str] = Field( 3236 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3237 ) 3238 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3239 ..., description="Reference to the stream template.", title="Stream Template" 3240 ) 3241 components_resolver: Union[ 3242 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3243 ] = Field( 3244 ..., 3245 description="Component resolve and populates stream templates with components values.", 3246 title="Components Resolver", 3247 ) 3248 use_parent_parameters: Optional[bool] = Field( 3249 True, 3250 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3251 title="Use Parent Parameters", 3252 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]