airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 jitter_range_in_seconds: Optional[float] = Field( 108 None, 109 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 110 examples=[15], 111 ge=0, 112 title="Jitter Range", 113 ) 114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 115 116 117class CursorPagination(BaseModel): 118 type: Literal["CursorPagination"] 119 cursor_value: str = Field( 120 ..., 121 description="Value of the cursor defining the next page to fetch.", 122 examples=[ 123 "{{ headers.link.next.cursor }}", 124 "{{ last_record['key'] }}", 125 "{{ response['nextPage'] }}", 126 ], 127 title="Cursor Value", 128 ) 129 page_size: Optional[Union[int, str]] = Field( 130 None, 131 description="The number of records to include in each pages.", 132 examples=[100, "{{ config['page_size'] }}"], 133 title="Page Size", 134 ) 135 stop_condition: Optional[str] = Field( 136 None, 137 description="Template string evaluating when to stop paginating.", 138 examples=[ 139 "{{ response.data.has_more is false }}", 140 "{{ 'next' not in headers['link'] }}", 141 ], 142 title="Stop Condition", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomAuthenticator(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomAuthenticator"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomBackoffStrategy(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomBackoffStrategy"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomBackoffStrategy"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomErrorHandler(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomErrorHandler"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomErrorHandler"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomPaginationStrategy(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomPaginationStrategy"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomPaginationStrategy"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordExtractor(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordExtractor"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomRecordExtractor"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRecordFilter(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRecordFilter"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomCustomRecordFilter"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRequester(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRequester"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRecordExtractor"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomRetriever(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomRetriever"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomRetriever"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomPartitionRouter(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomPartitionRouter"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomPartitionRouter"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaLoader(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaLoader"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 281 examples=["source_railz.components.MyCustomSchemaLoader"], 282 title="Class Name", 283 ) 284 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 285 286 287class CustomSchemaNormalization(BaseModel): 288 class Config: 289 extra = Extra.allow 290 291 type: Literal["CustomSchemaNormalization"] 292 class_name: str = Field( 293 ..., 294 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 295 examples=[ 296 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 297 ], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomStateMigration(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomStateMigration"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomStateMigration"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class CustomTransformation(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Literal["CustomTransformation"] 322 class_name: str = Field( 323 ..., 324 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 325 examples=["source_railz.components.MyCustomTransformation"], 326 title="Class Name", 327 ) 328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 329 330 331class LegacyToPerPartitionStateMigration(BaseModel): 332 class Config: 333 extra = Extra.allow 334 335 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 336 337 338class Clamping(BaseModel): 339 target: str = Field( 340 ..., 341 description="The period of time that datetime windows will be clamped by", 342 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 343 title="Target", 344 ) 345 target_details: Optional[Dict[str, Any]] = None 346 347 348class Algorithm(Enum): 349 HS256 = "HS256" 350 HS384 = "HS384" 351 HS512 = "HS512" 352 ES256 = "ES256" 353 ES256K = "ES256K" 354 ES384 = "ES384" 355 ES512 = "ES512" 356 RS256 = "RS256" 357 RS384 = "RS384" 358 RS512 = "RS512" 359 PS256 = "PS256" 360 PS384 = "PS384" 361 PS512 = "PS512" 362 EdDSA = "EdDSA" 363 364 365class JwtHeaders(BaseModel): 366 class Config: 367 extra = Extra.forbid 368 369 kid: Optional[str] = Field( 370 None, 371 description="Private key ID for user account.", 372 examples=["{{ config['kid'] }}"], 373 title="Key Identifier", 374 ) 375 typ: Optional[str] = Field( 376 "JWT", 377 description="The media type of the complete JWT.", 378 examples=["JWT"], 379 title="Type", 380 ) 381 cty: Optional[str] = Field( 382 None, 383 description="Content type of JWT header.", 384 examples=["JWT"], 385 title="Content Type", 386 ) 387 388 389class JwtPayload(BaseModel): 390 class Config: 391 extra = Extra.forbid 392 393 iss: Optional[str] = Field( 394 None, 395 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 396 examples=["{{ config['iss'] }}"], 397 title="Issuer", 398 ) 399 sub: Optional[str] = Field( 400 None, 401 description="The subject of the JWT. Commonly defined by the API.", 402 title="Subject", 403 ) 404 aud: Optional[str] = Field( 405 None, 406 description="The recipient that the JWT is intended for. Commonly defined by the API.", 407 examples=["appstoreconnect-v1"], 408 title="Audience", 409 ) 410 411 412class RefreshTokenUpdater(BaseModel): 413 refresh_token_name: Optional[str] = Field( 414 "refresh_token", 415 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 416 examples=["refresh_token"], 417 title="Refresh Token Property Name", 418 ) 419 access_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "access_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "access_token"], ["access_token"]], 423 title="Config Path To Access Token", 424 ) 425 refresh_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "refresh_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "refresh_token"], ["refresh_token"]], 429 title="Config Path To Refresh Token", 430 ) 431 token_expiry_date_config_path: Optional[List[str]] = Field( 432 ["credentials", "token_expiry_date"], 433 description="Config path to the expiry date. Make sure actually exists in the config.", 434 examples=[["credentials", "token_expiry_date"]], 435 title="Config Path To Expiry Date", 436 ) 437 refresh_token_error_status_codes: Optional[List[int]] = Field( 438 [], 439 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 440 examples=[[400, 500]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 442 ) 443 refresh_token_error_key: Optional[str] = Field( 444 "", 445 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 446 examples=["error"], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 448 ) 449 refresh_token_error_values: Optional[List[str]] = Field( 450 [], 451 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 452 examples=[["invalid_grant", "invalid_permissions"]], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 454 ) 455 456 457class Rate(BaseModel): 458 class Config: 459 extra = Extra.allow 460 461 limit: Union[int, str] = Field( 462 ..., 463 description="The maximum number of calls allowed within the interval.", 464 title="Limit", 465 ) 466 interval: str = Field( 467 ..., 468 description="The time interval for the rate limit.", 469 examples=["PT1H", "P1D"], 470 title="Interval", 471 ) 472 473 474class HttpRequestRegexMatcher(BaseModel): 475 class Config: 476 extra = Extra.allow 477 478 method: Optional[str] = Field( 479 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 480 ) 481 url_base: Optional[str] = Field( 482 None, 483 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 484 title="URL Base", 485 ) 486 url_path_pattern: Optional[str] = Field( 487 None, 488 description="A regular expression pattern to match the URL path.", 489 title="URL Path Pattern", 490 ) 491 params: Optional[Dict[str, Any]] = Field( 492 None, description="The query parameters to match.", title="Parameters" 493 ) 494 headers: Optional[Dict[str, Any]] = Field( 495 None, description="The headers to match.", title="Headers" 496 ) 497 weight: Optional[Union[int, str]] = Field( 498 None, 499 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 500 title="Weight", 501 ) 502 503 504class ResponseToFileExtractor(BaseModel): 505 type: Literal["ResponseToFileExtractor"] 506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 507 508 509class OnNoRecords(Enum): 510 skip = "skip" 511 emit_parent = "emit_parent" 512 513 514class ExponentialBackoffStrategy(BaseModel): 515 type: Literal["ExponentialBackoffStrategy"] 516 factor: Optional[Union[float, str]] = Field( 517 5, 518 description="Multiplicative constant applied on each retry.", 519 examples=[5, 5.5, "10"], 520 title="Factor", 521 ) 522 jitter_range_in_seconds: Optional[float] = Field( 523 None, 524 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 525 examples=[2], 526 ge=0, 527 title="Jitter Range", 528 ) 529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 530 531 532class GroupByKeyMergeStrategy(BaseModel): 533 type: Literal["GroupByKeyMergeStrategy"] 534 key: Union[str, List[str]] = Field( 535 ..., 536 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 537 examples=["id", ["parent_id", "end_date"]], 538 title="Key", 539 ) 540 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 541 542 543class SessionTokenRequestBearerAuthenticator(BaseModel): 544 type: Literal["Bearer"] 545 546 547class HttpMethod(Enum): 548 GET = "GET" 549 POST = "POST" 550 551 552class Action(Enum): 553 SUCCESS = "SUCCESS" 554 FAIL = "FAIL" 555 RETRY = "RETRY" 556 IGNORE = "IGNORE" 557 RESET_PAGINATION = "RESET_PAGINATION" 558 RATE_LIMITED = "RATE_LIMITED" 559 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 560 561 562class FailureType(Enum): 563 system_error = "system_error" 564 config_error = "config_error" 565 transient_error = "transient_error" 566 567 568class HttpResponseFilter(BaseModel): 569 type: Literal["HttpResponseFilter"] 570 action: Optional[Action] = Field( 571 None, 572 description="Action to execute if a response matches the filter.", 573 examples=[ 574 "SUCCESS", 575 "FAIL", 576 "RETRY", 577 "IGNORE", 578 "RESET_PAGINATION", 579 "RATE_LIMITED", 580 "REFRESH_TOKEN_THEN_RETRY", 581 ], 582 title="Action", 583 ) 584 failure_type: Optional[FailureType] = Field( 585 None, 586 description="Failure type of traced exception if a response matches the filter.", 587 examples=["system_error", "config_error", "transient_error"], 588 title="Failure Type", 589 ) 590 error_message: Optional[str] = Field( 591 None, 592 description="Error Message to display if the response matches the filter.", 593 title="Error Message", 594 ) 595 error_message_contains: Optional[str] = Field( 596 None, 597 description="Match the response if its error message contains the substring.", 598 example=["This API operation is not enabled for this site"], 599 title="Error Message Substring", 600 ) 601 http_codes: Optional[List[int]] = Field( 602 None, 603 description="Match the response if its HTTP code is included in this list.", 604 examples=[[420, 429], [500]], 605 title="HTTP Codes", 606 unique_items=True, 607 ) 608 predicate: Optional[str] = Field( 609 None, 610 description="Match the response if the predicate evaluates to true.", 611 examples=[ 612 "{{ 'Too much requests' in response }}", 613 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 614 ], 615 title="Predicate", 616 ) 617 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 618 619 620class ComplexFieldType(BaseModel): 621 field_type: str 622 items: Optional[Union[str, ComplexFieldType]] = None 623 624 625class TypesMap(BaseModel): 626 target_type: Union[str, List[str], ComplexFieldType] 627 current_type: Union[str, List[str]] 628 condition: Optional[str] = None 629 630 631class SchemaTypeIdentifier(BaseModel): 632 type: Optional[Literal["SchemaTypeIdentifier"]] = None 633 schema_pointer: Optional[List[str]] = Field( 634 [], 635 description="List of nested fields defining the schema field path to extract. Defaults to [].", 636 title="Schema Path", 637 ) 638 key_pointer: List[str] = Field( 639 ..., 640 description="List of potentially nested fields describing the full path of the field key to extract.", 641 title="Key Path", 642 ) 643 type_pointer: Optional[List[str]] = Field( 644 None, 645 description="List of potentially nested fields describing the full path of the field type to extract.", 646 title="Type Path", 647 ) 648 types_mapping: Optional[List[TypesMap]] = None 649 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 650 651 652class InlineSchemaLoader(BaseModel): 653 type: Literal["InlineSchemaLoader"] 654 schema_: Optional[Dict[str, Any]] = Field( 655 None, 656 alias="schema", 657 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 658 title="Schema", 659 ) 660 661 662class JsonFileSchemaLoader(BaseModel): 663 type: Literal["JsonFileSchemaLoader"] 664 file_path: Optional[str] = Field( 665 None, 666 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 667 example=["./schemas/users.json"], 668 title="File Path", 669 ) 670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 671 672 673class JsonDecoder(BaseModel): 674 type: Literal["JsonDecoder"] 675 676 677class JsonItemsDecoder(BaseModel): 678 type: Literal["JsonItemsDecoder"] 679 items_path: str = Field( 680 ..., 681 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 682 title="Items Path", 683 ) 684 encoding: Optional[str] = Field( 685 "utf-8", 686 description="The character encoding of the JSON data. Defaults to UTF-8.", 687 title="Encoding", 688 ) 689 690 691class JsonlDecoder(BaseModel): 692 type: Literal["JsonlDecoder"] 693 694 695class KeysToLower(BaseModel): 696 type: Literal["KeysToLower"] 697 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 698 699 700class KeysToSnakeCase(BaseModel): 701 type: Literal["KeysToSnakeCase"] 702 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 703 704 705class FlattenFields(BaseModel): 706 type: Literal["FlattenFields"] 707 flatten_lists: Optional[bool] = Field( 708 True, 709 description="Whether to flatten lists or leave it as is. Default is True.", 710 title="Flatten Lists", 711 ) 712 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 713 714 715class KeyTransformation(BaseModel): 716 type: Literal["KeyTransformation"] 717 prefix: Optional[str] = Field( 718 None, 719 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 720 examples=["flattened_"], 721 title="Key Prefix", 722 ) 723 suffix: Optional[str] = Field( 724 None, 725 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 726 examples=["_flattened"], 727 title="Key Suffix", 728 ) 729 730 731class DpathFlattenFields(BaseModel): 732 type: Literal["DpathFlattenFields"] 733 field_path: List[str] = Field( 734 ..., 735 description="A path to field that needs to be flattened.", 736 examples=[["data"], ["data", "*", "field"]], 737 title="Field Path", 738 ) 739 delete_origin_value: Optional[bool] = Field( 740 None, 741 description="Whether to delete the origin value or keep it. Default is False.", 742 title="Delete Origin Value", 743 ) 744 replace_record: Optional[bool] = Field( 745 None, 746 description="Whether to replace the origin record or not. Default is False.", 747 title="Replace Origin Record", 748 ) 749 key_transformation: Optional[KeyTransformation] = Field( 750 None, 751 description="Transformation for object keys. If not provided, original key will be used.", 752 title="Key transformation", 753 ) 754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 755 756 757class KeysReplace(BaseModel): 758 type: Literal["KeysReplace"] 759 old: str = Field( 760 ..., 761 description="Old value to replace.", 762 examples=[ 763 " ", 764 "{{ record.id }}", 765 "{{ config['id'] }}", 766 "{{ stream_slice['id'] }}", 767 ], 768 title="Old value", 769 ) 770 new: str = Field( 771 ..., 772 description="New value to set.", 773 examples=[ 774 "_", 775 "{{ record.id }}", 776 "{{ config['id'] }}", 777 "{{ stream_slice['id'] }}", 778 ], 779 title="New value", 780 ) 781 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 782 783 784class IterableDecoder(BaseModel): 785 type: Literal["IterableDecoder"] 786 787 788class XmlDecoder(BaseModel): 789 type: Literal["XmlDecoder"] 790 791 792class CustomDecoder(BaseModel): 793 class Config: 794 extra = Extra.allow 795 796 type: Literal["CustomDecoder"] 797 class_name: str = Field( 798 ..., 799 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 800 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 801 title="Class Name", 802 ) 803 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 804 805 806class MinMaxDatetime(BaseModel): 807 type: Literal["MinMaxDatetime"] 808 datetime: str = Field( 809 ..., 810 description="Datetime value.", 811 examples=[ 812 "2021-01-01", 813 "2021-01-01T00:00:00Z", 814 "{{ config['start_time'] }}", 815 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 816 ], 817 title="Datetime", 818 ) 819 datetime_format: Optional[str] = Field( 820 "", 821 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 822 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 823 title="Datetime Format", 824 ) 825 max_datetime: Optional[str] = Field( 826 None, 827 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 828 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 829 title="Max Datetime", 830 ) 831 min_datetime: Optional[str] = Field( 832 None, 833 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 834 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 835 title="Min Datetime", 836 ) 837 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 838 839 840class NoAuth(BaseModel): 841 type: Literal["NoAuth"] 842 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 843 844 845class NoPagination(BaseModel): 846 type: Literal["NoPagination"] 847 848 849class State(BaseModel): 850 class Config: 851 extra = Extra.allow 852 853 min: int 854 max: int 855 856 857class OAuthScope(BaseModel): 858 class Config: 859 extra = Extra.allow 860 861 scope: str = Field( 862 ..., 863 description="The OAuth scope string to request from the provider.", 864 ) 865 866 867class OauthConnectorInputSpecification(BaseModel): 868 class Config: 869 extra = Extra.allow 870 871 consent_url: str = Field( 872 ..., 873 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 874 examples=[ 875 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 876 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 877 ], 878 title="Consent URL", 879 ) 880 scope: Optional[str] = Field( 881 None, 882 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 883 examples=["user:read user:read_orders workspaces:read"], 884 title="Scopes", 885 ) 886 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 887 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 888 # The CDK schema defines the manifest contract; the platform reads these fields 889 # during the OAuth consent flow to build the authorization URL. 890 scopes: Optional[List[OAuthScope]] = Field( 891 None, 892 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 893 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 894 title="Scopes", 895 ) 896 optional_scopes: Optional[List[OAuthScope]] = Field( 897 None, 898 description="Optional OAuth scope objects that may or may not be granted.", 899 examples=[[{"scope": "admin:read"}]], 900 title="Optional Scopes", 901 ) 902 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 903 ScopesJoinStrategy.space, 904 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 905 title="Scopes Join Strategy", 906 ) 907 access_token_url: str = Field( 908 ..., 909 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 910 examples=[ 911 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 912 ], 913 title="Access Token URL", 914 ) 915 access_token_headers: Optional[Dict[str, Any]] = Field( 916 None, 917 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 918 examples=[ 919 { 920 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 921 } 922 ], 923 title="Access Token Headers", 924 ) 925 access_token_params: Optional[Dict[str, Any]] = Field( 926 None, 927 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 928 examples=[ 929 { 930 "{{ auth_code_key }}": "{{ auth_code_value }}", 931 "{{ client_id_key }}": "{{ client_id_value }}", 932 "{{ client_secret_key }}": "{{ client_secret_value }}", 933 } 934 ], 935 title="Access Token Query Params (Json Encoded)", 936 ) 937 extract_output: Optional[List[str]] = Field( 938 None, 939 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 940 examples=[["access_token", "refresh_token", "other_field"]], 941 title="Extract Output", 942 ) 943 state: Optional[State] = Field( 944 None, 945 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 946 examples=[{"min": 7, "max": 128}], 947 title="Configurable State Query Param", 948 ) 949 client_id_key: Optional[str] = Field( 950 None, 951 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 952 examples=["my_custom_client_id_key_name"], 953 title="Client ID Key Override", 954 ) 955 client_secret_key: Optional[str] = Field( 956 None, 957 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 958 examples=["my_custom_client_secret_key_name"], 959 title="Client Secret Key Override", 960 ) 961 scope_key: Optional[str] = Field( 962 None, 963 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 964 examples=["my_custom_scope_key_key_name"], 965 title="Scopes Key Override", 966 ) 967 state_key: Optional[str] = Field( 968 None, 969 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 970 examples=["my_custom_state_key_key_name"], 971 title="State Key Override", 972 ) 973 auth_code_key: Optional[str] = Field( 974 None, 975 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 976 examples=["my_custom_auth_code_key_name"], 977 title="Auth Code Key Override", 978 ) 979 redirect_uri_key: Optional[str] = Field( 980 None, 981 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 982 examples=["my_custom_redirect_uri_key_name"], 983 title="Redirect URI Key Override", 984 ) 985 986 987class OAuthConfigSpecification(BaseModel): 988 class Config: 989 extra = Extra.allow 990 991 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 992 None, 993 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 994 examples=[ 995 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 996 { 997 "app_id": { 998 "type": "string", 999 "path_in_connector_config": ["info", "app_id"], 1000 } 1001 }, 1002 ], 1003 title="OAuth user input", 1004 ) 1005 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1006 None, 1007 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1008 title="DeclarativeOAuth Connector Specification", 1009 ) 1010 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1011 None, 1012 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1013 examples=[ 1014 { 1015 "refresh_token": { 1016 "type": "string,", 1017 "path_in_connector_config": ["credentials", "refresh_token"], 1018 } 1019 } 1020 ], 1021 title="OAuth output specification", 1022 ) 1023 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1024 None, 1025 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1026 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1027 title="OAuth input specification", 1028 ) 1029 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1030 None, 1031 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1032 examples=[ 1033 { 1034 "client_id": { 1035 "type": "string,", 1036 "path_in_connector_config": ["credentials", "client_id"], 1037 }, 1038 "client_secret": { 1039 "type": "string,", 1040 "path_in_connector_config": ["credentials", "client_secret"], 1041 }, 1042 } 1043 ], 1044 title="OAuth server output specification", 1045 ) 1046 1047 1048class OffsetIncrement(BaseModel): 1049 type: Literal["OffsetIncrement"] 1050 page_size: Optional[Union[int, str]] = Field( 1051 None, 1052 description="The number of records to include in each pages.", 1053 examples=[100, "{{ config['page_size'] }}"], 1054 title="Limit", 1055 ) 1056 inject_on_first_request: Optional[bool] = Field( 1057 False, 1058 description="Using the `offset` with value `0` during the first request", 1059 title="Inject Offset on First Request", 1060 ) 1061 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1062 1063 1064class PageIncrement(BaseModel): 1065 type: Literal["PageIncrement"] 1066 page_size: Optional[Union[int, str]] = Field( 1067 None, 1068 description="The number of records to include in each pages.", 1069 examples=[100, "100", "{{ config['page_size'] }}"], 1070 title="Page Size", 1071 ) 1072 start_from_page: Optional[int] = Field( 1073 0, 1074 description="Index of the first page to request.", 1075 examples=[0, 1], 1076 title="Start From Page", 1077 ) 1078 inject_on_first_request: Optional[bool] = Field( 1079 False, 1080 description="Using the `page number` with value defined by `start_from_page` during the first request", 1081 title="Inject Page Number on First Request", 1082 ) 1083 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1084 1085 1086class PrimaryKey(BaseModel): 1087 __root__: Union[str, List[str], List[List[str]]] = Field( 1088 ..., 1089 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1090 examples=["id", ["code", "type"]], 1091 title="Primary Key", 1092 ) 1093 1094 1095class PropertyLimitType(Enum): 1096 characters = "characters" 1097 property_count = "property_count" 1098 1099 1100class PropertyChunking(BaseModel): 1101 type: Literal["PropertyChunking"] 1102 property_limit_type: PropertyLimitType = Field( 1103 ..., 1104 description="The type used to determine the maximum number of properties per chunk", 1105 title="Property Limit Type", 1106 ) 1107 property_limit: Optional[int] = Field( 1108 None, 1109 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1110 title="Property Limit", 1111 ) 1112 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1113 None, 1114 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1115 title="Record Merge Strategy", 1116 ) 1117 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1118 1119 1120class RecordFilter(BaseModel): 1121 type: Literal["RecordFilter"] 1122 condition: Optional[str] = Field( 1123 "", 1124 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1125 examples=[ 1126 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1127 "{{ record.status in ['active', 'expired'] }}", 1128 ], 1129 title="Condition", 1130 ) 1131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1132 1133 1134class SchemaNormalization(Enum): 1135 Default = "Default" 1136 None_ = "None" 1137 1138 1139class RemoveFields(BaseModel): 1140 type: Literal["RemoveFields"] 1141 condition: Optional[str] = Field( 1142 "", 1143 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1144 examples=[ 1145 "{{ property|string == '' }}", 1146 "{{ property is integer }}", 1147 "{{ property|length > 5 }}", 1148 "{{ property == 'some_string_to_match' }}", 1149 ], 1150 ) 1151 field_pointers: List[List[str]] = Field( 1152 ..., 1153 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1154 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1155 title="Field Paths", 1156 ) 1157 1158 1159class RequestPath(BaseModel): 1160 type: Literal["RequestPath"] 1161 1162 1163class InjectInto(Enum): 1164 request_parameter = "request_parameter" 1165 header = "header" 1166 body_data = "body_data" 1167 body_json = "body_json" 1168 1169 1170class RequestOption(BaseModel): 1171 type: Literal["RequestOption"] 1172 inject_into: InjectInto = Field( 1173 ..., 1174 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1175 examples=["request_parameter", "header", "body_data", "body_json"], 1176 title="Inject Into", 1177 ) 1178 field_name: Optional[str] = Field( 1179 None, 1180 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1181 examples=["segment_id"], 1182 title="Field Name", 1183 ) 1184 field_path: Optional[List[str]] = Field( 1185 None, 1186 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1187 examples=[["data", "viewer", "id"]], 1188 title="Field Path", 1189 ) 1190 1191 1192class Schemas(BaseModel): 1193 pass 1194 1195 class Config: 1196 extra = Extra.allow 1197 1198 1199class LegacySessionTokenAuthenticator(BaseModel): 1200 type: Literal["LegacySessionTokenAuthenticator"] 1201 header: str = Field( 1202 ..., 1203 description="The name of the session token header that will be injected in the request", 1204 examples=["X-Session"], 1205 title="Session Request Header", 1206 ) 1207 login_url: str = Field( 1208 ..., 1209 description="Path of the login URL (do not include the base URL)", 1210 examples=["session"], 1211 title="Login Path", 1212 ) 1213 session_token: Optional[str] = Field( 1214 None, 1215 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1216 example=["{{ config['session_token'] }}"], 1217 title="Session Token", 1218 ) 1219 session_token_response_key: str = Field( 1220 ..., 1221 description="Name of the key of the session token to be extracted from the response", 1222 examples=["id"], 1223 title="Response Token Response Key", 1224 ) 1225 username: Optional[str] = Field( 1226 None, 1227 description="Username used to authenticate and obtain a session token", 1228 examples=[" {{ config['username'] }}"], 1229 title="Username", 1230 ) 1231 password: Optional[str] = Field( 1232 "", 1233 description="Password used to authenticate and obtain a session token", 1234 examples=["{{ config['password'] }}", ""], 1235 title="Password", 1236 ) 1237 validate_session_url: str = Field( 1238 ..., 1239 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1240 examples=["user/current"], 1241 title="Validate Session Path", 1242 ) 1243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1244 1245 1246class Action1(Enum): 1247 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1248 RESET = "RESET" 1249 1250 1251class PaginationResetLimits(BaseModel): 1252 type: Literal["PaginationResetLimits"] 1253 number_of_records: Optional[int] = None 1254 1255 1256class CsvDecoder(BaseModel): 1257 type: Literal["CsvDecoder"] 1258 encoding: Optional[str] = "utf-8" 1259 delimiter: Optional[str] = "," 1260 set_values_to_none: Optional[List[str]] = None 1261 1262 1263class AsyncJobStatusMap(BaseModel): 1264 type: Optional[Literal["AsyncJobStatusMap"]] = None 1265 running: List[str] 1266 completed: List[str] 1267 failed: List[str] 1268 timeout: List[str] 1269 skipped: Optional[List[str]] = None 1270 1271 1272class ValueType(Enum): 1273 string = "string" 1274 number = "number" 1275 integer = "integer" 1276 boolean = "boolean" 1277 1278 1279class WaitTimeFromHeader(BaseModel): 1280 type: Literal["WaitTimeFromHeader"] 1281 header: str = Field( 1282 ..., 1283 description="The name of the response header defining how long to wait before retrying.", 1284 examples=["Retry-After"], 1285 title="Response Header Name", 1286 ) 1287 regex: Optional[str] = Field( 1288 None, 1289 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1290 examples=["([-+]?\\d+)"], 1291 title="Extraction Regex", 1292 ) 1293 max_waiting_time_in_seconds: Optional[float] = Field( 1294 None, 1295 description="Given the value extracted from the header is greater than this value, stop the stream.", 1296 examples=[3600], 1297 title="Max Waiting Time in Seconds", 1298 ) 1299 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1300 1301 1302class WaitUntilTimeFromHeader(BaseModel): 1303 type: Literal["WaitUntilTimeFromHeader"] 1304 header: str = Field( 1305 ..., 1306 description="The name of the response header defining how long to wait before retrying.", 1307 examples=["wait_time"], 1308 title="Response Header", 1309 ) 1310 min_wait: Optional[Union[float, str]] = Field( 1311 None, 1312 description="Minimum time to wait before retrying.", 1313 examples=[10, "60"], 1314 title="Minimum Wait Time", 1315 ) 1316 regex: Optional[str] = Field( 1317 None, 1318 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1319 examples=["([-+]?\\d+)"], 1320 title="Extraction Regex", 1321 ) 1322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1323 1324 1325class ComponentMappingDefinition(BaseModel): 1326 type: Literal["ComponentMappingDefinition"] 1327 field_path: List[str] = Field( 1328 ..., 1329 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1330 examples=[ 1331 ["name"], 1332 ["retriever", "requester", "url"], 1333 ["retriever", "requester", "{{ components_values.field }}"], 1334 ["*", "**", "name"], 1335 ], 1336 title="Field Path", 1337 ) 1338 value: str = Field( 1339 ..., 1340 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1341 examples=[ 1342 "{{ components_values['updates'] }}", 1343 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1344 "{{ config['segment_id'] }}", 1345 "{{ stream_slice['parent_id'] }}", 1346 "{{ stream_slice['extra_fields']['name'] }}", 1347 ], 1348 title="Value", 1349 ) 1350 value_type: Optional[ValueType] = Field( 1351 None, 1352 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1353 title="Value Type", 1354 ) 1355 create_or_update: Optional[bool] = Field( 1356 False, 1357 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1358 title="Create or Update", 1359 ) 1360 condition: Optional[str] = Field( 1361 None, 1362 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1363 examples=[ 1364 "{{ components_values.get('cursor_field', None) }}", 1365 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1366 ], 1367 title="Condition", 1368 ) 1369 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1370 1371 1372class StreamConfig(BaseModel): 1373 type: Literal["StreamConfig"] 1374 configs_pointer: List[str] = Field( 1375 ..., 1376 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1377 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1378 title="Configs Pointer", 1379 ) 1380 default_values: Optional[List[Dict[str, Any]]] = Field( 1381 None, 1382 description="A list of default values, each matching the structure expected from the parsed component value.", 1383 title="Default Values", 1384 ) 1385 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1386 1387 1388class ConfigComponentsResolver(BaseModel): 1389 type: Literal["ConfigComponentsResolver"] 1390 stream_config: Union[List[StreamConfig], StreamConfig] 1391 components_mapping: List[ComponentMappingDefinition] 1392 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1393 1394 1395class StreamParametersDefinition(BaseModel): 1396 type: Literal["StreamParametersDefinition"] 1397 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1398 ..., 1399 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1400 examples=[ 1401 [ 1402 { 1403 "name": "test stream", 1404 "$parameters": {"entity": "test entity"}, 1405 "primary_key": "test key", 1406 } 1407 ] 1408 ], 1409 title="Stream Parameters", 1410 ) 1411 1412 1413class ParametrizedComponentsResolver(BaseModel): 1414 type: Literal["ParametrizedComponentsResolver"] 1415 stream_parameters: StreamParametersDefinition 1416 components_mapping: List[ComponentMappingDefinition] 1417 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1418 1419 1420class RequestBodyPlainText(BaseModel): 1421 type: Literal["RequestBodyPlainText"] 1422 value: str 1423 1424 1425class RequestBodyUrlEncodedForm(BaseModel): 1426 type: Literal["RequestBodyUrlEncodedForm"] 1427 value: Dict[str, str] 1428 1429 1430class RequestBodyJsonObject(BaseModel): 1431 type: Literal["RequestBodyJsonObject"] 1432 value: Dict[str, Any] 1433 1434 1435class RequestBodyGraphQlQuery(BaseModel): 1436 class Config: 1437 extra = Extra.allow 1438 1439 query: str = Field(..., description="The GraphQL query to be executed") 1440 1441 1442class ValidateAdheresToSchema(BaseModel): 1443 type: Literal["ValidateAdheresToSchema"] 1444 base_schema: Union[str, Dict[str, Any]] = Field( 1445 ..., 1446 description="The base JSON schema against which the user-provided schema will be validated.", 1447 examples=[ 1448 "{{ config['report_validation_schema'] }}", 1449 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1450 { 1451 "$schema": "http://json-schema.org/draft-07/schema#", 1452 "title": "Person", 1453 "type": "object", 1454 "properties": { 1455 "name": {"type": "string", "description": "The person's name"}, 1456 "age": { 1457 "type": "integer", 1458 "minimum": 0, 1459 "description": "The person's age", 1460 }, 1461 }, 1462 "required": ["name", "age"], 1463 }, 1464 ], 1465 title="Base JSON Schema", 1466 ) 1467 1468 1469class CustomValidationStrategy(BaseModel): 1470 class Config: 1471 extra = Extra.allow 1472 1473 type: Literal["CustomValidationStrategy"] 1474 class_name: str = Field( 1475 ..., 1476 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1477 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1478 title="Class Name", 1479 ) 1480 1481 1482class ConfigRemapField(BaseModel): 1483 type: Literal["ConfigRemapField"] 1484 map: Union[Dict[str, Any], str] = Field( 1485 ..., 1486 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1487 examples=[ 1488 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1489 "{{ config['status_mapping'] }}", 1490 ], 1491 title="Value Mapping", 1492 ) 1493 field_path: List[str] = Field( 1494 ..., 1495 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1496 examples=[ 1497 ["status"], 1498 ["data", "status"], 1499 ["data", "{{ config.name }}", "status"], 1500 ["data", "*", "status"], 1501 ], 1502 title="Field Path", 1503 ) 1504 1505 1506class ConfigRemoveFields(BaseModel): 1507 type: Literal["ConfigRemoveFields"] 1508 field_pointers: List[List[str]] = Field( 1509 ..., 1510 description="A list of field pointers to be removed from the config.", 1511 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1512 title="Field Pointers", 1513 ) 1514 condition: Optional[str] = Field( 1515 "", 1516 description="Fields will be removed if expression is evaluated to True.", 1517 examples=[ 1518 "{{ config['environemnt'] == 'sandbox' }}", 1519 "{{ property is integer }}", 1520 "{{ property|length > 5 }}", 1521 "{{ property == 'some_string_to_match' }}", 1522 ], 1523 ) 1524 1525 1526class CustomConfigTransformation(BaseModel): 1527 type: Literal["CustomConfigTransformation"] 1528 class_name: str = Field( 1529 ..., 1530 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1531 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1532 ) 1533 parameters: Optional[Dict[str, Any]] = Field( 1534 None, 1535 alias="$parameters", 1536 description="Additional parameters to be passed to the custom config transformation.", 1537 ) 1538 1539 1540class AddedFieldDefinition(BaseModel): 1541 type: Literal["AddedFieldDefinition"] 1542 path: List[str] = Field( 1543 ..., 1544 description="List of strings defining the path where to add the value on the record.", 1545 examples=[["segment_id"], ["metadata", "segment_id"]], 1546 title="Path", 1547 ) 1548 value: str = Field( 1549 ..., 1550 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1551 examples=[ 1552 "{{ record['updates'] }}", 1553 "{{ record['MetaData']['LastUpdatedTime'] }}", 1554 "{{ stream_partition['segment_id'] }}", 1555 ], 1556 title="Value", 1557 ) 1558 value_type: Optional[ValueType] = Field( 1559 None, 1560 description="Type of the value. If not specified, the type will be inferred from the value.", 1561 title="Value Type", 1562 ) 1563 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1564 1565 1566class AddFields(BaseModel): 1567 type: Literal["AddFields"] 1568 fields: List[AddedFieldDefinition] = Field( 1569 ..., 1570 description="List of transformations (path and corresponding value) that will be added to the record.", 1571 title="Fields", 1572 ) 1573 condition: Optional[str] = Field( 1574 "", 1575 description="Fields will be added if expression is evaluated to True.", 1576 examples=[ 1577 "{{ property|string == '' }}", 1578 "{{ property is integer }}", 1579 "{{ property|length > 5 }}", 1580 "{{ property == 'some_string_to_match' }}", 1581 ], 1582 ) 1583 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1584 1585 1586class ApiKeyAuthenticator(BaseModel): 1587 type: Literal["ApiKeyAuthenticator"] 1588 api_token: Optional[str] = Field( 1589 None, 1590 description="The API key to inject in the request. Fill it in the user inputs.", 1591 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1592 title="API Key", 1593 ) 1594 header: Optional[str] = Field( 1595 None, 1596 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1597 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1598 title="Header Name", 1599 ) 1600 inject_into: Optional[RequestOption] = Field( 1601 None, 1602 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1603 examples=[ 1604 {"inject_into": "header", "field_name": "Authorization"}, 1605 {"inject_into": "request_parameter", "field_name": "authKey"}, 1606 ], 1607 title="Inject API Key Into Outgoing HTTP Request", 1608 ) 1609 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1610 1611 1612class AuthFlow(BaseModel): 1613 auth_flow_type: Optional[AuthFlowType] = Field( 1614 None, description="The type of auth to use", title="Auth flow type" 1615 ) 1616 predicate_key: Optional[List[str]] = Field( 1617 None, 1618 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1619 examples=[["credentials", "auth_type"]], 1620 title="Predicate key", 1621 ) 1622 predicate_value: Optional[str] = Field( 1623 None, 1624 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1625 examples=["Oauth"], 1626 title="Predicate value", 1627 ) 1628 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1629 1630 1631class CheckStream(BaseModel): 1632 type: Literal["CheckStream"] 1633 stream_names: Optional[List[str]] = Field( 1634 None, 1635 description="Names of the streams to try reading from when running a check operation.", 1636 examples=[["users"], ["users", "contacts"]], 1637 title="Stream Names", 1638 ) 1639 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1640 1641 1642class IncrementingCountCursor(BaseModel): 1643 type: Literal["IncrementingCountCursor"] 1644 cursor_field: str = Field( 1645 ..., 1646 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1647 examples=["created_at", "{{ config['record_cursor'] }}"], 1648 title="Cursor Field", 1649 ) 1650 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1651 None, 1652 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1653 title="Allow Catalog Defined Cursor Field", 1654 ) 1655 start_value: Optional[Union[str, int]] = Field( 1656 None, 1657 description="The value that determines the earliest record that should be synced.", 1658 examples=[0, "{{ config['start_value'] }}"], 1659 title="Start Value", 1660 ) 1661 start_value_option: Optional[RequestOption] = Field( 1662 None, 1663 description="Optionally configures how the start value will be sent in requests to the source API.", 1664 title="Inject Start Value Into Outgoing HTTP Request", 1665 ) 1666 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1667 1668 1669class DatetimeBasedCursor(BaseModel): 1670 type: Literal["DatetimeBasedCursor"] 1671 clamping: Optional[Clamping] = Field( 1672 None, 1673 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1674 title="Date Range Clamping", 1675 ) 1676 cursor_field: str = Field( 1677 ..., 1678 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1679 examples=["created_at", "{{ config['record_cursor'] }}"], 1680 title="Cursor Field", 1681 ) 1682 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1683 None, 1684 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1685 title="Allow Catalog Defined Cursor Field", 1686 ) 1687 cursor_datetime_formats: Optional[List[str]] = Field( 1688 None, 1689 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1690 examples=[ 1691 "%Y-%m-%d", 1692 "%Y-%m-%d %H:%M:%S", 1693 "%Y-%m-%dT%H:%M:%S", 1694 "%Y-%m-%dT%H:%M:%SZ", 1695 "%Y-%m-%dT%H:%M:%S%z", 1696 "%Y-%m-%dT%H:%M:%S.%fZ", 1697 "%Y-%m-%dT%H:%M:%S.%f%z", 1698 "%Y-%m-%d %H:%M:%S.%f+00:00", 1699 "%s", 1700 "%ms", 1701 ], 1702 title="Cursor Datetime Formats", 1703 ) 1704 start_datetime: Union[MinMaxDatetime, str] = Field( 1705 ..., 1706 description="The datetime that determines the earliest record that should be synced.", 1707 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1708 title="Start Datetime", 1709 ) 1710 start_time_option: Optional[RequestOption] = Field( 1711 None, 1712 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1713 title="Inject Start Time Into Outgoing HTTP Request", 1714 ) 1715 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1716 None, 1717 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1718 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1719 title="End Datetime", 1720 ) 1721 end_time_option: Optional[RequestOption] = Field( 1722 None, 1723 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1724 title="Inject End Time Into Outgoing HTTP Request", 1725 ) 1726 datetime_format: str = Field( 1727 ..., 1728 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1729 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1730 title="Outgoing Datetime Format", 1731 ) 1732 cursor_granularity: Optional[str] = Field( 1733 None, 1734 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1735 examples=["PT1S"], 1736 title="Cursor Granularity", 1737 ) 1738 is_data_feed: Optional[bool] = Field( 1739 None, 1740 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1741 title="Data Feed API", 1742 ) 1743 is_client_side_incremental: Optional[bool] = Field( 1744 None, 1745 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1746 title="Client-side Incremental Filtering", 1747 ) 1748 is_compare_strictly: Optional[bool] = Field( 1749 False, 1750 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1751 title="Strict Start-End Time Comparison", 1752 ) 1753 global_substream_cursor: Optional[bool] = Field( 1754 False, 1755 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1756 title="Global Substream Cursor", 1757 ) 1758 lookback_window: Optional[str] = Field( 1759 None, 1760 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1761 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1762 title="Lookback Window", 1763 ) 1764 partition_field_end: Optional[str] = Field( 1765 None, 1766 description="Name of the partition start time field.", 1767 examples=["ending_time"], 1768 title="Partition Field End", 1769 ) 1770 partition_field_start: Optional[str] = Field( 1771 None, 1772 description="Name of the partition end time field.", 1773 examples=["starting_time"], 1774 title="Partition Field Start", 1775 ) 1776 step: Optional[str] = Field( 1777 None, 1778 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1779 examples=["P1W", "{{ config['step_increment'] }}"], 1780 title="Step", 1781 ) 1782 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1783 1784 1785class JwtAuthenticator(BaseModel): 1786 type: Literal["JwtAuthenticator"] 1787 secret_key: str = Field( 1788 ..., 1789 description="Secret used to sign the JSON web token.", 1790 examples=["{{ config['secret_key'] }}"], 1791 title="Secret Key", 1792 ) 1793 base64_encode_secret_key: Optional[bool] = Field( 1794 False, 1795 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1796 title="Base64-encode Secret Key", 1797 ) 1798 algorithm: Algorithm = Field( 1799 ..., 1800 description="Algorithm used to sign the JSON web token.", 1801 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1802 title="Algorithm", 1803 ) 1804 token_duration: Optional[int] = Field( 1805 1200, 1806 description="The amount of time in seconds a JWT token can be valid after being issued.", 1807 examples=[1200, 3600], 1808 title="Token Duration", 1809 ) 1810 header_prefix: Optional[str] = Field( 1811 None, 1812 description="The prefix to be used within the Authentication header.", 1813 examples=["Bearer", "Basic"], 1814 title="Header Prefix", 1815 ) 1816 jwt_headers: Optional[JwtHeaders] = Field( 1817 None, 1818 description="JWT headers used when signing JSON web token.", 1819 title="JWT Headers", 1820 ) 1821 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1822 None, 1823 description="Additional headers to be included with the JWT headers object.", 1824 title="Additional JWT Headers", 1825 ) 1826 jwt_payload: Optional[JwtPayload] = Field( 1827 None, 1828 description="JWT Payload used when signing JSON web token.", 1829 title="JWT Payload", 1830 ) 1831 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1832 None, 1833 description="Additional properties to be added to the JWT payload.", 1834 title="Additional JWT Payload Properties", 1835 ) 1836 passphrase: Optional[str] = Field( 1837 None, 1838 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1839 examples=["{{ config['passphrase'] }}"], 1840 title="Passphrase", 1841 ) 1842 request_option: Optional[RequestOption] = Field( 1843 None, 1844 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1845 title="Request Option", 1846 ) 1847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1848 1849 1850class OAuthAuthenticator(BaseModel): 1851 type: Literal["OAuthAuthenticator"] 1852 client_id_name: Optional[str] = Field( 1853 "client_id", 1854 description="The name of the property to use to refresh the `access_token`.", 1855 examples=["custom_app_id"], 1856 title="Client ID Property Name", 1857 ) 1858 client_id: Optional[str] = Field( 1859 None, 1860 description="The OAuth client ID. Fill it in the user inputs.", 1861 examples=[ 1862 "{{ config['client_id'] }}", 1863 "{{ config['credentials']['client_id }}", 1864 ], 1865 title="Client ID", 1866 ) 1867 client_secret_name: Optional[str] = Field( 1868 "client_secret", 1869 description="The name of the property to use to refresh the `access_token`.", 1870 examples=["custom_app_secret"], 1871 title="Client Secret Property Name", 1872 ) 1873 client_secret: Optional[str] = Field( 1874 None, 1875 description="The OAuth client secret. Fill it in the user inputs.", 1876 examples=[ 1877 "{{ config['client_secret'] }}", 1878 "{{ config['credentials']['client_secret }}", 1879 ], 1880 title="Client Secret", 1881 ) 1882 refresh_token_name: Optional[str] = Field( 1883 "refresh_token", 1884 description="The name of the property to use to refresh the `access_token`.", 1885 examples=["custom_app_refresh_value"], 1886 title="Refresh Token Property Name", 1887 ) 1888 refresh_token: Optional[str] = Field( 1889 None, 1890 description="Credential artifact used to get a new access token.", 1891 examples=[ 1892 "{{ config['refresh_token'] }}", 1893 "{{ config['credentials]['refresh_token'] }}", 1894 ], 1895 title="Refresh Token", 1896 ) 1897 token_refresh_endpoint: Optional[str] = Field( 1898 None, 1899 description="The full URL to call to obtain a new access token.", 1900 examples=["https://connect.squareup.com/oauth2/token"], 1901 title="Token Refresh Endpoint", 1902 ) 1903 access_token_name: Optional[str] = Field( 1904 "access_token", 1905 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1906 examples=["access_token"], 1907 title="Access Token Property Name", 1908 ) 1909 access_token_value: Optional[str] = Field( 1910 None, 1911 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1912 examples=["secret_access_token_value"], 1913 title="Access Token Value", 1914 ) 1915 expires_in_name: Optional[str] = Field( 1916 "expires_in", 1917 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1918 examples=["expires_in"], 1919 title="Token Expiry Property Name", 1920 ) 1921 grant_type_name: Optional[str] = Field( 1922 "grant_type", 1923 description="The name of the property to use to refresh the `access_token`.", 1924 examples=["custom_grant_type"], 1925 title="Grant Type Property Name", 1926 ) 1927 grant_type: Optional[str] = Field( 1928 "refresh_token", 1929 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1930 examples=["refresh_token", "client_credentials"], 1931 title="Grant Type", 1932 ) 1933 refresh_request_body: Optional[Dict[str, Any]] = Field( 1934 None, 1935 description="Body of the request sent to get a new access token.", 1936 examples=[ 1937 { 1938 "applicationId": "{{ config['application_id'] }}", 1939 "applicationSecret": "{{ config['application_secret'] }}", 1940 "token": "{{ config['token'] }}", 1941 } 1942 ], 1943 title="Refresh Request Body", 1944 ) 1945 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1946 None, 1947 description="Headers of the request sent to get a new access token.", 1948 examples=[ 1949 { 1950 "Authorization": "<AUTH_TOKEN>", 1951 "Content-Type": "application/x-www-form-urlencoded", 1952 } 1953 ], 1954 title="Refresh Request Headers", 1955 ) 1956 send_refresh_request_as_query_params: Optional[bool] = Field( 1957 False, 1958 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 1959 examples=[True], 1960 title="Send Refresh Request As Query Params", 1961 ) 1962 scopes: Optional[List[str]] = Field( 1963 None, 1964 description="List of scopes that should be granted to the access token.", 1965 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1966 title="Scopes", 1967 ) 1968 token_expiry_date: Optional[str] = Field( 1969 None, 1970 description="The access token expiry date.", 1971 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1972 title="Token Expiry Date", 1973 ) 1974 token_expiry_date_format: Optional[str] = Field( 1975 None, 1976 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1977 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1978 title="Token Expiry Date Format", 1979 ) 1980 refresh_token_error_status_codes: Optional[List[int]] = Field( 1981 None, 1982 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1983 examples=[[400, 500]], 1984 title="Refresh Token Error Status Codes", 1985 ) 1986 refresh_token_error_key: Optional[str] = Field( 1987 None, 1988 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1989 examples=["error"], 1990 title="Refresh Token Error Key", 1991 ) 1992 refresh_token_error_values: Optional[List[str]] = Field( 1993 None, 1994 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1995 examples=[["invalid_grant", "invalid_permissions"]], 1996 title="Refresh Token Error Values", 1997 ) 1998 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1999 None, 2000 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2001 title="Refresh Token Updater", 2002 ) 2003 profile_assertion: Optional[JwtAuthenticator] = Field( 2004 None, 2005 description="The authenticator being used to authenticate the client authenticator.", 2006 title="Profile Assertion", 2007 ) 2008 use_profile_assertion: Optional[bool] = Field( 2009 False, 2010 description="Enable using profile assertion as a flow for OAuth authorization.", 2011 title="Use Profile Assertion", 2012 ) 2013 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2014 2015 2016class FixedWindowCallRatePolicy(BaseModel): 2017 class Config: 2018 extra = Extra.allow 2019 2020 type: Literal["FixedWindowCallRatePolicy"] 2021 period: str = Field( 2022 ..., description="The time interval for the rate limit window.", title="Period" 2023 ) 2024 call_limit: int = Field( 2025 ..., 2026 description="The maximum number of calls allowed within the period.", 2027 title="Call Limit", 2028 ) 2029 matchers: List[HttpRequestRegexMatcher] = Field( 2030 ..., 2031 description="List of matchers that define which requests this policy applies to.", 2032 title="Matchers", 2033 ) 2034 2035 2036class MovingWindowCallRatePolicy(BaseModel): 2037 class Config: 2038 extra = Extra.allow 2039 2040 type: Literal["MovingWindowCallRatePolicy"] 2041 rates: List[Rate] = Field( 2042 ..., 2043 description="List of rates that define the call limits for different time intervals.", 2044 title="Rates", 2045 ) 2046 matchers: List[HttpRequestRegexMatcher] = Field( 2047 ..., 2048 description="List of matchers that define which requests this policy applies to.", 2049 title="Matchers", 2050 ) 2051 2052 2053class UnlimitedCallRatePolicy(BaseModel): 2054 class Config: 2055 extra = Extra.allow 2056 2057 type: Literal["UnlimitedCallRatePolicy"] 2058 matchers: List[HttpRequestRegexMatcher] = Field( 2059 ..., 2060 description="List of matchers that define which requests this policy applies to.", 2061 title="Matchers", 2062 ) 2063 2064 2065class DefaultErrorHandler(BaseModel): 2066 type: Literal["DefaultErrorHandler"] 2067 backoff_strategies: Optional[ 2068 List[ 2069 Union[ 2070 ConstantBackoffStrategy, 2071 ExponentialBackoffStrategy, 2072 WaitTimeFromHeader, 2073 WaitUntilTimeFromHeader, 2074 CustomBackoffStrategy, 2075 ] 2076 ] 2077 ] = Field( 2078 None, 2079 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2080 title="Backoff Strategies", 2081 ) 2082 max_retries: Optional[Union[int, str]] = Field( 2083 5, 2084 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2085 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2086 title="Max Retry Count", 2087 ) 2088 response_filters: Optional[List[HttpResponseFilter]] = Field( 2089 None, 2090 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2091 title="Response Filters", 2092 ) 2093 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2094 2095 2096class DefaultPaginator(BaseModel): 2097 type: Literal["DefaultPaginator"] 2098 pagination_strategy: Union[ 2099 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2100 ] = Field( 2101 ..., 2102 description="Strategy defining how records are paginated.", 2103 title="Pagination Strategy", 2104 ) 2105 page_size_option: Optional[RequestOption] = Field( 2106 None, title="Inject Page Size Into Outgoing HTTP Request" 2107 ) 2108 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2109 None, 2110 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2111 title="Inject Page Token Into Outgoing HTTP Request", 2112 ) 2113 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2114 2115 2116class RecordExpander(BaseModel): 2117 type: Literal["RecordExpander"] 2118 expand_records_from_field: List[str] = Field( 2119 ..., 2120 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2121 examples=[ 2122 ["lines", "data"], 2123 ["items"], 2124 ["nested", "array"], 2125 ["sections", "*", "items"], 2126 ], 2127 title="Expand Records From Field", 2128 ) 2129 remain_original_record: Optional[bool] = Field( 2130 False, 2131 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2132 title="Remain Original Record", 2133 ) 2134 on_no_records: Optional[OnNoRecords] = Field( 2135 OnNoRecords.skip, 2136 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2137 title="On No Records", 2138 ) 2139 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2140 2141 2142class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2143 type: Literal["ApiKey"] 2144 inject_into: RequestOption = Field( 2145 ..., 2146 description="Configure how the API Key will be sent in requests to the source API.", 2147 examples=[ 2148 {"inject_into": "header", "field_name": "Authorization"}, 2149 {"inject_into": "request_parameter", "field_name": "authKey"}, 2150 ], 2151 title="Inject API Key Into Outgoing HTTP Request", 2152 ) 2153 api_token: Optional[str] = Field( 2154 "{{ session_token }}", 2155 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2156 examples=[ 2157 "{{ session_token }}", 2158 "Token {{ session_token }}", 2159 "Bearer {{ session_token }}", 2160 ], 2161 title="API Token Template", 2162 ) 2163 2164 2165class JsonSchemaPropertySelector(BaseModel): 2166 type: Literal["JsonSchemaPropertySelector"] 2167 transformations: Optional[ 2168 List[ 2169 Union[ 2170 AddFields, 2171 RemoveFields, 2172 KeysToLower, 2173 KeysToSnakeCase, 2174 FlattenFields, 2175 DpathFlattenFields, 2176 KeysReplace, 2177 CustomTransformation, 2178 ] 2179 ] 2180 ] = Field( 2181 None, 2182 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2183 title="Transformations", 2184 ) 2185 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2186 2187 2188class ListPartitionRouter(BaseModel): 2189 type: Literal["ListPartitionRouter"] 2190 cursor_field: str = Field( 2191 ..., 2192 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2193 examples=["section", "{{ config['section_key'] }}"], 2194 title="Current Partition Value Identifier", 2195 ) 2196 values: Union[str, List[str]] = Field( 2197 ..., 2198 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2199 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2200 title="Partition Values", 2201 ) 2202 request_option: Optional[RequestOption] = Field( 2203 None, 2204 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2205 title="Inject Partition Value Into Outgoing HTTP Request", 2206 ) 2207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2208 2209 2210class PaginationReset(BaseModel): 2211 type: Literal["PaginationReset"] 2212 action: Action1 2213 limits: Optional[PaginationResetLimits] = None 2214 2215 2216class GzipDecoder(BaseModel): 2217 type: Literal["GzipDecoder"] 2218 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2219 2220 2221class RequestBodyGraphQL(BaseModel): 2222 type: Literal["RequestBodyGraphQL"] 2223 value: RequestBodyGraphQlQuery 2224 2225 2226class DpathValidator(BaseModel): 2227 type: Literal["DpathValidator"] 2228 field_path: List[str] = Field( 2229 ..., 2230 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2231 examples=[ 2232 ["data"], 2233 ["data", "records"], 2234 ["data", "{{ parameters.name }}"], 2235 ["data", "*", "record"], 2236 ], 2237 title="Field Path", 2238 ) 2239 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2240 ..., 2241 description="The condition that the specified config value will be evaluated against", 2242 title="Validation Strategy", 2243 ) 2244 2245 2246class PredicateValidator(BaseModel): 2247 type: Literal["PredicateValidator"] 2248 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2249 ..., 2250 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2251 examples=[ 2252 "test-value", 2253 "{{ config['api_version'] }}", 2254 "{{ config['tenant_id'] }}", 2255 123, 2256 ], 2257 title="Value", 2258 ) 2259 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2260 ..., 2261 description="The validation strategy to apply to the value.", 2262 title="Validation Strategy", 2263 ) 2264 2265 2266class ConfigAddFields(BaseModel): 2267 type: Literal["ConfigAddFields"] 2268 fields: List[AddedFieldDefinition] = Field( 2269 ..., 2270 description="A list of transformations (path and corresponding value) that will be added to the config.", 2271 title="Fields", 2272 ) 2273 condition: Optional[str] = Field( 2274 "", 2275 description="Fields will be added if expression is evaluated to True.", 2276 examples=[ 2277 "{{ config['environemnt'] == 'sandbox' }}", 2278 "{{ property is integer }}", 2279 "{{ property|length > 5 }}", 2280 "{{ property == 'some_string_to_match' }}", 2281 ], 2282 ) 2283 2284 2285class CompositeErrorHandler(BaseModel): 2286 type: Literal["CompositeErrorHandler"] 2287 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2288 Field( 2289 ..., 2290 description="List of error handlers to iterate on to determine how to handle a failed response.", 2291 title="Error Handlers", 2292 ) 2293 ) 2294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2295 2296 2297class HTTPAPIBudget(BaseModel): 2298 class Config: 2299 extra = Extra.allow 2300 2301 type: Literal["HTTPAPIBudget"] 2302 policies: List[ 2303 Union[ 2304 FixedWindowCallRatePolicy, 2305 MovingWindowCallRatePolicy, 2306 UnlimitedCallRatePolicy, 2307 ] 2308 ] = Field( 2309 ..., 2310 description="List of call rate policies that define how many calls are allowed.", 2311 title="Policies", 2312 ) 2313 ratelimit_reset_header: Optional[str] = Field( 2314 "ratelimit-reset", 2315 description="The HTTP response header name that indicates when the rate limit resets.", 2316 title="Rate Limit Reset Header", 2317 ) 2318 ratelimit_remaining_header: Optional[str] = Field( 2319 "ratelimit-remaining", 2320 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2321 title="Rate Limit Remaining Header", 2322 ) 2323 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2324 [429], 2325 description="List of HTTP status codes that indicate a rate limit has been hit.", 2326 title="Status Codes for Rate Limit Hit", 2327 ) 2328 2329 2330class DpathExtractor(BaseModel): 2331 type: Literal["DpathExtractor"] 2332 field_path: List[str] = Field( 2333 ..., 2334 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2335 examples=[ 2336 ["data"], 2337 ["data", "records"], 2338 ["data", "{{ parameters.name }}"], 2339 ["data", "*", "record"], 2340 ], 2341 title="Field Path", 2342 ) 2343 record_expander: Optional[RecordExpander] = Field( 2344 None, 2345 description="Optional component to expand records by extracting items from nested array fields.", 2346 title="Record Expander", 2347 ) 2348 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2349 2350 2351class ZipfileDecoder(BaseModel): 2352 class Config: 2353 extra = Extra.allow 2354 2355 type: Literal["ZipfileDecoder"] 2356 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2357 ..., 2358 description="Parser to parse the decompressed data from the zipfile(s).", 2359 title="Parser", 2360 ) 2361 2362 2363class RecordSelector(BaseModel): 2364 type: Literal["RecordSelector"] 2365 extractor: Union[DpathExtractor, CustomRecordExtractor] 2366 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2367 None, 2368 description="Responsible for filtering records to be emitted by the Source.", 2369 title="Record Filter", 2370 ) 2371 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2372 None, 2373 description="Responsible for normalization according to the schema.", 2374 title="Schema Normalization", 2375 ) 2376 transform_before_filtering: Optional[bool] = Field( 2377 None, 2378 description="If true, transformation will be applied before record filtering.", 2379 title="Transform Before Filtering", 2380 ) 2381 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2382 2383 2384class ConfigMigration(BaseModel): 2385 type: Literal["ConfigMigration"] 2386 description: Optional[str] = Field( 2387 None, description="The description/purpose of the config migration." 2388 ) 2389 transformations: List[ 2390 Union[ 2391 ConfigRemapField, 2392 ConfigAddFields, 2393 ConfigRemoveFields, 2394 CustomConfigTransformation, 2395 ] 2396 ] = Field( 2397 ..., 2398 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2399 title="Transformations", 2400 ) 2401 2402 2403class ConfigNormalizationRules(BaseModel): 2404 class Config: 2405 extra = Extra.forbid 2406 2407 type: Literal["ConfigNormalizationRules"] 2408 config_migrations: Optional[List[ConfigMigration]] = Field( 2409 [], 2410 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2411 title="Config Migrations", 2412 ) 2413 transformations: Optional[ 2414 List[ 2415 Union[ 2416 ConfigRemapField, 2417 ConfigAddFields, 2418 ConfigRemoveFields, 2419 CustomConfigTransformation, 2420 ] 2421 ] 2422 ] = Field( 2423 [], 2424 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2425 title="Transformations", 2426 ) 2427 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2428 [], 2429 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2430 title="Validations", 2431 ) 2432 2433 2434class Spec(BaseModel): 2435 type: Literal["Spec"] 2436 connection_specification: Dict[str, Any] = Field( 2437 ..., 2438 description="A connection specification describing how a the connector can be configured.", 2439 title="Connection Specification", 2440 ) 2441 documentation_url: Optional[str] = Field( 2442 None, 2443 description="URL of the connector's documentation page.", 2444 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2445 title="Documentation URL", 2446 ) 2447 advanced_auth: Optional[AuthFlow] = Field( 2448 None, 2449 description="Advanced specification for configuring the authentication flow.", 2450 title="Advanced Auth", 2451 ) 2452 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2453 None, title="Config Normalization Rules" 2454 ) 2455 2456 2457class DeclarativeSource1(BaseModel): 2458 class Config: 2459 extra = Extra.forbid 2460 2461 type: Literal["DeclarativeSource"] 2462 check: Union[CheckStream, CheckDynamicStream] 2463 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2464 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2465 version: str = Field( 2466 ..., 2467 description="The version of the Airbyte CDK used to build and test the source.", 2468 ) 2469 schemas: Optional[Schemas] = None 2470 definitions: Optional[Dict[str, Any]] = None 2471 spec: Optional[Spec] = None 2472 concurrency_level: Optional[ConcurrencyLevel] = None 2473 api_budget: Optional[HTTPAPIBudget] = None 2474 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2475 None, 2476 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2477 title="Stream Groups", 2478 ) 2479 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2480 None, 2481 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2482 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2483 title="Maximum Concurrent Asynchronous Jobs", 2484 ) 2485 metadata: Optional[Dict[str, Any]] = Field( 2486 None, 2487 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2488 ) 2489 description: Optional[str] = Field( 2490 None, 2491 description="A description of the connector. It will be presented on the Source documentation page.", 2492 ) 2493 2494 2495class DeclarativeSource2(BaseModel): 2496 class Config: 2497 extra = Extra.forbid 2498 2499 type: Literal["DeclarativeSource"] 2500 check: Union[CheckStream, CheckDynamicStream] 2501 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2502 None 2503 ) 2504 dynamic_streams: List[DynamicDeclarativeStream] 2505 version: str = Field( 2506 ..., 2507 description="The version of the Airbyte CDK used to build and test the source.", 2508 ) 2509 schemas: Optional[Schemas] = None 2510 definitions: Optional[Dict[str, Any]] = None 2511 spec: Optional[Spec] = None 2512 concurrency_level: Optional[ConcurrencyLevel] = None 2513 api_budget: Optional[HTTPAPIBudget] = None 2514 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2515 None, 2516 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2517 title="Stream Groups", 2518 ) 2519 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2520 None, 2521 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2522 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2523 title="Maximum Concurrent Asynchronous Jobs", 2524 ) 2525 metadata: Optional[Dict[str, Any]] = Field( 2526 None, 2527 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2528 ) 2529 description: Optional[str] = Field( 2530 None, 2531 description="A description of the connector. It will be presented on the Source documentation page.", 2532 ) 2533 2534 2535class DeclarativeSource(BaseModel): 2536 class Config: 2537 extra = Extra.forbid 2538 2539 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2540 ..., 2541 description="An API source that extracts data according to its declarative components.", 2542 title="DeclarativeSource", 2543 ) 2544 2545 2546class SelectiveAuthenticator(BaseModel): 2547 class Config: 2548 extra = Extra.allow 2549 2550 type: Literal["SelectiveAuthenticator"] 2551 authenticator_selection_path: List[str] = Field( 2552 ..., 2553 description="Path of the field in config with selected authenticator name", 2554 examples=[["auth"], ["auth", "type"]], 2555 title="Authenticator Selection Path", 2556 ) 2557 authenticators: Dict[ 2558 str, 2559 Union[ 2560 ApiKeyAuthenticator, 2561 BasicHttpAuthenticator, 2562 BearerAuthenticator, 2563 OAuthAuthenticator, 2564 JwtAuthenticator, 2565 SessionTokenAuthenticator, 2566 LegacySessionTokenAuthenticator, 2567 CustomAuthenticator, 2568 NoAuth, 2569 ], 2570 ] = Field( 2571 ..., 2572 description="Authenticators to select from.", 2573 examples=[ 2574 { 2575 "authenticators": { 2576 "token": "#/definitions/ApiKeyAuthenticator", 2577 "oauth": "#/definitions/OAuthAuthenticator", 2578 "jwt": "#/definitions/JwtAuthenticator", 2579 } 2580 } 2581 ], 2582 title="Authenticators", 2583 ) 2584 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2585 2586 2587class ConditionalStreams(BaseModel): 2588 type: Literal["ConditionalStreams"] 2589 condition: str = Field( 2590 ..., 2591 description="Condition that will be evaluated to determine if a set of streams should be available.", 2592 examples=["{{ config['is_sandbox'] }}"], 2593 title="Condition", 2594 ) 2595 streams: List[DeclarativeStream] = Field( 2596 ..., 2597 description="Streams that will be used during an operation based on the condition.", 2598 title="Streams", 2599 ) 2600 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2601 2602 2603class FileUploader(BaseModel): 2604 type: Literal["FileUploader"] 2605 requester: Union[HttpRequester, CustomRequester] = Field( 2606 ..., 2607 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2608 ) 2609 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2610 ..., 2611 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2612 ) 2613 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2614 None, 2615 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2616 ) 2617 filename_extractor: Optional[str] = Field( 2618 None, 2619 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2620 examples=[ 2621 "{{ record.id }}/{{ record.file_name }}/", 2622 "{{ record.id }}_{{ record.file_name }}/", 2623 ], 2624 ) 2625 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2626 2627 2628class DeclarativeStream(BaseModel): 2629 class Config: 2630 extra = Extra.allow 2631 2632 type: Literal["DeclarativeStream"] 2633 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2634 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2635 ..., 2636 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2637 title="Retriever", 2638 ) 2639 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2640 None, 2641 description="Component used to fetch data incrementally based on a time field in the data.", 2642 title="Incremental Sync", 2643 ) 2644 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2645 schema_loader: Optional[ 2646 Union[ 2647 InlineSchemaLoader, 2648 DynamicSchemaLoader, 2649 JsonFileSchemaLoader, 2650 List[ 2651 Union[ 2652 InlineSchemaLoader, 2653 DynamicSchemaLoader, 2654 JsonFileSchemaLoader, 2655 CustomSchemaLoader, 2656 ] 2657 ], 2658 CustomSchemaLoader, 2659 ] 2660 ] = Field( 2661 None, 2662 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2663 title="Schema Loader", 2664 ) 2665 transformations: Optional[ 2666 List[ 2667 Union[ 2668 AddFields, 2669 RemoveFields, 2670 KeysToLower, 2671 KeysToSnakeCase, 2672 FlattenFields, 2673 DpathFlattenFields, 2674 KeysReplace, 2675 CustomTransformation, 2676 ] 2677 ] 2678 ] = Field( 2679 None, 2680 description="A list of transformations to be applied to each output record.", 2681 title="Transformations", 2682 ) 2683 state_migrations: Optional[ 2684 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2685 ] = Field( 2686 [], 2687 description="Array of state migrations to be applied on the input state", 2688 title="State Migrations", 2689 ) 2690 file_uploader: Optional[FileUploader] = Field( 2691 None, 2692 description="(experimental) Describes how to fetch a file", 2693 title="File Uploader", 2694 ) 2695 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2696 2697 2698class SessionTokenAuthenticator(BaseModel): 2699 type: Literal["SessionTokenAuthenticator"] 2700 login_requester: HttpRequester = Field( 2701 ..., 2702 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2703 examples=[ 2704 { 2705 "type": "HttpRequester", 2706 "url_base": "https://my_api.com", 2707 "path": "/login", 2708 "authenticator": { 2709 "type": "BasicHttpAuthenticator", 2710 "username": "{{ config.username }}", 2711 "password": "{{ config.password }}", 2712 }, 2713 } 2714 ], 2715 title="Login Requester", 2716 ) 2717 session_token_path: List[str] = Field( 2718 ..., 2719 description="The path in the response body returned from the login requester to the session token.", 2720 examples=[["access_token"], ["result", "token"]], 2721 title="Session Token Path", 2722 ) 2723 expiration_duration: Optional[str] = Field( 2724 None, 2725 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2726 examples=["PT1H", "P1D"], 2727 title="Expiration Duration", 2728 ) 2729 request_authentication: Union[ 2730 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2731 ] = Field( 2732 ..., 2733 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2734 title="Data Request Authentication", 2735 ) 2736 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2737 None, description="Component used to decode the response.", title="Decoder" 2738 ) 2739 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2740 2741 2742class HttpRequester(BaseModelWithDeprecations): 2743 type: Literal["HttpRequester"] 2744 url_base: Optional[str] = Field( 2745 None, 2746 deprecated=True, 2747 deprecation_message="Use `url` field instead.", 2748 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2749 examples=[ 2750 "https://connect.squareup.com/v2", 2751 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2752 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2753 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2754 ], 2755 title="API Base URL", 2756 ) 2757 url: Optional[str] = Field( 2758 None, 2759 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2760 examples=[ 2761 "https://connect.squareup.com/v2", 2762 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2763 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2764 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2765 ], 2766 title="API Endpoint URL", 2767 ) 2768 path: Optional[str] = Field( 2769 None, 2770 deprecated=True, 2771 deprecation_message="Use `url` field instead.", 2772 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2773 examples=[ 2774 "/products", 2775 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2776 "/trades/{{ config['symbol_id'] }}/history", 2777 ], 2778 title="URL Path", 2779 ) 2780 http_method: Optional[HttpMethod] = Field( 2781 HttpMethod.GET, 2782 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2783 examples=["GET", "POST"], 2784 title="HTTP Method", 2785 ) 2786 authenticator: Optional[ 2787 Union[ 2788 ApiKeyAuthenticator, 2789 BasicHttpAuthenticator, 2790 BearerAuthenticator, 2791 OAuthAuthenticator, 2792 JwtAuthenticator, 2793 SessionTokenAuthenticator, 2794 SelectiveAuthenticator, 2795 CustomAuthenticator, 2796 NoAuth, 2797 LegacySessionTokenAuthenticator, 2798 ] 2799 ] = Field( 2800 None, 2801 description="Authentication method to use for requests sent to the API.", 2802 title="Authenticator", 2803 ) 2804 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2805 None, 2806 deprecated=True, 2807 deprecation_message="Use `query_properties` field instead.", 2808 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2809 title="Fetch Properties from Endpoint", 2810 ) 2811 query_properties: Optional[QueryProperties] = Field( 2812 None, 2813 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2814 title="Query Properties", 2815 ) 2816 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2817 None, 2818 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2819 examples=[ 2820 {"unit": "day"}, 2821 { 2822 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2823 }, 2824 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2825 {"sort_by[asc]": "updated_at"}, 2826 ], 2827 title="Query Parameters", 2828 ) 2829 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2830 None, 2831 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2832 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2833 title="Request Headers", 2834 ) 2835 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2836 None, 2837 deprecated=True, 2838 deprecation_message="Use `request_body` field instead.", 2839 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2840 examples=[ 2841 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2842 ], 2843 title="Request Body Payload (Non-JSON)", 2844 ) 2845 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2846 None, 2847 deprecated=True, 2848 deprecation_message="Use `request_body` field instead.", 2849 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2850 examples=[ 2851 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2852 {"key": "{{ config['value'] }}"}, 2853 {"sort": {"field": "updated_at", "order": "ascending"}}, 2854 ], 2855 title="Request Body JSON Payload", 2856 ) 2857 request_body: Optional[ 2858 Union[ 2859 RequestBodyPlainText, 2860 RequestBodyUrlEncodedForm, 2861 RequestBodyJsonObject, 2862 RequestBodyGraphQL, 2863 ] 2864 ] = Field( 2865 None, 2866 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2867 title="Request Body", 2868 ) 2869 error_handler: Optional[ 2870 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2871 ] = Field( 2872 None, 2873 description="Error handler component that defines how to handle errors.", 2874 title="Error Handler", 2875 ) 2876 use_cache: Optional[bool] = Field( 2877 False, 2878 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2879 title="Use Cache", 2880 ) 2881 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2882 2883 2884class DynamicSchemaLoader(BaseModel): 2885 type: Literal["DynamicSchemaLoader"] 2886 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2887 ..., 2888 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2889 title="Retriever", 2890 ) 2891 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2892 None, 2893 description="Responsible for filtering fields to be added to json schema.", 2894 title="Schema Filter", 2895 ) 2896 schema_transformations: Optional[ 2897 List[ 2898 Union[ 2899 AddFields, 2900 RemoveFields, 2901 KeysToLower, 2902 KeysToSnakeCase, 2903 FlattenFields, 2904 DpathFlattenFields, 2905 KeysReplace, 2906 CustomTransformation, 2907 ] 2908 ] 2909 ] = Field( 2910 None, 2911 description="A list of transformations to be applied to the schema.", 2912 title="Schema Transformations", 2913 ) 2914 schema_type_identifier: SchemaTypeIdentifier 2915 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2916 2917 2918class ParentStreamConfig(BaseModel): 2919 type: Literal["ParentStreamConfig"] 2920 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2921 ..., description="Reference to the parent stream.", title="Parent Stream" 2922 ) 2923 parent_key: str = Field( 2924 ..., 2925 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2926 examples=["id", "{{ config['parent_record_id'] }}"], 2927 title="Parent Key", 2928 ) 2929 partition_field: str = Field( 2930 ..., 2931 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2932 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2933 title="Current Parent Key Value Identifier", 2934 ) 2935 request_option: Optional[RequestOption] = Field( 2936 None, 2937 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2938 title="Request Option", 2939 ) 2940 incremental_dependency: Optional[bool] = Field( 2941 False, 2942 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2943 title="Incremental Dependency", 2944 ) 2945 lazy_read_pointer: Optional[List[str]] = Field( 2946 [], 2947 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2948 title="Lazy Read Pointer", 2949 ) 2950 extra_fields: Optional[List[List[str]]] = Field( 2951 None, 2952 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2953 title="Extra Fields", 2954 ) 2955 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2956 2957 2958class PropertiesFromEndpoint(BaseModel): 2959 type: Literal["PropertiesFromEndpoint"] 2960 property_field_path: List[str] = Field( 2961 ..., 2962 description="Describes the path to the field that should be extracted", 2963 examples=[["name"]], 2964 ) 2965 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2966 ..., 2967 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2968 ) 2969 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2970 2971 2972class QueryProperties(BaseModel): 2973 type: Literal["QueryProperties"] 2974 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2975 ..., 2976 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2977 title="Property List", 2978 ) 2979 always_include_properties: Optional[List[str]] = Field( 2980 None, 2981 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2982 title="Always Include Properties", 2983 ) 2984 property_chunking: Optional[PropertyChunking] = Field( 2985 None, 2986 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2987 title="Property Chunking", 2988 ) 2989 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2990 None, 2991 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2992 title="Property Selector", 2993 ) 2994 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2995 2996 2997class StateDelegatingStream(BaseModel): 2998 type: Literal["StateDelegatingStream"] 2999 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3000 full_refresh_stream: DeclarativeStream = Field( 3001 ..., 3002 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3003 title="Full Refresh Stream", 3004 ) 3005 incremental_stream: DeclarativeStream = Field( 3006 ..., 3007 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3008 title="Incremental Stream", 3009 ) 3010 api_retention_period: Optional[str] = Field( 3011 None, 3012 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3013 examples=["P30D", "P90D", "P1Y"], 3014 title="API Retention Period", 3015 ) 3016 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3017 3018 3019class SimpleRetriever(BaseModel): 3020 type: Literal["SimpleRetriever"] 3021 requester: Union[HttpRequester, CustomRequester] = Field( 3022 ..., 3023 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3024 ) 3025 decoder: Optional[ 3026 Union[ 3027 JsonDecoder, 3028 JsonItemsDecoder, 3029 XmlDecoder, 3030 CsvDecoder, 3031 JsonlDecoder, 3032 GzipDecoder, 3033 IterableDecoder, 3034 ZipfileDecoder, 3035 CustomDecoder, 3036 ] 3037 ] = Field( 3038 None, 3039 description="Component decoding the response so records can be extracted.", 3040 title="HTTP Response Format", 3041 ) 3042 record_selector: RecordSelector = Field( 3043 ..., 3044 description="Component that describes how to extract records from a HTTP response.", 3045 ) 3046 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3047 None, 3048 description="Paginator component that describes how to navigate through the API's pages.", 3049 ) 3050 pagination_reset: Optional[PaginationReset] = Field( 3051 None, 3052 description="Describes what triggers pagination reset and how to handle it.", 3053 ) 3054 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3055 False, 3056 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3057 ) 3058 partition_router: Optional[ 3059 Union[ 3060 SubstreamPartitionRouter, 3061 ListPartitionRouter, 3062 GroupingPartitionRouter, 3063 CustomPartitionRouter, 3064 List[ 3065 Union[ 3066 SubstreamPartitionRouter, 3067 ListPartitionRouter, 3068 GroupingPartitionRouter, 3069 CustomPartitionRouter, 3070 ] 3071 ], 3072 ] 3073 ] = Field( 3074 None, 3075 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3076 title="Partition Router", 3077 ) 3078 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3079 3080 3081class AsyncRetriever(BaseModel): 3082 type: Literal["AsyncRetriever"] 3083 record_selector: RecordSelector = Field( 3084 ..., 3085 description="Component that describes how to extract records from a HTTP response.", 3086 ) 3087 status_mapping: AsyncJobStatusMap = Field( 3088 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3089 ) 3090 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3091 ..., description="Responsible for fetching the actual status of the async job." 3092 ) 3093 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3094 None, 3095 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3096 ) 3097 download_extractor: Optional[ 3098 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3099 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3100 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3101 ..., 3102 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3103 ) 3104 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3105 ..., 3106 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3107 ) 3108 polling_job_timeout: Optional[Union[int, str]] = Field( 3109 None, 3110 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3111 ) 3112 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3113 None, 3114 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3115 ge=1, 3116 ) 3117 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3118 None, 3119 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3120 ) 3121 download_requester: Union[HttpRequester, CustomRequester] = Field( 3122 ..., 3123 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3124 ) 3125 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3126 None, 3127 description="Paginator component that describes how to navigate through the API's pages during download.", 3128 ) 3129 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3130 None, 3131 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3132 ) 3133 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3134 None, 3135 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3136 ) 3137 partition_router: Optional[ 3138 Union[ 3139 ListPartitionRouter, 3140 SubstreamPartitionRouter, 3141 GroupingPartitionRouter, 3142 CustomPartitionRouter, 3143 List[ 3144 Union[ 3145 ListPartitionRouter, 3146 SubstreamPartitionRouter, 3147 GroupingPartitionRouter, 3148 CustomPartitionRouter, 3149 ] 3150 ], 3151 ] 3152 ] = Field( 3153 [], 3154 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3155 title="Partition Router", 3156 ) 3157 decoder: Optional[ 3158 Union[ 3159 CsvDecoder, 3160 GzipDecoder, 3161 JsonDecoder, 3162 JsonItemsDecoder, 3163 JsonlDecoder, 3164 IterableDecoder, 3165 XmlDecoder, 3166 ZipfileDecoder, 3167 CustomDecoder, 3168 ] 3169 ] = Field( 3170 None, 3171 description="Component decoding the response so records can be extracted.", 3172 title="HTTP Response Format", 3173 ) 3174 download_decoder: Optional[ 3175 Union[ 3176 CsvDecoder, 3177 GzipDecoder, 3178 JsonDecoder, 3179 JsonItemsDecoder, 3180 JsonlDecoder, 3181 IterableDecoder, 3182 XmlDecoder, 3183 ZipfileDecoder, 3184 CustomDecoder, 3185 ] 3186 ] = Field( 3187 None, 3188 description="Component decoding the download response so records can be extracted.", 3189 title="Download HTTP Response Format", 3190 ) 3191 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3192 3193 3194class BlockSimultaneousSyncsAction(BaseModel): 3195 type: Literal["BlockSimultaneousSyncsAction"] 3196 3197 3198class StreamGroup(BaseModel): 3199 streams: List[str] = Field( 3200 ..., 3201 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3202 title="Streams", 3203 ) 3204 action: BlockSimultaneousSyncsAction = Field( 3205 ..., 3206 description="The action to apply to streams in this group.", 3207 title="Action", 3208 ) 3209 3210 3211class SubstreamPartitionRouter(BaseModel): 3212 type: Literal["SubstreamPartitionRouter"] 3213 parent_stream_configs: List[ParentStreamConfig] = Field( 3214 ..., 3215 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3216 title="Parent Stream Configs", 3217 ) 3218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3219 3220 3221class GroupingPartitionRouter(BaseModel): 3222 type: Literal["GroupingPartitionRouter"] 3223 group_size: int = Field( 3224 ..., 3225 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3226 examples=[10, 50], 3227 title="Group Size", 3228 ) 3229 underlying_partition_router: Union[ 3230 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3231 ] = Field( 3232 ..., 3233 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3234 title="Underlying Partition Router", 3235 ) 3236 deduplicate: Optional[bool] = Field( 3237 True, 3238 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3239 title="Deduplicate Partitions", 3240 ) 3241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3242 3243 3244class HttpComponentsResolver(BaseModel): 3245 type: Literal["HttpComponentsResolver"] 3246 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3247 ..., 3248 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3249 title="Retriever", 3250 ) 3251 components_mapping: List[ComponentMappingDefinition] 3252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3253 3254 3255class DynamicDeclarativeStream(BaseModel): 3256 type: Literal["DynamicDeclarativeStream"] 3257 name: Optional[str] = Field( 3258 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3259 ) 3260 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3261 ..., description="Reference to the stream template.", title="Stream Template" 3262 ) 3263 components_resolver: Union[ 3264 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3265 ] = Field( 3266 ..., 3267 description="Component resolve and populates stream templates with components values.", 3268 title="Components Resolver", 3269 ) 3270 use_parent_parameters: Optional[bool] = Field( 3271 True, 3272 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3273 title="Use Parent Parameters", 3274 ) 3275 3276 3277ComplexFieldType.update_forward_refs() 3278GzipDecoder.update_forward_refs() 3279CompositeErrorHandler.update_forward_refs() 3280DeclarativeSource1.update_forward_refs() 3281DeclarativeSource2.update_forward_refs() 3282SelectiveAuthenticator.update_forward_refs() 3283ConditionalStreams.update_forward_refs() 3284FileUploader.update_forward_refs() 3285DeclarativeStream.update_forward_refs() 3286SessionTokenAuthenticator.update_forward_refs() 3287HttpRequester.update_forward_refs() 3288DynamicSchemaLoader.update_forward_refs() 3289ParentStreamConfig.update_forward_refs() 3290PropertiesFromEndpoint.update_forward_refs() 3291SimpleRetriever.update_forward_refs() 3292AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 jitter_range_in_seconds: Optional[float] = Field( 109 None, 110 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 111 examples=[15], 112 ge=0, 113 title="Jitter Range", 114 ) 115 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
118class CursorPagination(BaseModel): 119 type: Literal["CursorPagination"] 120 cursor_value: str = Field( 121 ..., 122 description="Value of the cursor defining the next page to fetch.", 123 examples=[ 124 "{{ headers.link.next.cursor }}", 125 "{{ last_record['key'] }}", 126 "{{ response['nextPage'] }}", 127 ], 128 title="Cursor Value", 129 ) 130 page_size: Optional[Union[int, str]] = Field( 131 None, 132 description="The number of records to include in each pages.", 133 examples=[100, "{{ config['page_size'] }}"], 134 title="Page Size", 135 ) 136 stop_condition: Optional[str] = Field( 137 None, 138 description="Template string evaluating when to stop paginating.", 139 examples=[ 140 "{{ response.data.has_more is false }}", 141 "{{ 'next' not in headers['link'] }}", 142 ], 143 title="Stop Condition", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
148class CustomAuthenticator(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomAuthenticator"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
162class CustomBackoffStrategy(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomBackoffStrategy"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomBackoffStrategy"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
176class CustomErrorHandler(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomErrorHandler"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomErrorHandler"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
190class CustomPaginationStrategy(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomPaginationStrategy"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomPaginationStrategy"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
204class CustomRecordExtractor(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordExtractor"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomRecordExtractor"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
218class CustomRecordFilter(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRecordFilter"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomCustomRecordFilter"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
232class CustomRequester(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRequester"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRecordExtractor"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
246class CustomRetriever(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomRetriever"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomRetriever"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
260class CustomPartitionRouter(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomPartitionRouter"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomPartitionRouter"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
274class CustomSchemaLoader(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaLoader"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 282 examples=["source_railz.components.MyCustomSchemaLoader"], 283 title="Class Name", 284 ) 285 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
288class CustomSchemaNormalization(BaseModel): 289 class Config: 290 extra = Extra.allow 291 292 type: Literal["CustomSchemaNormalization"] 293 class_name: str = Field( 294 ..., 295 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 296 examples=[ 297 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 298 ], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
304class CustomStateMigration(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomStateMigration"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomStateMigration"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
318class CustomTransformation(BaseModel): 319 class Config: 320 extra = Extra.allow 321 322 type: Literal["CustomTransformation"] 323 class_name: str = Field( 324 ..., 325 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 326 examples=["source_railz.components.MyCustomTransformation"], 327 title="Class Name", 328 ) 329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
349class Algorithm(Enum): 350 HS256 = "HS256" 351 HS384 = "HS384" 352 HS512 = "HS512" 353 ES256 = "ES256" 354 ES256K = "ES256K" 355 ES384 = "ES384" 356 ES512 = "ES512" 357 RS256 = "RS256" 358 RS384 = "RS384" 359 RS512 = "RS512" 360 PS256 = "PS256" 361 PS384 = "PS384" 362 PS512 = "PS512" 363 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
366class JwtHeaders(BaseModel): 367 class Config: 368 extra = Extra.forbid 369 370 kid: Optional[str] = Field( 371 None, 372 description="Private key ID for user account.", 373 examples=["{{ config['kid'] }}"], 374 title="Key Identifier", 375 ) 376 typ: Optional[str] = Field( 377 "JWT", 378 description="The media type of the complete JWT.", 379 examples=["JWT"], 380 title="Type", 381 ) 382 cty: Optional[str] = Field( 383 None, 384 description="Content type of JWT header.", 385 examples=["JWT"], 386 title="Content Type", 387 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
390class JwtPayload(BaseModel): 391 class Config: 392 extra = Extra.forbid 393 394 iss: Optional[str] = Field( 395 None, 396 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 397 examples=["{{ config['iss'] }}"], 398 title="Issuer", 399 ) 400 sub: Optional[str] = Field( 401 None, 402 description="The subject of the JWT. Commonly defined by the API.", 403 title="Subject", 404 ) 405 aud: Optional[str] = Field( 406 None, 407 description="The recipient that the JWT is intended for. Commonly defined by the API.", 408 examples=["appstoreconnect-v1"], 409 title="Audience", 410 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
413class RefreshTokenUpdater(BaseModel): 414 refresh_token_name: Optional[str] = Field( 415 "refresh_token", 416 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 417 examples=["refresh_token"], 418 title="Refresh Token Property Name", 419 ) 420 access_token_config_path: Optional[List[str]] = Field( 421 ["credentials", "access_token"], 422 description="Config path to the access token. Make sure the field actually exists in the config.", 423 examples=[["credentials", "access_token"], ["access_token"]], 424 title="Config Path To Access Token", 425 ) 426 refresh_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "refresh_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "refresh_token"], ["refresh_token"]], 430 title="Config Path To Refresh Token", 431 ) 432 token_expiry_date_config_path: Optional[List[str]] = Field( 433 ["credentials", "token_expiry_date"], 434 description="Config path to the expiry date. Make sure actually exists in the config.", 435 examples=[["credentials", "token_expiry_date"]], 436 title="Config Path To Expiry Date", 437 ) 438 refresh_token_error_status_codes: Optional[List[int]] = Field( 439 [], 440 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 441 examples=[[400, 500]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 443 ) 444 refresh_token_error_key: Optional[str] = Field( 445 "", 446 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 447 examples=["error"], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 449 ) 450 refresh_token_error_values: Optional[List[str]] = Field( 451 [], 452 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 453 examples=[["invalid_grant", "invalid_permissions"]], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 455 )
class
Rate(pydantic.v1.main.BaseModel):
458class Rate(BaseModel): 459 class Config: 460 extra = Extra.allow 461 462 limit: Union[int, str] = Field( 463 ..., 464 description="The maximum number of calls allowed within the interval.", 465 title="Limit", 466 ) 467 interval: str = Field( 468 ..., 469 description="The time interval for the rate limit.", 470 examples=["PT1H", "P1D"], 471 title="Interval", 472 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
475class HttpRequestRegexMatcher(BaseModel): 476 class Config: 477 extra = Extra.allow 478 479 method: Optional[str] = Field( 480 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 481 ) 482 url_base: Optional[str] = Field( 483 None, 484 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 485 title="URL Base", 486 ) 487 url_path_pattern: Optional[str] = Field( 488 None, 489 description="A regular expression pattern to match the URL path.", 490 title="URL Path Pattern", 491 ) 492 params: Optional[Dict[str, Any]] = Field( 493 None, description="The query parameters to match.", title="Parameters" 494 ) 495 headers: Optional[Dict[str, Any]] = Field( 496 None, description="The headers to match.", title="Headers" 497 ) 498 weight: Optional[Union[int, str]] = Field( 499 None, 500 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 501 title="Weight", 502 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
OnNoRecords(enum.Enum):
An enumeration.
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
515class ExponentialBackoffStrategy(BaseModel): 516 type: Literal["ExponentialBackoffStrategy"] 517 factor: Optional[Union[float, str]] = Field( 518 5, 519 description="Multiplicative constant applied on each retry.", 520 examples=[5, 5.5, "10"], 521 title="Factor", 522 ) 523 jitter_range_in_seconds: Optional[float] = Field( 524 None, 525 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 526 examples=[2], 527 ge=0, 528 title="Jitter Range", 529 ) 530 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
533class GroupByKeyMergeStrategy(BaseModel): 534 type: Literal["GroupByKeyMergeStrategy"] 535 key: Union[str, List[str]] = Field( 536 ..., 537 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 538 examples=["id", ["parent_id", "end_date"]], 539 title="Key", 540 ) 541 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
553class Action(Enum): 554 SUCCESS = "SUCCESS" 555 FAIL = "FAIL" 556 RETRY = "RETRY" 557 IGNORE = "IGNORE" 558 RESET_PAGINATION = "RESET_PAGINATION" 559 RATE_LIMITED = "RATE_LIMITED" 560 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
563class FailureType(Enum): 564 system_error = "system_error" 565 config_error = "config_error" 566 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
569class HttpResponseFilter(BaseModel): 570 type: Literal["HttpResponseFilter"] 571 action: Optional[Action] = Field( 572 None, 573 description="Action to execute if a response matches the filter.", 574 examples=[ 575 "SUCCESS", 576 "FAIL", 577 "RETRY", 578 "IGNORE", 579 "RESET_PAGINATION", 580 "RATE_LIMITED", 581 "REFRESH_TOKEN_THEN_RETRY", 582 ], 583 title="Action", 584 ) 585 failure_type: Optional[FailureType] = Field( 586 None, 587 description="Failure type of traced exception if a response matches the filter.", 588 examples=["system_error", "config_error", "transient_error"], 589 title="Failure Type", 590 ) 591 error_message: Optional[str] = Field( 592 None, 593 description="Error Message to display if the response matches the filter.", 594 title="Error Message", 595 ) 596 error_message_contains: Optional[str] = Field( 597 None, 598 description="Match the response if its error message contains the substring.", 599 example=["This API operation is not enabled for this site"], 600 title="Error Message Substring", 601 ) 602 http_codes: Optional[List[int]] = Field( 603 None, 604 description="Match the response if its HTTP code is included in this list.", 605 examples=[[420, 429], [500]], 606 title="HTTP Codes", 607 unique_items=True, 608 ) 609 predicate: Optional[str] = Field( 610 None, 611 description="Match the response if the predicate evaluates to true.", 612 examples=[ 613 "{{ 'Too much requests' in response }}", 614 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 615 ], 616 title="Predicate", 617 ) 618 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
621class ComplexFieldType(BaseModel): 622 field_type: str 623 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
626class TypesMap(BaseModel): 627 target_type: Union[str, List[str], ComplexFieldType] 628 current_type: Union[str, List[str]] 629 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
632class SchemaTypeIdentifier(BaseModel): 633 type: Optional[Literal["SchemaTypeIdentifier"]] = None 634 schema_pointer: Optional[List[str]] = Field( 635 [], 636 description="List of nested fields defining the schema field path to extract. Defaults to [].", 637 title="Schema Path", 638 ) 639 key_pointer: List[str] = Field( 640 ..., 641 description="List of potentially nested fields describing the full path of the field key to extract.", 642 title="Key Path", 643 ) 644 type_pointer: Optional[List[str]] = Field( 645 None, 646 description="List of potentially nested fields describing the full path of the field type to extract.", 647 title="Type Path", 648 ) 649 types_mapping: Optional[List[TypesMap]] = None 650 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
653class InlineSchemaLoader(BaseModel): 654 type: Literal["InlineSchemaLoader"] 655 schema_: Optional[Dict[str, Any]] = Field( 656 None, 657 alias="schema", 658 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 659 title="Schema", 660 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
663class JsonFileSchemaLoader(BaseModel): 664 type: Literal["JsonFileSchemaLoader"] 665 file_path: Optional[str] = Field( 666 None, 667 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 668 example=["./schemas/users.json"], 669 title="File Path", 670 ) 671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
678class JsonItemsDecoder(BaseModel): 679 type: Literal["JsonItemsDecoder"] 680 items_path: str = Field( 681 ..., 682 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 683 title="Items Path", 684 ) 685 encoding: Optional[str] = Field( 686 "utf-8", 687 description="The character encoding of the JSON data. Defaults to UTF-8.", 688 title="Encoding", 689 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
706class FlattenFields(BaseModel): 707 type: Literal["FlattenFields"] 708 flatten_lists: Optional[bool] = Field( 709 True, 710 description="Whether to flatten lists or leave it as is. Default is True.", 711 title="Flatten Lists", 712 ) 713 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
716class KeyTransformation(BaseModel): 717 type: Literal["KeyTransformation"] 718 prefix: Optional[str] = Field( 719 None, 720 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 721 examples=["flattened_"], 722 title="Key Prefix", 723 ) 724 suffix: Optional[str] = Field( 725 None, 726 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 727 examples=["_flattened"], 728 title="Key Suffix", 729 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
732class DpathFlattenFields(BaseModel): 733 type: Literal["DpathFlattenFields"] 734 field_path: List[str] = Field( 735 ..., 736 description="A path to field that needs to be flattened.", 737 examples=[["data"], ["data", "*", "field"]], 738 title="Field Path", 739 ) 740 delete_origin_value: Optional[bool] = Field( 741 None, 742 description="Whether to delete the origin value or keep it. Default is False.", 743 title="Delete Origin Value", 744 ) 745 replace_record: Optional[bool] = Field( 746 None, 747 description="Whether to replace the origin record or not. Default is False.", 748 title="Replace Origin Record", 749 ) 750 key_transformation: Optional[KeyTransformation] = Field( 751 None, 752 description="Transformation for object keys. If not provided, original key will be used.", 753 title="Key transformation", 754 ) 755 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
758class KeysReplace(BaseModel): 759 type: Literal["KeysReplace"] 760 old: str = Field( 761 ..., 762 description="Old value to replace.", 763 examples=[ 764 " ", 765 "{{ record.id }}", 766 "{{ config['id'] }}", 767 "{{ stream_slice['id'] }}", 768 ], 769 title="Old value", 770 ) 771 new: str = Field( 772 ..., 773 description="New value to set.", 774 examples=[ 775 "_", 776 "{{ record.id }}", 777 "{{ config['id'] }}", 778 "{{ stream_slice['id'] }}", 779 ], 780 title="New value", 781 ) 782 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
793class CustomDecoder(BaseModel): 794 class Config: 795 extra = Extra.allow 796 797 type: Literal["CustomDecoder"] 798 class_name: str = Field( 799 ..., 800 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 801 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 802 title="Class Name", 803 ) 804 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
807class MinMaxDatetime(BaseModel): 808 type: Literal["MinMaxDatetime"] 809 datetime: str = Field( 810 ..., 811 description="Datetime value.", 812 examples=[ 813 "2021-01-01", 814 "2021-01-01T00:00:00Z", 815 "{{ config['start_time'] }}", 816 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 817 ], 818 title="Datetime", 819 ) 820 datetime_format: Optional[str] = Field( 821 "", 822 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 823 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 824 title="Datetime Format", 825 ) 826 max_datetime: Optional[str] = Field( 827 None, 828 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 829 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 830 title="Max Datetime", 831 ) 832 min_datetime: Optional[str] = Field( 833 None, 834 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 835 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 836 title="Min Datetime", 837 ) 838 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
868class OauthConnectorInputSpecification(BaseModel): 869 class Config: 870 extra = Extra.allow 871 872 consent_url: str = Field( 873 ..., 874 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 875 examples=[ 876 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 877 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 878 ], 879 title="Consent URL", 880 ) 881 scope: Optional[str] = Field( 882 None, 883 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 884 examples=["user:read user:read_orders workspaces:read"], 885 title="Scopes", 886 ) 887 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 888 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 889 # The CDK schema defines the manifest contract; the platform reads these fields 890 # during the OAuth consent flow to build the authorization URL. 891 scopes: Optional[List[OAuthScope]] = Field( 892 None, 893 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 894 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 895 title="Scopes", 896 ) 897 optional_scopes: Optional[List[OAuthScope]] = Field( 898 None, 899 description="Optional OAuth scope objects that may or may not be granted.", 900 examples=[[{"scope": "admin:read"}]], 901 title="Optional Scopes", 902 ) 903 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 904 ScopesJoinStrategy.space, 905 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 906 title="Scopes Join Strategy", 907 ) 908 access_token_url: str = Field( 909 ..., 910 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 911 examples=[ 912 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 913 ], 914 title="Access Token URL", 915 ) 916 access_token_headers: Optional[Dict[str, Any]] = Field( 917 None, 918 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 919 examples=[ 920 { 921 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 922 } 923 ], 924 title="Access Token Headers", 925 ) 926 access_token_params: Optional[Dict[str, Any]] = Field( 927 None, 928 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 929 examples=[ 930 { 931 "{{ auth_code_key }}": "{{ auth_code_value }}", 932 "{{ client_id_key }}": "{{ client_id_value }}", 933 "{{ client_secret_key }}": "{{ client_secret_value }}", 934 } 935 ], 936 title="Access Token Query Params (Json Encoded)", 937 ) 938 extract_output: Optional[List[str]] = Field( 939 None, 940 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 941 examples=[["access_token", "refresh_token", "other_field"]], 942 title="Extract Output", 943 ) 944 state: Optional[State] = Field( 945 None, 946 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 947 examples=[{"min": 7, "max": 128}], 948 title="Configurable State Query Param", 949 ) 950 client_id_key: Optional[str] = Field( 951 None, 952 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 953 examples=["my_custom_client_id_key_name"], 954 title="Client ID Key Override", 955 ) 956 client_secret_key: Optional[str] = Field( 957 None, 958 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 959 examples=["my_custom_client_secret_key_name"], 960 title="Client Secret Key Override", 961 ) 962 scope_key: Optional[str] = Field( 963 None, 964 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 965 examples=["my_custom_scope_key_key_name"], 966 title="Scopes Key Override", 967 ) 968 state_key: Optional[str] = Field( 969 None, 970 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 971 examples=["my_custom_state_key_key_name"], 972 title="State Key Override", 973 ) 974 auth_code_key: Optional[str] = Field( 975 None, 976 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 977 examples=["my_custom_auth_code_key_name"], 978 title="Auth Code Key Override", 979 ) 980 redirect_uri_key: Optional[str] = Field( 981 None, 982 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 983 examples=["my_custom_redirect_uri_key_name"], 984 title="Redirect URI Key Override", 985 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
988class OAuthConfigSpecification(BaseModel): 989 class Config: 990 extra = Extra.allow 991 992 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 993 None, 994 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 995 examples=[ 996 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 997 { 998 "app_id": { 999 "type": "string", 1000 "path_in_connector_config": ["info", "app_id"], 1001 } 1002 }, 1003 ], 1004 title="OAuth user input", 1005 ) 1006 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1007 None, 1008 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1009 title="DeclarativeOAuth Connector Specification", 1010 ) 1011 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1012 None, 1013 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1014 examples=[ 1015 { 1016 "refresh_token": { 1017 "type": "string,", 1018 "path_in_connector_config": ["credentials", "refresh_token"], 1019 } 1020 } 1021 ], 1022 title="OAuth output specification", 1023 ) 1024 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1025 None, 1026 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1027 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1028 title="OAuth input specification", 1029 ) 1030 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1031 None, 1032 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1033 examples=[ 1034 { 1035 "client_id": { 1036 "type": "string,", 1037 "path_in_connector_config": ["credentials", "client_id"], 1038 }, 1039 "client_secret": { 1040 "type": "string,", 1041 "path_in_connector_config": ["credentials", "client_secret"], 1042 }, 1043 } 1044 ], 1045 title="OAuth server output specification", 1046 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1049class OffsetIncrement(BaseModel): 1050 type: Literal["OffsetIncrement"] 1051 page_size: Optional[Union[int, str]] = Field( 1052 None, 1053 description="The number of records to include in each pages.", 1054 examples=[100, "{{ config['page_size'] }}"], 1055 title="Limit", 1056 ) 1057 inject_on_first_request: Optional[bool] = Field( 1058 False, 1059 description="Using the `offset` with value `0` during the first request", 1060 title="Inject Offset on First Request", 1061 ) 1062 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1065class PageIncrement(BaseModel): 1066 type: Literal["PageIncrement"] 1067 page_size: Optional[Union[int, str]] = Field( 1068 None, 1069 description="The number of records to include in each pages.", 1070 examples=[100, "100", "{{ config['page_size'] }}"], 1071 title="Page Size", 1072 ) 1073 start_from_page: Optional[int] = Field( 1074 0, 1075 description="Index of the first page to request.", 1076 examples=[0, 1], 1077 title="Start From Page", 1078 ) 1079 inject_on_first_request: Optional[bool] = Field( 1080 False, 1081 description="Using the `page number` with value defined by `start_from_page` during the first request", 1082 title="Inject Page Number on First Request", 1083 ) 1084 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1087class PrimaryKey(BaseModel): 1088 __root__: Union[str, List[str], List[List[str]]] = Field( 1089 ..., 1090 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1091 examples=["id", ["code", "type"]], 1092 title="Primary Key", 1093 )
class
PropertyLimitType(enum.Enum):
1096class PropertyLimitType(Enum): 1097 characters = "characters" 1098 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1101class PropertyChunking(BaseModel): 1102 type: Literal["PropertyChunking"] 1103 property_limit_type: PropertyLimitType = Field( 1104 ..., 1105 description="The type used to determine the maximum number of properties per chunk", 1106 title="Property Limit Type", 1107 ) 1108 property_limit: Optional[int] = Field( 1109 None, 1110 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1111 title="Property Limit", 1112 ) 1113 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1114 None, 1115 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1116 title="Record Merge Strategy", 1117 ) 1118 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1121class RecordFilter(BaseModel): 1122 type: Literal["RecordFilter"] 1123 condition: Optional[str] = Field( 1124 "", 1125 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1126 examples=[ 1127 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1128 "{{ record.status in ['active', 'expired'] }}", 1129 ], 1130 title="Condition", 1131 ) 1132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1140class RemoveFields(BaseModel): 1141 type: Literal["RemoveFields"] 1142 condition: Optional[str] = Field( 1143 "", 1144 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1145 examples=[ 1146 "{{ property|string == '' }}", 1147 "{{ property is integer }}", 1148 "{{ property|length > 5 }}", 1149 "{{ property == 'some_string_to_match' }}", 1150 ], 1151 ) 1152 field_pointers: List[List[str]] = Field( 1153 ..., 1154 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1155 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1156 title="Field Paths", 1157 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1164class InjectInto(Enum): 1165 request_parameter = "request_parameter" 1166 header = "header" 1167 body_data = "body_data" 1168 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1171class RequestOption(BaseModel): 1172 type: Literal["RequestOption"] 1173 inject_into: InjectInto = Field( 1174 ..., 1175 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1176 examples=["request_parameter", "header", "body_data", "body_json"], 1177 title="Inject Into", 1178 ) 1179 field_name: Optional[str] = Field( 1180 None, 1181 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1182 examples=["segment_id"], 1183 title="Field Name", 1184 ) 1185 field_path: Optional[List[str]] = Field( 1186 None, 1187 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1188 examples=[["data", "viewer", "id"]], 1189 title="Field Path", 1190 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1200class LegacySessionTokenAuthenticator(BaseModel): 1201 type: Literal["LegacySessionTokenAuthenticator"] 1202 header: str = Field( 1203 ..., 1204 description="The name of the session token header that will be injected in the request", 1205 examples=["X-Session"], 1206 title="Session Request Header", 1207 ) 1208 login_url: str = Field( 1209 ..., 1210 description="Path of the login URL (do not include the base URL)", 1211 examples=["session"], 1212 title="Login Path", 1213 ) 1214 session_token: Optional[str] = Field( 1215 None, 1216 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1217 example=["{{ config['session_token'] }}"], 1218 title="Session Token", 1219 ) 1220 session_token_response_key: str = Field( 1221 ..., 1222 description="Name of the key of the session token to be extracted from the response", 1223 examples=["id"], 1224 title="Response Token Response Key", 1225 ) 1226 username: Optional[str] = Field( 1227 None, 1228 description="Username used to authenticate and obtain a session token", 1229 examples=[" {{ config['username'] }}"], 1230 title="Username", 1231 ) 1232 password: Optional[str] = Field( 1233 "", 1234 description="Password used to authenticate and obtain a session token", 1235 examples=["{{ config['password'] }}", ""], 1236 title="Password", 1237 ) 1238 validate_session_url: str = Field( 1239 ..., 1240 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1241 examples=["user/current"], 1242 title="Validate Session Path", 1243 ) 1244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1273class ValueType(Enum): 1274 string = "string" 1275 number = "number" 1276 integer = "integer" 1277 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1280class WaitTimeFromHeader(BaseModel): 1281 type: Literal["WaitTimeFromHeader"] 1282 header: str = Field( 1283 ..., 1284 description="The name of the response header defining how long to wait before retrying.", 1285 examples=["Retry-After"], 1286 title="Response Header Name", 1287 ) 1288 regex: Optional[str] = Field( 1289 None, 1290 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1291 examples=["([-+]?\\d+)"], 1292 title="Extraction Regex", 1293 ) 1294 max_waiting_time_in_seconds: Optional[float] = Field( 1295 None, 1296 description="Given the value extracted from the header is greater than this value, stop the stream.", 1297 examples=[3600], 1298 title="Max Waiting Time in Seconds", 1299 ) 1300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1303class WaitUntilTimeFromHeader(BaseModel): 1304 type: Literal["WaitUntilTimeFromHeader"] 1305 header: str = Field( 1306 ..., 1307 description="The name of the response header defining how long to wait before retrying.", 1308 examples=["wait_time"], 1309 title="Response Header", 1310 ) 1311 min_wait: Optional[Union[float, str]] = Field( 1312 None, 1313 description="Minimum time to wait before retrying.", 1314 examples=[10, "60"], 1315 title="Minimum Wait Time", 1316 ) 1317 regex: Optional[str] = Field( 1318 None, 1319 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1320 examples=["([-+]?\\d+)"], 1321 title="Extraction Regex", 1322 ) 1323 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1326class ComponentMappingDefinition(BaseModel): 1327 type: Literal["ComponentMappingDefinition"] 1328 field_path: List[str] = Field( 1329 ..., 1330 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1331 examples=[ 1332 ["name"], 1333 ["retriever", "requester", "url"], 1334 ["retriever", "requester", "{{ components_values.field }}"], 1335 ["*", "**", "name"], 1336 ], 1337 title="Field Path", 1338 ) 1339 value: str = Field( 1340 ..., 1341 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1342 examples=[ 1343 "{{ components_values['updates'] }}", 1344 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1345 "{{ config['segment_id'] }}", 1346 "{{ stream_slice['parent_id'] }}", 1347 "{{ stream_slice['extra_fields']['name'] }}", 1348 ], 1349 title="Value", 1350 ) 1351 value_type: Optional[ValueType] = Field( 1352 None, 1353 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1354 title="Value Type", 1355 ) 1356 create_or_update: Optional[bool] = Field( 1357 False, 1358 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1359 title="Create or Update", 1360 ) 1361 condition: Optional[str] = Field( 1362 None, 1363 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1364 examples=[ 1365 "{{ components_values.get('cursor_field', None) }}", 1366 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1367 ], 1368 title="Condition", 1369 ) 1370 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1373class StreamConfig(BaseModel): 1374 type: Literal["StreamConfig"] 1375 configs_pointer: List[str] = Field( 1376 ..., 1377 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1378 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1379 title="Configs Pointer", 1380 ) 1381 default_values: Optional[List[Dict[str, Any]]] = Field( 1382 None, 1383 description="A list of default values, each matching the structure expected from the parsed component value.", 1384 title="Default Values", 1385 ) 1386 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1389class ConfigComponentsResolver(BaseModel): 1390 type: Literal["ConfigComponentsResolver"] 1391 stream_config: Union[List[StreamConfig], StreamConfig] 1392 components_mapping: List[ComponentMappingDefinition] 1393 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1396class StreamParametersDefinition(BaseModel): 1397 type: Literal["StreamParametersDefinition"] 1398 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1399 ..., 1400 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1401 examples=[ 1402 [ 1403 { 1404 "name": "test stream", 1405 "$parameters": {"entity": "test entity"}, 1406 "primary_key": "test key", 1407 } 1408 ] 1409 ], 1410 title="Stream Parameters", 1411 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1414class ParametrizedComponentsResolver(BaseModel): 1415 type: Literal["ParametrizedComponentsResolver"] 1416 stream_parameters: StreamParametersDefinition 1417 components_mapping: List[ComponentMappingDefinition] 1418 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1443class ValidateAdheresToSchema(BaseModel): 1444 type: Literal["ValidateAdheresToSchema"] 1445 base_schema: Union[str, Dict[str, Any]] = Field( 1446 ..., 1447 description="The base JSON schema against which the user-provided schema will be validated.", 1448 examples=[ 1449 "{{ config['report_validation_schema'] }}", 1450 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1451 { 1452 "$schema": "http://json-schema.org/draft-07/schema#", 1453 "title": "Person", 1454 "type": "object", 1455 "properties": { 1456 "name": {"type": "string", "description": "The person's name"}, 1457 "age": { 1458 "type": "integer", 1459 "minimum": 0, 1460 "description": "The person's age", 1461 }, 1462 }, 1463 "required": ["name", "age"], 1464 }, 1465 ], 1466 title="Base JSON Schema", 1467 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1470class CustomValidationStrategy(BaseModel): 1471 class Config: 1472 extra = Extra.allow 1473 1474 type: Literal["CustomValidationStrategy"] 1475 class_name: str = Field( 1476 ..., 1477 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1478 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1479 title="Class Name", 1480 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1483class ConfigRemapField(BaseModel): 1484 type: Literal["ConfigRemapField"] 1485 map: Union[Dict[str, Any], str] = Field( 1486 ..., 1487 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1488 examples=[ 1489 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1490 "{{ config['status_mapping'] }}", 1491 ], 1492 title="Value Mapping", 1493 ) 1494 field_path: List[str] = Field( 1495 ..., 1496 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1497 examples=[ 1498 ["status"], 1499 ["data", "status"], 1500 ["data", "{{ config.name }}", "status"], 1501 ["data", "*", "status"], 1502 ], 1503 title="Field Path", 1504 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1507class ConfigRemoveFields(BaseModel): 1508 type: Literal["ConfigRemoveFields"] 1509 field_pointers: List[List[str]] = Field( 1510 ..., 1511 description="A list of field pointers to be removed from the config.", 1512 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1513 title="Field Pointers", 1514 ) 1515 condition: Optional[str] = Field( 1516 "", 1517 description="Fields will be removed if expression is evaluated to True.", 1518 examples=[ 1519 "{{ config['environemnt'] == 'sandbox' }}", 1520 "{{ property is integer }}", 1521 "{{ property|length > 5 }}", 1522 "{{ property == 'some_string_to_match' }}", 1523 ], 1524 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1527class CustomConfigTransformation(BaseModel): 1528 type: Literal["CustomConfigTransformation"] 1529 class_name: str = Field( 1530 ..., 1531 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1532 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1533 ) 1534 parameters: Optional[Dict[str, Any]] = Field( 1535 None, 1536 alias="$parameters", 1537 description="Additional parameters to be passed to the custom config transformation.", 1538 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1541class AddedFieldDefinition(BaseModel): 1542 type: Literal["AddedFieldDefinition"] 1543 path: List[str] = Field( 1544 ..., 1545 description="List of strings defining the path where to add the value on the record.", 1546 examples=[["segment_id"], ["metadata", "segment_id"]], 1547 title="Path", 1548 ) 1549 value: str = Field( 1550 ..., 1551 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1552 examples=[ 1553 "{{ record['updates'] }}", 1554 "{{ record['MetaData']['LastUpdatedTime'] }}", 1555 "{{ stream_partition['segment_id'] }}", 1556 ], 1557 title="Value", 1558 ) 1559 value_type: Optional[ValueType] = Field( 1560 None, 1561 description="Type of the value. If not specified, the type will be inferred from the value.", 1562 title="Value Type", 1563 ) 1564 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1567class AddFields(BaseModel): 1568 type: Literal["AddFields"] 1569 fields: List[AddedFieldDefinition] = Field( 1570 ..., 1571 description="List of transformations (path and corresponding value) that will be added to the record.", 1572 title="Fields", 1573 ) 1574 condition: Optional[str] = Field( 1575 "", 1576 description="Fields will be added if expression is evaluated to True.", 1577 examples=[ 1578 "{{ property|string == '' }}", 1579 "{{ property is integer }}", 1580 "{{ property|length > 5 }}", 1581 "{{ property == 'some_string_to_match' }}", 1582 ], 1583 ) 1584 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1587class ApiKeyAuthenticator(BaseModel): 1588 type: Literal["ApiKeyAuthenticator"] 1589 api_token: Optional[str] = Field( 1590 None, 1591 description="The API key to inject in the request. Fill it in the user inputs.", 1592 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1593 title="API Key", 1594 ) 1595 header: Optional[str] = Field( 1596 None, 1597 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1598 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1599 title="Header Name", 1600 ) 1601 inject_into: Optional[RequestOption] = Field( 1602 None, 1603 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1604 examples=[ 1605 {"inject_into": "header", "field_name": "Authorization"}, 1606 {"inject_into": "request_parameter", "field_name": "authKey"}, 1607 ], 1608 title="Inject API Key Into Outgoing HTTP Request", 1609 ) 1610 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1613class AuthFlow(BaseModel): 1614 auth_flow_type: Optional[AuthFlowType] = Field( 1615 None, description="The type of auth to use", title="Auth flow type" 1616 ) 1617 predicate_key: Optional[List[str]] = Field( 1618 None, 1619 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1620 examples=[["credentials", "auth_type"]], 1621 title="Predicate key", 1622 ) 1623 predicate_value: Optional[str] = Field( 1624 None, 1625 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1626 examples=["Oauth"], 1627 title="Predicate value", 1628 ) 1629 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1632class CheckStream(BaseModel): 1633 type: Literal["CheckStream"] 1634 stream_names: Optional[List[str]] = Field( 1635 None, 1636 description="Names of the streams to try reading from when running a check operation.", 1637 examples=[["users"], ["users", "contacts"]], 1638 title="Stream Names", 1639 ) 1640 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1643class IncrementingCountCursor(BaseModel): 1644 type: Literal["IncrementingCountCursor"] 1645 cursor_field: str = Field( 1646 ..., 1647 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1648 examples=["created_at", "{{ config['record_cursor'] }}"], 1649 title="Cursor Field", 1650 ) 1651 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1652 None, 1653 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1654 title="Allow Catalog Defined Cursor Field", 1655 ) 1656 start_value: Optional[Union[str, int]] = Field( 1657 None, 1658 description="The value that determines the earliest record that should be synced.", 1659 examples=[0, "{{ config['start_value'] }}"], 1660 title="Start Value", 1661 ) 1662 start_value_option: Optional[RequestOption] = Field( 1663 None, 1664 description="Optionally configures how the start value will be sent in requests to the source API.", 1665 title="Inject Start Value Into Outgoing HTTP Request", 1666 ) 1667 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1670class DatetimeBasedCursor(BaseModel): 1671 type: Literal["DatetimeBasedCursor"] 1672 clamping: Optional[Clamping] = Field( 1673 None, 1674 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1675 title="Date Range Clamping", 1676 ) 1677 cursor_field: str = Field( 1678 ..., 1679 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1680 examples=["created_at", "{{ config['record_cursor'] }}"], 1681 title="Cursor Field", 1682 ) 1683 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1684 None, 1685 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1686 title="Allow Catalog Defined Cursor Field", 1687 ) 1688 cursor_datetime_formats: Optional[List[str]] = Field( 1689 None, 1690 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1691 examples=[ 1692 "%Y-%m-%d", 1693 "%Y-%m-%d %H:%M:%S", 1694 "%Y-%m-%dT%H:%M:%S", 1695 "%Y-%m-%dT%H:%M:%SZ", 1696 "%Y-%m-%dT%H:%M:%S%z", 1697 "%Y-%m-%dT%H:%M:%S.%fZ", 1698 "%Y-%m-%dT%H:%M:%S.%f%z", 1699 "%Y-%m-%d %H:%M:%S.%f+00:00", 1700 "%s", 1701 "%ms", 1702 ], 1703 title="Cursor Datetime Formats", 1704 ) 1705 start_datetime: Union[MinMaxDatetime, str] = Field( 1706 ..., 1707 description="The datetime that determines the earliest record that should be synced.", 1708 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1709 title="Start Datetime", 1710 ) 1711 start_time_option: Optional[RequestOption] = Field( 1712 None, 1713 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1714 title="Inject Start Time Into Outgoing HTTP Request", 1715 ) 1716 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1717 None, 1718 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1719 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1720 title="End Datetime", 1721 ) 1722 end_time_option: Optional[RequestOption] = Field( 1723 None, 1724 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1725 title="Inject End Time Into Outgoing HTTP Request", 1726 ) 1727 datetime_format: str = Field( 1728 ..., 1729 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1730 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1731 title="Outgoing Datetime Format", 1732 ) 1733 cursor_granularity: Optional[str] = Field( 1734 None, 1735 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1736 examples=["PT1S"], 1737 title="Cursor Granularity", 1738 ) 1739 is_data_feed: Optional[bool] = Field( 1740 None, 1741 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1742 title="Data Feed API", 1743 ) 1744 is_client_side_incremental: Optional[bool] = Field( 1745 None, 1746 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1747 title="Client-side Incremental Filtering", 1748 ) 1749 is_compare_strictly: Optional[bool] = Field( 1750 False, 1751 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1752 title="Strict Start-End Time Comparison", 1753 ) 1754 global_substream_cursor: Optional[bool] = Field( 1755 False, 1756 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1757 title="Global Substream Cursor", 1758 ) 1759 lookback_window: Optional[str] = Field( 1760 None, 1761 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1762 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1763 title="Lookback Window", 1764 ) 1765 partition_field_end: Optional[str] = Field( 1766 None, 1767 description="Name of the partition start time field.", 1768 examples=["ending_time"], 1769 title="Partition Field End", 1770 ) 1771 partition_field_start: Optional[str] = Field( 1772 None, 1773 description="Name of the partition end time field.", 1774 examples=["starting_time"], 1775 title="Partition Field Start", 1776 ) 1777 step: Optional[str] = Field( 1778 None, 1779 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1780 examples=["P1W", "{{ config['step_increment'] }}"], 1781 title="Step", 1782 ) 1783 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1786class JwtAuthenticator(BaseModel): 1787 type: Literal["JwtAuthenticator"] 1788 secret_key: str = Field( 1789 ..., 1790 description="Secret used to sign the JSON web token.", 1791 examples=["{{ config['secret_key'] }}"], 1792 title="Secret Key", 1793 ) 1794 base64_encode_secret_key: Optional[bool] = Field( 1795 False, 1796 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1797 title="Base64-encode Secret Key", 1798 ) 1799 algorithm: Algorithm = Field( 1800 ..., 1801 description="Algorithm used to sign the JSON web token.", 1802 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1803 title="Algorithm", 1804 ) 1805 token_duration: Optional[int] = Field( 1806 1200, 1807 description="The amount of time in seconds a JWT token can be valid after being issued.", 1808 examples=[1200, 3600], 1809 title="Token Duration", 1810 ) 1811 header_prefix: Optional[str] = Field( 1812 None, 1813 description="The prefix to be used within the Authentication header.", 1814 examples=["Bearer", "Basic"], 1815 title="Header Prefix", 1816 ) 1817 jwt_headers: Optional[JwtHeaders] = Field( 1818 None, 1819 description="JWT headers used when signing JSON web token.", 1820 title="JWT Headers", 1821 ) 1822 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1823 None, 1824 description="Additional headers to be included with the JWT headers object.", 1825 title="Additional JWT Headers", 1826 ) 1827 jwt_payload: Optional[JwtPayload] = Field( 1828 None, 1829 description="JWT Payload used when signing JSON web token.", 1830 title="JWT Payload", 1831 ) 1832 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1833 None, 1834 description="Additional properties to be added to the JWT payload.", 1835 title="Additional JWT Payload Properties", 1836 ) 1837 passphrase: Optional[str] = Field( 1838 None, 1839 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1840 examples=["{{ config['passphrase'] }}"], 1841 title="Passphrase", 1842 ) 1843 request_option: Optional[RequestOption] = Field( 1844 None, 1845 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1846 title="Request Option", 1847 ) 1848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1851class OAuthAuthenticator(BaseModel): 1852 type: Literal["OAuthAuthenticator"] 1853 client_id_name: Optional[str] = Field( 1854 "client_id", 1855 description="The name of the property to use to refresh the `access_token`.", 1856 examples=["custom_app_id"], 1857 title="Client ID Property Name", 1858 ) 1859 client_id: Optional[str] = Field( 1860 None, 1861 description="The OAuth client ID. Fill it in the user inputs.", 1862 examples=[ 1863 "{{ config['client_id'] }}", 1864 "{{ config['credentials']['client_id }}", 1865 ], 1866 title="Client ID", 1867 ) 1868 client_secret_name: Optional[str] = Field( 1869 "client_secret", 1870 description="The name of the property to use to refresh the `access_token`.", 1871 examples=["custom_app_secret"], 1872 title="Client Secret Property Name", 1873 ) 1874 client_secret: Optional[str] = Field( 1875 None, 1876 description="The OAuth client secret. Fill it in the user inputs.", 1877 examples=[ 1878 "{{ config['client_secret'] }}", 1879 "{{ config['credentials']['client_secret }}", 1880 ], 1881 title="Client Secret", 1882 ) 1883 refresh_token_name: Optional[str] = Field( 1884 "refresh_token", 1885 description="The name of the property to use to refresh the `access_token`.", 1886 examples=["custom_app_refresh_value"], 1887 title="Refresh Token Property Name", 1888 ) 1889 refresh_token: Optional[str] = Field( 1890 None, 1891 description="Credential artifact used to get a new access token.", 1892 examples=[ 1893 "{{ config['refresh_token'] }}", 1894 "{{ config['credentials]['refresh_token'] }}", 1895 ], 1896 title="Refresh Token", 1897 ) 1898 token_refresh_endpoint: Optional[str] = Field( 1899 None, 1900 description="The full URL to call to obtain a new access token.", 1901 examples=["https://connect.squareup.com/oauth2/token"], 1902 title="Token Refresh Endpoint", 1903 ) 1904 access_token_name: Optional[str] = Field( 1905 "access_token", 1906 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1907 examples=["access_token"], 1908 title="Access Token Property Name", 1909 ) 1910 access_token_value: Optional[str] = Field( 1911 None, 1912 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1913 examples=["secret_access_token_value"], 1914 title="Access Token Value", 1915 ) 1916 expires_in_name: Optional[str] = Field( 1917 "expires_in", 1918 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1919 examples=["expires_in"], 1920 title="Token Expiry Property Name", 1921 ) 1922 grant_type_name: Optional[str] = Field( 1923 "grant_type", 1924 description="The name of the property to use to refresh the `access_token`.", 1925 examples=["custom_grant_type"], 1926 title="Grant Type Property Name", 1927 ) 1928 grant_type: Optional[str] = Field( 1929 "refresh_token", 1930 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1931 examples=["refresh_token", "client_credentials"], 1932 title="Grant Type", 1933 ) 1934 refresh_request_body: Optional[Dict[str, Any]] = Field( 1935 None, 1936 description="Body of the request sent to get a new access token.", 1937 examples=[ 1938 { 1939 "applicationId": "{{ config['application_id'] }}", 1940 "applicationSecret": "{{ config['application_secret'] }}", 1941 "token": "{{ config['token'] }}", 1942 } 1943 ], 1944 title="Refresh Request Body", 1945 ) 1946 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1947 None, 1948 description="Headers of the request sent to get a new access token.", 1949 examples=[ 1950 { 1951 "Authorization": "<AUTH_TOKEN>", 1952 "Content-Type": "application/x-www-form-urlencoded", 1953 } 1954 ], 1955 title="Refresh Request Headers", 1956 ) 1957 send_refresh_request_as_query_params: Optional[bool] = Field( 1958 False, 1959 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 1960 examples=[True], 1961 title="Send Refresh Request As Query Params", 1962 ) 1963 scopes: Optional[List[str]] = Field( 1964 None, 1965 description="List of scopes that should be granted to the access token.", 1966 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1967 title="Scopes", 1968 ) 1969 token_expiry_date: Optional[str] = Field( 1970 None, 1971 description="The access token expiry date.", 1972 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1973 title="Token Expiry Date", 1974 ) 1975 token_expiry_date_format: Optional[str] = Field( 1976 None, 1977 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1978 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1979 title="Token Expiry Date Format", 1980 ) 1981 refresh_token_error_status_codes: Optional[List[int]] = Field( 1982 None, 1983 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1984 examples=[[400, 500]], 1985 title="Refresh Token Error Status Codes", 1986 ) 1987 refresh_token_error_key: Optional[str] = Field( 1988 None, 1989 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1990 examples=["error"], 1991 title="Refresh Token Error Key", 1992 ) 1993 refresh_token_error_values: Optional[List[str]] = Field( 1994 None, 1995 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1996 examples=[["invalid_grant", "invalid_permissions"]], 1997 title="Refresh Token Error Values", 1998 ) 1999 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2000 None, 2001 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2002 title="Refresh Token Updater", 2003 ) 2004 profile_assertion: Optional[JwtAuthenticator] = Field( 2005 None, 2006 description="The authenticator being used to authenticate the client authenticator.", 2007 title="Profile Assertion", 2008 ) 2009 use_profile_assertion: Optional[bool] = Field( 2010 False, 2011 description="Enable using profile assertion as a flow for OAuth authorization.", 2012 title="Use Profile Assertion", 2013 ) 2014 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2017class FixedWindowCallRatePolicy(BaseModel): 2018 class Config: 2019 extra = Extra.allow 2020 2021 type: Literal["FixedWindowCallRatePolicy"] 2022 period: str = Field( 2023 ..., description="The time interval for the rate limit window.", title="Period" 2024 ) 2025 call_limit: int = Field( 2026 ..., 2027 description="The maximum number of calls allowed within the period.", 2028 title="Call Limit", 2029 ) 2030 matchers: List[HttpRequestRegexMatcher] = Field( 2031 ..., 2032 description="List of matchers that define which requests this policy applies to.", 2033 title="Matchers", 2034 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2037class MovingWindowCallRatePolicy(BaseModel): 2038 class Config: 2039 extra = Extra.allow 2040 2041 type: Literal["MovingWindowCallRatePolicy"] 2042 rates: List[Rate] = Field( 2043 ..., 2044 description="List of rates that define the call limits for different time intervals.", 2045 title="Rates", 2046 ) 2047 matchers: List[HttpRequestRegexMatcher] = Field( 2048 ..., 2049 description="List of matchers that define which requests this policy applies to.", 2050 title="Matchers", 2051 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2054class UnlimitedCallRatePolicy(BaseModel): 2055 class Config: 2056 extra = Extra.allow 2057 2058 type: Literal["UnlimitedCallRatePolicy"] 2059 matchers: List[HttpRequestRegexMatcher] = Field( 2060 ..., 2061 description="List of matchers that define which requests this policy applies to.", 2062 title="Matchers", 2063 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2066class DefaultErrorHandler(BaseModel): 2067 type: Literal["DefaultErrorHandler"] 2068 backoff_strategies: Optional[ 2069 List[ 2070 Union[ 2071 ConstantBackoffStrategy, 2072 ExponentialBackoffStrategy, 2073 WaitTimeFromHeader, 2074 WaitUntilTimeFromHeader, 2075 CustomBackoffStrategy, 2076 ] 2077 ] 2078 ] = Field( 2079 None, 2080 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2081 title="Backoff Strategies", 2082 ) 2083 max_retries: Optional[Union[int, str]] = Field( 2084 5, 2085 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2086 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2087 title="Max Retry Count", 2088 ) 2089 response_filters: Optional[List[HttpResponseFilter]] = Field( 2090 None, 2091 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2092 title="Response Filters", 2093 ) 2094 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2097class DefaultPaginator(BaseModel): 2098 type: Literal["DefaultPaginator"] 2099 pagination_strategy: Union[ 2100 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2101 ] = Field( 2102 ..., 2103 description="Strategy defining how records are paginated.", 2104 title="Pagination Strategy", 2105 ) 2106 page_size_option: Optional[RequestOption] = Field( 2107 None, title="Inject Page Size Into Outgoing HTTP Request" 2108 ) 2109 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2110 None, 2111 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2112 title="Inject Page Token Into Outgoing HTTP Request", 2113 ) 2114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2117class RecordExpander(BaseModel): 2118 type: Literal["RecordExpander"] 2119 expand_records_from_field: List[str] = Field( 2120 ..., 2121 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2122 examples=[ 2123 ["lines", "data"], 2124 ["items"], 2125 ["nested", "array"], 2126 ["sections", "*", "items"], 2127 ], 2128 title="Expand Records From Field", 2129 ) 2130 remain_original_record: Optional[bool] = Field( 2131 False, 2132 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2133 title="Remain Original Record", 2134 ) 2135 on_no_records: Optional[OnNoRecords] = Field( 2136 OnNoRecords.skip, 2137 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2138 title="On No Records", 2139 ) 2140 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2143class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2144 type: Literal["ApiKey"] 2145 inject_into: RequestOption = Field( 2146 ..., 2147 description="Configure how the API Key will be sent in requests to the source API.", 2148 examples=[ 2149 {"inject_into": "header", "field_name": "Authorization"}, 2150 {"inject_into": "request_parameter", "field_name": "authKey"}, 2151 ], 2152 title="Inject API Key Into Outgoing HTTP Request", 2153 ) 2154 api_token: Optional[str] = Field( 2155 "{{ session_token }}", 2156 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2157 examples=[ 2158 "{{ session_token }}", 2159 "Token {{ session_token }}", 2160 "Bearer {{ session_token }}", 2161 ], 2162 title="API Token Template", 2163 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2166class JsonSchemaPropertySelector(BaseModel): 2167 type: Literal["JsonSchemaPropertySelector"] 2168 transformations: Optional[ 2169 List[ 2170 Union[ 2171 AddFields, 2172 RemoveFields, 2173 KeysToLower, 2174 KeysToSnakeCase, 2175 FlattenFields, 2176 DpathFlattenFields, 2177 KeysReplace, 2178 CustomTransformation, 2179 ] 2180 ] 2181 ] = Field( 2182 None, 2183 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2184 title="Transformations", 2185 ) 2186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2189class ListPartitionRouter(BaseModel): 2190 type: Literal["ListPartitionRouter"] 2191 cursor_field: str = Field( 2192 ..., 2193 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2194 examples=["section", "{{ config['section_key'] }}"], 2195 title="Current Partition Value Identifier", 2196 ) 2197 values: Union[str, List[str]] = Field( 2198 ..., 2199 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2200 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2201 title="Partition Values", 2202 ) 2203 request_option: Optional[RequestOption] = Field( 2204 None, 2205 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2206 title="Inject Partition Value Into Outgoing HTTP Request", 2207 ) 2208 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2211class PaginationReset(BaseModel): 2212 type: Literal["PaginationReset"] 2213 action: Action1 2214 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2217class GzipDecoder(BaseModel): 2218 type: Literal["GzipDecoder"] 2219 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2222class RequestBodyGraphQL(BaseModel): 2223 type: Literal["RequestBodyGraphQL"] 2224 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2227class DpathValidator(BaseModel): 2228 type: Literal["DpathValidator"] 2229 field_path: List[str] = Field( 2230 ..., 2231 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2232 examples=[ 2233 ["data"], 2234 ["data", "records"], 2235 ["data", "{{ parameters.name }}"], 2236 ["data", "*", "record"], 2237 ], 2238 title="Field Path", 2239 ) 2240 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2241 ..., 2242 description="The condition that the specified config value will be evaluated against", 2243 title="Validation Strategy", 2244 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2247class PredicateValidator(BaseModel): 2248 type: Literal["PredicateValidator"] 2249 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2250 ..., 2251 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2252 examples=[ 2253 "test-value", 2254 "{{ config['api_version'] }}", 2255 "{{ config['tenant_id'] }}", 2256 123, 2257 ], 2258 title="Value", 2259 ) 2260 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2261 ..., 2262 description="The validation strategy to apply to the value.", 2263 title="Validation Strategy", 2264 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2267class ConfigAddFields(BaseModel): 2268 type: Literal["ConfigAddFields"] 2269 fields: List[AddedFieldDefinition] = Field( 2270 ..., 2271 description="A list of transformations (path and corresponding value) that will be added to the config.", 2272 title="Fields", 2273 ) 2274 condition: Optional[str] = Field( 2275 "", 2276 description="Fields will be added if expression is evaluated to True.", 2277 examples=[ 2278 "{{ config['environemnt'] == 'sandbox' }}", 2279 "{{ property is integer }}", 2280 "{{ property|length > 5 }}", 2281 "{{ property == 'some_string_to_match' }}", 2282 ], 2283 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2286class CompositeErrorHandler(BaseModel): 2287 type: Literal["CompositeErrorHandler"] 2288 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2289 Field( 2290 ..., 2291 description="List of error handlers to iterate on to determine how to handle a failed response.", 2292 title="Error Handlers", 2293 ) 2294 ) 2295 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2298class HTTPAPIBudget(BaseModel): 2299 class Config: 2300 extra = Extra.allow 2301 2302 type: Literal["HTTPAPIBudget"] 2303 policies: List[ 2304 Union[ 2305 FixedWindowCallRatePolicy, 2306 MovingWindowCallRatePolicy, 2307 UnlimitedCallRatePolicy, 2308 ] 2309 ] = Field( 2310 ..., 2311 description="List of call rate policies that define how many calls are allowed.", 2312 title="Policies", 2313 ) 2314 ratelimit_reset_header: Optional[str] = Field( 2315 "ratelimit-reset", 2316 description="The HTTP response header name that indicates when the rate limit resets.", 2317 title="Rate Limit Reset Header", 2318 ) 2319 ratelimit_remaining_header: Optional[str] = Field( 2320 "ratelimit-remaining", 2321 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2322 title="Rate Limit Remaining Header", 2323 ) 2324 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2325 [429], 2326 description="List of HTTP status codes that indicate a rate limit has been hit.", 2327 title="Status Codes for Rate Limit Hit", 2328 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2331class DpathExtractor(BaseModel): 2332 type: Literal["DpathExtractor"] 2333 field_path: List[str] = Field( 2334 ..., 2335 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2336 examples=[ 2337 ["data"], 2338 ["data", "records"], 2339 ["data", "{{ parameters.name }}"], 2340 ["data", "*", "record"], 2341 ], 2342 title="Field Path", 2343 ) 2344 record_expander: Optional[RecordExpander] = Field( 2345 None, 2346 description="Optional component to expand records by extracting items from nested array fields.", 2347 title="Record Expander", 2348 ) 2349 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2352class ZipfileDecoder(BaseModel): 2353 class Config: 2354 extra = Extra.allow 2355 2356 type: Literal["ZipfileDecoder"] 2357 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2358 ..., 2359 description="Parser to parse the decompressed data from the zipfile(s).", 2360 title="Parser", 2361 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2364class RecordSelector(BaseModel): 2365 type: Literal["RecordSelector"] 2366 extractor: Union[DpathExtractor, CustomRecordExtractor] 2367 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2368 None, 2369 description="Responsible for filtering records to be emitted by the Source.", 2370 title="Record Filter", 2371 ) 2372 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2373 None, 2374 description="Responsible for normalization according to the schema.", 2375 title="Schema Normalization", 2376 ) 2377 transform_before_filtering: Optional[bool] = Field( 2378 None, 2379 description="If true, transformation will be applied before record filtering.", 2380 title="Transform Before Filtering", 2381 ) 2382 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2385class ConfigMigration(BaseModel): 2386 type: Literal["ConfigMigration"] 2387 description: Optional[str] = Field( 2388 None, description="The description/purpose of the config migration." 2389 ) 2390 transformations: List[ 2391 Union[ 2392 ConfigRemapField, 2393 ConfigAddFields, 2394 ConfigRemoveFields, 2395 CustomConfigTransformation, 2396 ] 2397 ] = Field( 2398 ..., 2399 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2400 title="Transformations", 2401 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2404class ConfigNormalizationRules(BaseModel): 2405 class Config: 2406 extra = Extra.forbid 2407 2408 type: Literal["ConfigNormalizationRules"] 2409 config_migrations: Optional[List[ConfigMigration]] = Field( 2410 [], 2411 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2412 title="Config Migrations", 2413 ) 2414 transformations: Optional[ 2415 List[ 2416 Union[ 2417 ConfigRemapField, 2418 ConfigAddFields, 2419 ConfigRemoveFields, 2420 CustomConfigTransformation, 2421 ] 2422 ] 2423 ] = Field( 2424 [], 2425 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2426 title="Transformations", 2427 ) 2428 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2429 [], 2430 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2431 title="Validations", 2432 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2435class Spec(BaseModel): 2436 type: Literal["Spec"] 2437 connection_specification: Dict[str, Any] = Field( 2438 ..., 2439 description="A connection specification describing how a the connector can be configured.", 2440 title="Connection Specification", 2441 ) 2442 documentation_url: Optional[str] = Field( 2443 None, 2444 description="URL of the connector's documentation page.", 2445 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2446 title="Documentation URL", 2447 ) 2448 advanced_auth: Optional[AuthFlow] = Field( 2449 None, 2450 description="Advanced specification for configuring the authentication flow.", 2451 title="Advanced Auth", 2452 ) 2453 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2454 None, title="Config Normalization Rules" 2455 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2458class DeclarativeSource1(BaseModel): 2459 class Config: 2460 extra = Extra.forbid 2461 2462 type: Literal["DeclarativeSource"] 2463 check: Union[CheckStream, CheckDynamicStream] 2464 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2465 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2466 version: str = Field( 2467 ..., 2468 description="The version of the Airbyte CDK used to build and test the source.", 2469 ) 2470 schemas: Optional[Schemas] = None 2471 definitions: Optional[Dict[str, Any]] = None 2472 spec: Optional[Spec] = None 2473 concurrency_level: Optional[ConcurrencyLevel] = None 2474 api_budget: Optional[HTTPAPIBudget] = None 2475 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2476 None, 2477 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2478 title="Stream Groups", 2479 ) 2480 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2481 None, 2482 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2483 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2484 title="Maximum Concurrent Asynchronous Jobs", 2485 ) 2486 metadata: Optional[Dict[str, Any]] = Field( 2487 None, 2488 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2489 ) 2490 description: Optional[str] = Field( 2491 None, 2492 description="A description of the connector. It will be presented on the Source documentation page.", 2493 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2496class DeclarativeSource2(BaseModel): 2497 class Config: 2498 extra = Extra.forbid 2499 2500 type: Literal["DeclarativeSource"] 2501 check: Union[CheckStream, CheckDynamicStream] 2502 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2503 None 2504 ) 2505 dynamic_streams: List[DynamicDeclarativeStream] 2506 version: str = Field( 2507 ..., 2508 description="The version of the Airbyte CDK used to build and test the source.", 2509 ) 2510 schemas: Optional[Schemas] = None 2511 definitions: Optional[Dict[str, Any]] = None 2512 spec: Optional[Spec] = None 2513 concurrency_level: Optional[ConcurrencyLevel] = None 2514 api_budget: Optional[HTTPAPIBudget] = None 2515 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2516 None, 2517 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2518 title="Stream Groups", 2519 ) 2520 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2521 None, 2522 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2523 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2524 title="Maximum Concurrent Asynchronous Jobs", 2525 ) 2526 metadata: Optional[Dict[str, Any]] = Field( 2527 None, 2528 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2529 ) 2530 description: Optional[str] = Field( 2531 None, 2532 description="A description of the connector. It will be presented on the Source documentation page.", 2533 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2536class DeclarativeSource(BaseModel): 2537 class Config: 2538 extra = Extra.forbid 2539 2540 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2541 ..., 2542 description="An API source that extracts data according to its declarative components.", 2543 title="DeclarativeSource", 2544 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2547class SelectiveAuthenticator(BaseModel): 2548 class Config: 2549 extra = Extra.allow 2550 2551 type: Literal["SelectiveAuthenticator"] 2552 authenticator_selection_path: List[str] = Field( 2553 ..., 2554 description="Path of the field in config with selected authenticator name", 2555 examples=[["auth"], ["auth", "type"]], 2556 title="Authenticator Selection Path", 2557 ) 2558 authenticators: Dict[ 2559 str, 2560 Union[ 2561 ApiKeyAuthenticator, 2562 BasicHttpAuthenticator, 2563 BearerAuthenticator, 2564 OAuthAuthenticator, 2565 JwtAuthenticator, 2566 SessionTokenAuthenticator, 2567 LegacySessionTokenAuthenticator, 2568 CustomAuthenticator, 2569 NoAuth, 2570 ], 2571 ] = Field( 2572 ..., 2573 description="Authenticators to select from.", 2574 examples=[ 2575 { 2576 "authenticators": { 2577 "token": "#/definitions/ApiKeyAuthenticator", 2578 "oauth": "#/definitions/OAuthAuthenticator", 2579 "jwt": "#/definitions/JwtAuthenticator", 2580 } 2581 } 2582 ], 2583 title="Authenticators", 2584 ) 2585 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2588class ConditionalStreams(BaseModel): 2589 type: Literal["ConditionalStreams"] 2590 condition: str = Field( 2591 ..., 2592 description="Condition that will be evaluated to determine if a set of streams should be available.", 2593 examples=["{{ config['is_sandbox'] }}"], 2594 title="Condition", 2595 ) 2596 streams: List[DeclarativeStream] = Field( 2597 ..., 2598 description="Streams that will be used during an operation based on the condition.", 2599 title="Streams", 2600 ) 2601 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2604class FileUploader(BaseModel): 2605 type: Literal["FileUploader"] 2606 requester: Union[HttpRequester, CustomRequester] = Field( 2607 ..., 2608 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2609 ) 2610 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2611 ..., 2612 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2613 ) 2614 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2615 None, 2616 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2617 ) 2618 filename_extractor: Optional[str] = Field( 2619 None, 2620 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2621 examples=[ 2622 "{{ record.id }}/{{ record.file_name }}/", 2623 "{{ record.id }}_{{ record.file_name }}/", 2624 ], 2625 ) 2626 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2629class DeclarativeStream(BaseModel): 2630 class Config: 2631 extra = Extra.allow 2632 2633 type: Literal["DeclarativeStream"] 2634 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2635 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2636 ..., 2637 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2638 title="Retriever", 2639 ) 2640 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2641 None, 2642 description="Component used to fetch data incrementally based on a time field in the data.", 2643 title="Incremental Sync", 2644 ) 2645 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2646 schema_loader: Optional[ 2647 Union[ 2648 InlineSchemaLoader, 2649 DynamicSchemaLoader, 2650 JsonFileSchemaLoader, 2651 List[ 2652 Union[ 2653 InlineSchemaLoader, 2654 DynamicSchemaLoader, 2655 JsonFileSchemaLoader, 2656 CustomSchemaLoader, 2657 ] 2658 ], 2659 CustomSchemaLoader, 2660 ] 2661 ] = Field( 2662 None, 2663 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2664 title="Schema Loader", 2665 ) 2666 transformations: Optional[ 2667 List[ 2668 Union[ 2669 AddFields, 2670 RemoveFields, 2671 KeysToLower, 2672 KeysToSnakeCase, 2673 FlattenFields, 2674 DpathFlattenFields, 2675 KeysReplace, 2676 CustomTransformation, 2677 ] 2678 ] 2679 ] = Field( 2680 None, 2681 description="A list of transformations to be applied to each output record.", 2682 title="Transformations", 2683 ) 2684 state_migrations: Optional[ 2685 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2686 ] = Field( 2687 [], 2688 description="Array of state migrations to be applied on the input state", 2689 title="State Migrations", 2690 ) 2691 file_uploader: Optional[FileUploader] = Field( 2692 None, 2693 description="(experimental) Describes how to fetch a file", 2694 title="File Uploader", 2695 ) 2696 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2699class SessionTokenAuthenticator(BaseModel): 2700 type: Literal["SessionTokenAuthenticator"] 2701 login_requester: HttpRequester = Field( 2702 ..., 2703 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2704 examples=[ 2705 { 2706 "type": "HttpRequester", 2707 "url_base": "https://my_api.com", 2708 "path": "/login", 2709 "authenticator": { 2710 "type": "BasicHttpAuthenticator", 2711 "username": "{{ config.username }}", 2712 "password": "{{ config.password }}", 2713 }, 2714 } 2715 ], 2716 title="Login Requester", 2717 ) 2718 session_token_path: List[str] = Field( 2719 ..., 2720 description="The path in the response body returned from the login requester to the session token.", 2721 examples=[["access_token"], ["result", "token"]], 2722 title="Session Token Path", 2723 ) 2724 expiration_duration: Optional[str] = Field( 2725 None, 2726 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2727 examples=["PT1H", "P1D"], 2728 title="Expiration Duration", 2729 ) 2730 request_authentication: Union[ 2731 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2732 ] = Field( 2733 ..., 2734 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2735 title="Data Request Authentication", 2736 ) 2737 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2738 None, description="Component used to decode the response.", title="Decoder" 2739 ) 2740 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2743class HttpRequester(BaseModelWithDeprecations): 2744 type: Literal["HttpRequester"] 2745 url_base: Optional[str] = Field( 2746 None, 2747 deprecated=True, 2748 deprecation_message="Use `url` field instead.", 2749 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2750 examples=[ 2751 "https://connect.squareup.com/v2", 2752 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2753 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2754 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2755 ], 2756 title="API Base URL", 2757 ) 2758 url: Optional[str] = Field( 2759 None, 2760 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2761 examples=[ 2762 "https://connect.squareup.com/v2", 2763 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2764 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2765 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2766 ], 2767 title="API Endpoint URL", 2768 ) 2769 path: Optional[str] = Field( 2770 None, 2771 deprecated=True, 2772 deprecation_message="Use `url` field instead.", 2773 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2774 examples=[ 2775 "/products", 2776 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2777 "/trades/{{ config['symbol_id'] }}/history", 2778 ], 2779 title="URL Path", 2780 ) 2781 http_method: Optional[HttpMethod] = Field( 2782 HttpMethod.GET, 2783 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2784 examples=["GET", "POST"], 2785 title="HTTP Method", 2786 ) 2787 authenticator: Optional[ 2788 Union[ 2789 ApiKeyAuthenticator, 2790 BasicHttpAuthenticator, 2791 BearerAuthenticator, 2792 OAuthAuthenticator, 2793 JwtAuthenticator, 2794 SessionTokenAuthenticator, 2795 SelectiveAuthenticator, 2796 CustomAuthenticator, 2797 NoAuth, 2798 LegacySessionTokenAuthenticator, 2799 ] 2800 ] = Field( 2801 None, 2802 description="Authentication method to use for requests sent to the API.", 2803 title="Authenticator", 2804 ) 2805 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2806 None, 2807 deprecated=True, 2808 deprecation_message="Use `query_properties` field instead.", 2809 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2810 title="Fetch Properties from Endpoint", 2811 ) 2812 query_properties: Optional[QueryProperties] = Field( 2813 None, 2814 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2815 title="Query Properties", 2816 ) 2817 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2818 None, 2819 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2820 examples=[ 2821 {"unit": "day"}, 2822 { 2823 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2824 }, 2825 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2826 {"sort_by[asc]": "updated_at"}, 2827 ], 2828 title="Query Parameters", 2829 ) 2830 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2831 None, 2832 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2833 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2834 title="Request Headers", 2835 ) 2836 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2837 None, 2838 deprecated=True, 2839 deprecation_message="Use `request_body` field instead.", 2840 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2841 examples=[ 2842 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2843 ], 2844 title="Request Body Payload (Non-JSON)", 2845 ) 2846 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2847 None, 2848 deprecated=True, 2849 deprecation_message="Use `request_body` field instead.", 2850 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2851 examples=[ 2852 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2853 {"key": "{{ config['value'] }}"}, 2854 {"sort": {"field": "updated_at", "order": "ascending"}}, 2855 ], 2856 title="Request Body JSON Payload", 2857 ) 2858 request_body: Optional[ 2859 Union[ 2860 RequestBodyPlainText, 2861 RequestBodyUrlEncodedForm, 2862 RequestBodyJsonObject, 2863 RequestBodyGraphQL, 2864 ] 2865 ] = Field( 2866 None, 2867 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2868 title="Request Body", 2869 ) 2870 error_handler: Optional[ 2871 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2872 ] = Field( 2873 None, 2874 description="Error handler component that defines how to handle errors.", 2875 title="Error Handler", 2876 ) 2877 use_cache: Optional[bool] = Field( 2878 False, 2879 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2880 title="Use Cache", 2881 ) 2882 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2885class DynamicSchemaLoader(BaseModel): 2886 type: Literal["DynamicSchemaLoader"] 2887 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2888 ..., 2889 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2890 title="Retriever", 2891 ) 2892 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2893 None, 2894 description="Responsible for filtering fields to be added to json schema.", 2895 title="Schema Filter", 2896 ) 2897 schema_transformations: Optional[ 2898 List[ 2899 Union[ 2900 AddFields, 2901 RemoveFields, 2902 KeysToLower, 2903 KeysToSnakeCase, 2904 FlattenFields, 2905 DpathFlattenFields, 2906 KeysReplace, 2907 CustomTransformation, 2908 ] 2909 ] 2910 ] = Field( 2911 None, 2912 description="A list of transformations to be applied to the schema.", 2913 title="Schema Transformations", 2914 ) 2915 schema_type_identifier: SchemaTypeIdentifier 2916 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2919class ParentStreamConfig(BaseModel): 2920 type: Literal["ParentStreamConfig"] 2921 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2922 ..., description="Reference to the parent stream.", title="Parent Stream" 2923 ) 2924 parent_key: str = Field( 2925 ..., 2926 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2927 examples=["id", "{{ config['parent_record_id'] }}"], 2928 title="Parent Key", 2929 ) 2930 partition_field: str = Field( 2931 ..., 2932 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2933 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2934 title="Current Parent Key Value Identifier", 2935 ) 2936 request_option: Optional[RequestOption] = Field( 2937 None, 2938 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2939 title="Request Option", 2940 ) 2941 incremental_dependency: Optional[bool] = Field( 2942 False, 2943 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2944 title="Incremental Dependency", 2945 ) 2946 lazy_read_pointer: Optional[List[str]] = Field( 2947 [], 2948 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2949 title="Lazy Read Pointer", 2950 ) 2951 extra_fields: Optional[List[List[str]]] = Field( 2952 None, 2953 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2954 title="Extra Fields", 2955 ) 2956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2959class PropertiesFromEndpoint(BaseModel): 2960 type: Literal["PropertiesFromEndpoint"] 2961 property_field_path: List[str] = Field( 2962 ..., 2963 description="Describes the path to the field that should be extracted", 2964 examples=[["name"]], 2965 ) 2966 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2967 ..., 2968 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2969 ) 2970 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2973class QueryProperties(BaseModel): 2974 type: Literal["QueryProperties"] 2975 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2976 ..., 2977 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2978 title="Property List", 2979 ) 2980 always_include_properties: Optional[List[str]] = Field( 2981 None, 2982 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2983 title="Always Include Properties", 2984 ) 2985 property_chunking: Optional[PropertyChunking] = Field( 2986 None, 2987 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2988 title="Property Chunking", 2989 ) 2990 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2991 None, 2992 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2993 title="Property Selector", 2994 ) 2995 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2998class StateDelegatingStream(BaseModel): 2999 type: Literal["StateDelegatingStream"] 3000 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3001 full_refresh_stream: DeclarativeStream = Field( 3002 ..., 3003 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3004 title="Full Refresh Stream", 3005 ) 3006 incremental_stream: DeclarativeStream = Field( 3007 ..., 3008 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3009 title="Incremental Stream", 3010 ) 3011 api_retention_period: Optional[str] = Field( 3012 None, 3013 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3014 examples=["P30D", "P90D", "P1Y"], 3015 title="API Retention Period", 3016 ) 3017 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3020class SimpleRetriever(BaseModel): 3021 type: Literal["SimpleRetriever"] 3022 requester: Union[HttpRequester, CustomRequester] = Field( 3023 ..., 3024 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3025 ) 3026 decoder: Optional[ 3027 Union[ 3028 JsonDecoder, 3029 JsonItemsDecoder, 3030 XmlDecoder, 3031 CsvDecoder, 3032 JsonlDecoder, 3033 GzipDecoder, 3034 IterableDecoder, 3035 ZipfileDecoder, 3036 CustomDecoder, 3037 ] 3038 ] = Field( 3039 None, 3040 description="Component decoding the response so records can be extracted.", 3041 title="HTTP Response Format", 3042 ) 3043 record_selector: RecordSelector = Field( 3044 ..., 3045 description="Component that describes how to extract records from a HTTP response.", 3046 ) 3047 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3048 None, 3049 description="Paginator component that describes how to navigate through the API's pages.", 3050 ) 3051 pagination_reset: Optional[PaginationReset] = Field( 3052 None, 3053 description="Describes what triggers pagination reset and how to handle it.", 3054 ) 3055 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3056 False, 3057 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3058 ) 3059 partition_router: Optional[ 3060 Union[ 3061 SubstreamPartitionRouter, 3062 ListPartitionRouter, 3063 GroupingPartitionRouter, 3064 CustomPartitionRouter, 3065 List[ 3066 Union[ 3067 SubstreamPartitionRouter, 3068 ListPartitionRouter, 3069 GroupingPartitionRouter, 3070 CustomPartitionRouter, 3071 ] 3072 ], 3073 ] 3074 ] = Field( 3075 None, 3076 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3077 title="Partition Router", 3078 ) 3079 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3082class AsyncRetriever(BaseModel): 3083 type: Literal["AsyncRetriever"] 3084 record_selector: RecordSelector = Field( 3085 ..., 3086 description="Component that describes how to extract records from a HTTP response.", 3087 ) 3088 status_mapping: AsyncJobStatusMap = Field( 3089 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3090 ) 3091 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3092 ..., description="Responsible for fetching the actual status of the async job." 3093 ) 3094 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3095 None, 3096 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3097 ) 3098 download_extractor: Optional[ 3099 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3100 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3101 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3102 ..., 3103 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3104 ) 3105 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3106 ..., 3107 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3108 ) 3109 polling_job_timeout: Optional[Union[int, str]] = Field( 3110 None, 3111 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3112 ) 3113 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3114 None, 3115 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3116 ge=1, 3117 ) 3118 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3119 None, 3120 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3121 ) 3122 download_requester: Union[HttpRequester, CustomRequester] = Field( 3123 ..., 3124 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3125 ) 3126 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3127 None, 3128 description="Paginator component that describes how to navigate through the API's pages during download.", 3129 ) 3130 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3131 None, 3132 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3133 ) 3134 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3135 None, 3136 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3137 ) 3138 partition_router: Optional[ 3139 Union[ 3140 ListPartitionRouter, 3141 SubstreamPartitionRouter, 3142 GroupingPartitionRouter, 3143 CustomPartitionRouter, 3144 List[ 3145 Union[ 3146 ListPartitionRouter, 3147 SubstreamPartitionRouter, 3148 GroupingPartitionRouter, 3149 CustomPartitionRouter, 3150 ] 3151 ], 3152 ] 3153 ] = Field( 3154 [], 3155 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3156 title="Partition Router", 3157 ) 3158 decoder: Optional[ 3159 Union[ 3160 CsvDecoder, 3161 GzipDecoder, 3162 JsonDecoder, 3163 JsonItemsDecoder, 3164 JsonlDecoder, 3165 IterableDecoder, 3166 XmlDecoder, 3167 ZipfileDecoder, 3168 CustomDecoder, 3169 ] 3170 ] = Field( 3171 None, 3172 description="Component decoding the response so records can be extracted.", 3173 title="HTTP Response Format", 3174 ) 3175 download_decoder: Optional[ 3176 Union[ 3177 CsvDecoder, 3178 GzipDecoder, 3179 JsonDecoder, 3180 JsonItemsDecoder, 3181 JsonlDecoder, 3182 IterableDecoder, 3183 XmlDecoder, 3184 ZipfileDecoder, 3185 CustomDecoder, 3186 ] 3187 ] = Field( 3188 None, 3189 description="Component decoding the download response so records can be extracted.", 3190 title="Download HTTP Response Format", 3191 ) 3192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3199class StreamGroup(BaseModel): 3200 streams: List[str] = Field( 3201 ..., 3202 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3203 title="Streams", 3204 ) 3205 action: BlockSimultaneousSyncsAction = Field( 3206 ..., 3207 description="The action to apply to streams in this group.", 3208 title="Action", 3209 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3212class SubstreamPartitionRouter(BaseModel): 3213 type: Literal["SubstreamPartitionRouter"] 3214 parent_stream_configs: List[ParentStreamConfig] = Field( 3215 ..., 3216 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3217 title="Parent Stream Configs", 3218 ) 3219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3222class GroupingPartitionRouter(BaseModel): 3223 type: Literal["GroupingPartitionRouter"] 3224 group_size: int = Field( 3225 ..., 3226 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3227 examples=[10, 50], 3228 title="Group Size", 3229 ) 3230 underlying_partition_router: Union[ 3231 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3232 ] = Field( 3233 ..., 3234 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3235 title="Underlying Partition Router", 3236 ) 3237 deduplicate: Optional[bool] = Field( 3238 True, 3239 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3240 title="Deduplicate Partitions", 3241 ) 3242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3245class HttpComponentsResolver(BaseModel): 3246 type: Literal["HttpComponentsResolver"] 3247 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3248 ..., 3249 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3250 title="Retriever", 3251 ) 3252 components_mapping: List[ComponentMappingDefinition] 3253 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3256class DynamicDeclarativeStream(BaseModel): 3257 type: Literal["DynamicDeclarativeStream"] 3258 name: Optional[str] = Field( 3259 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3260 ) 3261 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3262 ..., description="Reference to the stream template.", title="Stream Template" 3263 ) 3264 components_resolver: Union[ 3265 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3266 ] = Field( 3267 ..., 3268 description="Component resolve and populates stream templates with components values.", 3269 title="Components Resolver", 3270 ) 3271 use_parent_parameters: Optional[bool] = Field( 3272 True, 3273 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3274 title="Use Parent Parameters", 3275 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]