airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 jitter_range_in_seconds: Optional[float] = Field( 108 None, 109 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 110 examples=[15], 111 ge=0, 112 title="Jitter Range", 113 ) 114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 115 116 117class CursorPagination(BaseModel): 118 type: Literal["CursorPagination"] 119 cursor_value: str = Field( 120 ..., 121 description="Value of the cursor defining the next page to fetch.", 122 examples=[ 123 "{{ headers.link.next.cursor }}", 124 "{{ last_record['key'] }}", 125 "{{ response['nextPage'] }}", 126 ], 127 title="Cursor Value", 128 ) 129 page_size: Optional[Union[int, str]] = Field( 130 None, 131 description="The number of records to include in each pages.", 132 examples=[100, "{{ config['page_size'] }}"], 133 title="Page Size", 134 ) 135 stop_condition: Optional[str] = Field( 136 None, 137 description="Template string evaluating when to stop paginating.", 138 examples=[ 139 "{{ response.data.has_more is false }}", 140 "{{ 'next' not in headers['link'] }}", 141 ], 142 title="Stop Condition", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomAuthenticator(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomAuthenticator"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomBackoffStrategy(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomBackoffStrategy"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomBackoffStrategy"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomErrorHandler(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomErrorHandler"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomErrorHandler"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomPaginationStrategy(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomPaginationStrategy"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomPaginationStrategy"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordExtractor(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordExtractor"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomRecordExtractor"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRecordFilter(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRecordFilter"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomCustomRecordFilter"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRequester(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRequester"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRecordExtractor"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomRetriever(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomRetriever"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomRetriever"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomPartitionRouter(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomPartitionRouter"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomPartitionRouter"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaLoader(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaLoader"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 281 examples=["source_railz.components.MyCustomSchemaLoader"], 282 title="Class Name", 283 ) 284 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 285 286 287class CustomSchemaNormalization(BaseModel): 288 class Config: 289 extra = Extra.allow 290 291 type: Literal["CustomSchemaNormalization"] 292 class_name: str = Field( 293 ..., 294 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 295 examples=[ 296 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 297 ], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomStateMigration(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomStateMigration"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomStateMigration"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class CustomTransformation(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Literal["CustomTransformation"] 322 class_name: str = Field( 323 ..., 324 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 325 examples=["source_railz.components.MyCustomTransformation"], 326 title="Class Name", 327 ) 328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 329 330 331class LegacyToPerPartitionStateMigration(BaseModel): 332 class Config: 333 extra = Extra.allow 334 335 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 336 337 338class Clamping(BaseModel): 339 target: str = Field( 340 ..., 341 description="The period of time that datetime windows will be clamped by", 342 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 343 title="Target", 344 ) 345 target_details: Optional[Dict[str, Any]] = None 346 347 348class Algorithm(Enum): 349 HS256 = "HS256" 350 HS384 = "HS384" 351 HS512 = "HS512" 352 ES256 = "ES256" 353 ES256K = "ES256K" 354 ES384 = "ES384" 355 ES512 = "ES512" 356 RS256 = "RS256" 357 RS384 = "RS384" 358 RS512 = "RS512" 359 PS256 = "PS256" 360 PS384 = "PS384" 361 PS512 = "PS512" 362 EdDSA = "EdDSA" 363 364 365class JwtHeaders(BaseModel): 366 class Config: 367 extra = Extra.forbid 368 369 kid: Optional[str] = Field( 370 None, 371 description="Private key ID for user account.", 372 examples=["{{ config['kid'] }}"], 373 title="Key Identifier", 374 ) 375 typ: Optional[str] = Field( 376 "JWT", 377 description="The media type of the complete JWT.", 378 examples=["JWT"], 379 title="Type", 380 ) 381 cty: Optional[str] = Field( 382 None, 383 description="Content type of JWT header.", 384 examples=["JWT"], 385 title="Content Type", 386 ) 387 388 389class JwtPayload(BaseModel): 390 class Config: 391 extra = Extra.forbid 392 393 iss: Optional[str] = Field( 394 None, 395 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 396 examples=["{{ config['iss'] }}"], 397 title="Issuer", 398 ) 399 sub: Optional[str] = Field( 400 None, 401 description="The subject of the JWT. Commonly defined by the API.", 402 title="Subject", 403 ) 404 aud: Optional[str] = Field( 405 None, 406 description="The recipient that the JWT is intended for. Commonly defined by the API.", 407 examples=["appstoreconnect-v1"], 408 title="Audience", 409 ) 410 411 412class RefreshTokenUpdater(BaseModel): 413 refresh_token_name: Optional[str] = Field( 414 "refresh_token", 415 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 416 examples=["refresh_token"], 417 title="Refresh Token Property Name", 418 ) 419 access_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "access_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "access_token"], ["access_token"]], 423 title="Config Path To Access Token", 424 ) 425 refresh_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "refresh_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "refresh_token"], ["refresh_token"]], 429 title="Config Path To Refresh Token", 430 ) 431 token_expiry_date_config_path: Optional[List[str]] = Field( 432 ["credentials", "token_expiry_date"], 433 description="Config path to the expiry date. Make sure actually exists in the config.", 434 examples=[["credentials", "token_expiry_date"]], 435 title="Config Path To Expiry Date", 436 ) 437 refresh_token_error_status_codes: Optional[List[int]] = Field( 438 [], 439 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 440 examples=[[400, 500]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 442 ) 443 refresh_token_error_key: Optional[str] = Field( 444 "", 445 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 446 examples=["error"], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 448 ) 449 refresh_token_error_values: Optional[List[str]] = Field( 450 [], 451 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 452 examples=[["invalid_grant", "invalid_permissions"]], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 454 ) 455 456 457class Rate(BaseModel): 458 class Config: 459 extra = Extra.allow 460 461 limit: Union[int, str] = Field( 462 ..., 463 description="The maximum number of calls allowed within the interval.", 464 title="Limit", 465 ) 466 interval: str = Field( 467 ..., 468 description="The time interval for the rate limit.", 469 examples=["PT1H", "P1D"], 470 title="Interval", 471 ) 472 473 474class HttpRequestRegexMatcher(BaseModel): 475 class Config: 476 extra = Extra.allow 477 478 method: Optional[str] = Field( 479 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 480 ) 481 url_base: Optional[str] = Field( 482 None, 483 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 484 title="URL Base", 485 ) 486 url_path_pattern: Optional[str] = Field( 487 None, 488 description="A regular expression pattern to match the URL path.", 489 title="URL Path Pattern", 490 ) 491 params: Optional[Dict[str, Any]] = Field( 492 None, description="The query parameters to match.", title="Parameters" 493 ) 494 headers: Optional[Dict[str, Any]] = Field( 495 None, description="The headers to match.", title="Headers" 496 ) 497 weight: Optional[Union[int, str]] = Field( 498 None, 499 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 500 title="Weight", 501 ) 502 503 504class ResponseToFileExtractor(BaseModel): 505 type: Literal["ResponseToFileExtractor"] 506 preserve_na_values: Optional[bool] = Field( 507 False, 508 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 509 title="Preserve NA Values", 510 ) 511 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 512 513 514class OnNoRecords(Enum): 515 skip = "skip" 516 emit_parent = "emit_parent" 517 518 519class ExponentialBackoffStrategy(BaseModel): 520 type: Literal["ExponentialBackoffStrategy"] 521 factor: Optional[Union[float, str]] = Field( 522 5, 523 description="Multiplicative constant applied on each retry.", 524 examples=[5, 5.5, "10"], 525 title="Factor", 526 ) 527 jitter_range_in_seconds: Optional[float] = Field( 528 None, 529 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 530 examples=[2], 531 ge=0, 532 title="Jitter Range", 533 ) 534 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 535 536 537class GroupByKeyMergeStrategy(BaseModel): 538 type: Literal["GroupByKeyMergeStrategy"] 539 key: Union[str, List[str]] = Field( 540 ..., 541 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 542 examples=["id", ["parent_id", "end_date"]], 543 title="Key", 544 ) 545 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 546 547 548class SessionTokenRequestBearerAuthenticator(BaseModel): 549 type: Literal["Bearer"] 550 551 552class HttpMethod(Enum): 553 GET = "GET" 554 POST = "POST" 555 556 557class Action(Enum): 558 SUCCESS = "SUCCESS" 559 FAIL = "FAIL" 560 RETRY = "RETRY" 561 IGNORE = "IGNORE" 562 RESET_PAGINATION = "RESET_PAGINATION" 563 RATE_LIMITED = "RATE_LIMITED" 564 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 565 566 567class FailureType(Enum): 568 system_error = "system_error" 569 config_error = "config_error" 570 transient_error = "transient_error" 571 572 573class HttpResponseFilter(BaseModel): 574 type: Literal["HttpResponseFilter"] 575 action: Optional[Action] = Field( 576 None, 577 description="Action to execute if a response matches the filter.", 578 examples=[ 579 "SUCCESS", 580 "FAIL", 581 "RETRY", 582 "IGNORE", 583 "RESET_PAGINATION", 584 "RATE_LIMITED", 585 "REFRESH_TOKEN_THEN_RETRY", 586 ], 587 title="Action", 588 ) 589 failure_type: Optional[FailureType] = Field( 590 None, 591 description="Failure type of traced exception if a response matches the filter.", 592 examples=["system_error", "config_error", "transient_error"], 593 title="Failure Type", 594 ) 595 error_message: Optional[str] = Field( 596 None, 597 description="Error Message to display if the response matches the filter.", 598 title="Error Message", 599 ) 600 error_message_contains: Optional[str] = Field( 601 None, 602 description="Match the response if its error message contains the substring.", 603 example=["This API operation is not enabled for this site"], 604 title="Error Message Substring", 605 ) 606 http_codes: Optional[List[int]] = Field( 607 None, 608 description="Match the response if its HTTP code is included in this list.", 609 examples=[[420, 429], [500]], 610 title="HTTP Codes", 611 unique_items=True, 612 ) 613 predicate: Optional[str] = Field( 614 None, 615 description="Match the response if the predicate evaluates to true.", 616 examples=[ 617 "{{ 'Too much requests' in response }}", 618 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 619 ], 620 title="Predicate", 621 ) 622 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 623 624 625class ComplexFieldType(BaseModel): 626 field_type: str 627 items: Optional[Union[str, ComplexFieldType]] = None 628 629 630class TypesMap(BaseModel): 631 target_type: Union[str, List[str], ComplexFieldType] 632 current_type: Union[str, List[str]] 633 condition: Optional[str] = None 634 635 636class SchemaTypeIdentifier(BaseModel): 637 type: Optional[Literal["SchemaTypeIdentifier"]] = None 638 schema_pointer: Optional[List[str]] = Field( 639 [], 640 description="List of nested fields defining the schema field path to extract. Defaults to [].", 641 title="Schema Path", 642 ) 643 key_pointer: List[str] = Field( 644 ..., 645 description="List of potentially nested fields describing the full path of the field key to extract.", 646 title="Key Path", 647 ) 648 type_pointer: Optional[List[str]] = Field( 649 None, 650 description="List of potentially nested fields describing the full path of the field type to extract.", 651 title="Type Path", 652 ) 653 types_mapping: Optional[List[TypesMap]] = None 654 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 655 656 657class InlineSchemaLoader(BaseModel): 658 type: Literal["InlineSchemaLoader"] 659 schema_: Optional[Dict[str, Any]] = Field( 660 None, 661 alias="schema", 662 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 663 title="Schema", 664 ) 665 666 667class JsonFileSchemaLoader(BaseModel): 668 type: Literal["JsonFileSchemaLoader"] 669 file_path: Optional[str] = Field( 670 None, 671 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 672 example=["./schemas/users.json"], 673 title="File Path", 674 ) 675 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 676 677 678class JsonDecoder(BaseModel): 679 type: Literal["JsonDecoder"] 680 681 682class JsonItemsDecoder(BaseModel): 683 type: Literal["JsonItemsDecoder"] 684 items_path: str = Field( 685 ..., 686 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 687 title="Items Path", 688 ) 689 encoding: Optional[str] = Field( 690 "utf-8", 691 description="The character encoding of the JSON data. Defaults to UTF-8.", 692 title="Encoding", 693 ) 694 695 696class JsonlDecoder(BaseModel): 697 type: Literal["JsonlDecoder"] 698 699 700class KeysToLower(BaseModel): 701 type: Literal["KeysToLower"] 702 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 703 704 705class KeysToSnakeCase(BaseModel): 706 type: Literal["KeysToSnakeCase"] 707 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 708 709 710class FlattenFields(BaseModel): 711 type: Literal["FlattenFields"] 712 flatten_lists: Optional[bool] = Field( 713 True, 714 description="Whether to flatten lists or leave it as is. Default is True.", 715 title="Flatten Lists", 716 ) 717 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 718 719 720class KeyTransformation(BaseModel): 721 type: Literal["KeyTransformation"] 722 prefix: Optional[str] = Field( 723 None, 724 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 725 examples=["flattened_"], 726 title="Key Prefix", 727 ) 728 suffix: Optional[str] = Field( 729 None, 730 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 731 examples=["_flattened"], 732 title="Key Suffix", 733 ) 734 735 736class DpathFlattenFields(BaseModel): 737 type: Literal["DpathFlattenFields"] 738 field_path: List[str] = Field( 739 ..., 740 description="A path to field that needs to be flattened.", 741 examples=[["data"], ["data", "*", "field"]], 742 title="Field Path", 743 ) 744 delete_origin_value: Optional[bool] = Field( 745 None, 746 description="Whether to delete the origin value or keep it. Default is False.", 747 title="Delete Origin Value", 748 ) 749 replace_record: Optional[bool] = Field( 750 None, 751 description="Whether to replace the origin record or not. Default is False.", 752 title="Replace Origin Record", 753 ) 754 key_transformation: Optional[KeyTransformation] = Field( 755 None, 756 description="Transformation for object keys. If not provided, original key will be used.", 757 title="Key transformation", 758 ) 759 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 760 761 762class KeysReplace(BaseModel): 763 type: Literal["KeysReplace"] 764 old: str = Field( 765 ..., 766 description="Old value to replace.", 767 examples=[ 768 " ", 769 "{{ record.id }}", 770 "{{ config['id'] }}", 771 "{{ stream_slice['id'] }}", 772 ], 773 title="Old value", 774 ) 775 new: str = Field( 776 ..., 777 description="New value to set.", 778 examples=[ 779 "_", 780 "{{ record.id }}", 781 "{{ config['id'] }}", 782 "{{ stream_slice['id'] }}", 783 ], 784 title="New value", 785 ) 786 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 787 788 789class IterableDecoder(BaseModel): 790 type: Literal["IterableDecoder"] 791 792 793class XmlDecoder(BaseModel): 794 type: Literal["XmlDecoder"] 795 796 797class CustomDecoder(BaseModel): 798 class Config: 799 extra = Extra.allow 800 801 type: Literal["CustomDecoder"] 802 class_name: str = Field( 803 ..., 804 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 805 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 806 title="Class Name", 807 ) 808 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 809 810 811class MinMaxDatetime(BaseModel): 812 type: Literal["MinMaxDatetime"] 813 datetime: str = Field( 814 ..., 815 description="Datetime value.", 816 examples=[ 817 "2021-01-01", 818 "2021-01-01T00:00:00Z", 819 "{{ config['start_time'] }}", 820 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 821 ], 822 title="Datetime", 823 ) 824 datetime_format: Optional[str] = Field( 825 "", 826 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 827 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 828 title="Datetime Format", 829 ) 830 max_datetime: Optional[str] = Field( 831 None, 832 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 833 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 834 title="Max Datetime", 835 ) 836 min_datetime: Optional[str] = Field( 837 None, 838 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 839 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 840 title="Min Datetime", 841 ) 842 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 843 844 845class NoAuth(BaseModel): 846 type: Literal["NoAuth"] 847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 848 849 850class NoPagination(BaseModel): 851 type: Literal["NoPagination"] 852 853 854class State(BaseModel): 855 class Config: 856 extra = Extra.allow 857 858 min: int 859 max: int 860 861 862class OAuthScope(BaseModel): 863 class Config: 864 extra = Extra.allow 865 866 scope: str = Field( 867 ..., 868 description="The OAuth scope string to request from the provider.", 869 ) 870 871 872class OauthConnectorInputSpecification(BaseModel): 873 class Config: 874 extra = Extra.allow 875 876 consent_url: str = Field( 877 ..., 878 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 879 examples=[ 880 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 881 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 882 ], 883 title="Consent URL", 884 ) 885 scope: Optional[str] = Field( 886 None, 887 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 888 examples=["user:read user:read_orders workspaces:read"], 889 title="Scopes", 890 ) 891 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 892 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 893 # The CDK schema defines the manifest contract; the platform reads these fields 894 # during the OAuth consent flow to build the authorization URL. 895 scopes: Optional[List[OAuthScope]] = Field( 896 None, 897 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 898 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 899 title="Scopes", 900 ) 901 optional_scopes: Optional[List[OAuthScope]] = Field( 902 None, 903 description="Optional OAuth scope objects that may or may not be granted.", 904 examples=[[{"scope": "admin:read"}]], 905 title="Optional Scopes", 906 ) 907 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 908 ScopesJoinStrategy.space, 909 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 910 title="Scopes Join Strategy", 911 ) 912 access_token_url: str = Field( 913 ..., 914 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 915 examples=[ 916 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 917 ], 918 title="Access Token URL", 919 ) 920 access_token_headers: Optional[Dict[str, Any]] = Field( 921 None, 922 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 923 examples=[ 924 { 925 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 926 } 927 ], 928 title="Access Token Headers", 929 ) 930 access_token_params: Optional[Dict[str, Any]] = Field( 931 None, 932 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 933 examples=[ 934 { 935 "{{ auth_code_key }}": "{{ auth_code_value }}", 936 "{{ client_id_key }}": "{{ client_id_value }}", 937 "{{ client_secret_key }}": "{{ client_secret_value }}", 938 } 939 ], 940 title="Access Token Query Params (Json Encoded)", 941 ) 942 extract_output: Optional[List[str]] = Field( 943 None, 944 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 945 examples=[["access_token", "refresh_token", "other_field"]], 946 title="Extract Output", 947 ) 948 state: Optional[State] = Field( 949 None, 950 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 951 examples=[{"min": 7, "max": 128}], 952 title="Configurable State Query Param", 953 ) 954 client_id_key: Optional[str] = Field( 955 None, 956 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 957 examples=["my_custom_client_id_key_name"], 958 title="Client ID Key Override", 959 ) 960 client_secret_key: Optional[str] = Field( 961 None, 962 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 963 examples=["my_custom_client_secret_key_name"], 964 title="Client Secret Key Override", 965 ) 966 scope_key: Optional[str] = Field( 967 None, 968 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 969 examples=["my_custom_scope_key_key_name"], 970 title="Scopes Key Override", 971 ) 972 state_key: Optional[str] = Field( 973 None, 974 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 975 examples=["my_custom_state_key_key_name"], 976 title="State Key Override", 977 ) 978 auth_code_key: Optional[str] = Field( 979 None, 980 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 981 examples=["my_custom_auth_code_key_name"], 982 title="Auth Code Key Override", 983 ) 984 redirect_uri_key: Optional[str] = Field( 985 None, 986 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 987 examples=["my_custom_redirect_uri_key_name"], 988 title="Redirect URI Key Override", 989 ) 990 991 992class OAuthConfigSpecification(BaseModel): 993 class Config: 994 extra = Extra.allow 995 996 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 997 None, 998 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 999 examples=[ 1000 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1001 { 1002 "app_id": { 1003 "type": "string", 1004 "path_in_connector_config": ["info", "app_id"], 1005 } 1006 }, 1007 ], 1008 title="OAuth user input", 1009 ) 1010 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1011 None, 1012 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1013 title="DeclarativeOAuth Connector Specification", 1014 ) 1015 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1016 None, 1017 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1018 examples=[ 1019 { 1020 "refresh_token": { 1021 "type": "string,", 1022 "path_in_connector_config": ["credentials", "refresh_token"], 1023 } 1024 } 1025 ], 1026 title="OAuth output specification", 1027 ) 1028 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1029 None, 1030 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1031 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1032 title="OAuth input specification", 1033 ) 1034 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1035 None, 1036 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1037 examples=[ 1038 { 1039 "client_id": { 1040 "type": "string,", 1041 "path_in_connector_config": ["credentials", "client_id"], 1042 }, 1043 "client_secret": { 1044 "type": "string,", 1045 "path_in_connector_config": ["credentials", "client_secret"], 1046 }, 1047 } 1048 ], 1049 title="OAuth server output specification", 1050 ) 1051 1052 1053class OffsetIncrement(BaseModel): 1054 type: Literal["OffsetIncrement"] 1055 page_size: Optional[Union[int, str]] = Field( 1056 None, 1057 description="The number of records to include in each pages.", 1058 examples=[100, "{{ config['page_size'] }}"], 1059 title="Limit", 1060 ) 1061 inject_on_first_request: Optional[bool] = Field( 1062 False, 1063 description="Using the `offset` with value `0` during the first request", 1064 title="Inject Offset on First Request", 1065 ) 1066 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1067 1068 1069class PageIncrement(BaseModel): 1070 type: Literal["PageIncrement"] 1071 page_size: Optional[Union[int, str]] = Field( 1072 None, 1073 description="The number of records to include in each pages.", 1074 examples=[100, "100", "{{ config['page_size'] }}"], 1075 title="Page Size", 1076 ) 1077 start_from_page: Optional[int] = Field( 1078 0, 1079 description="Index of the first page to request.", 1080 examples=[0, 1], 1081 title="Start From Page", 1082 ) 1083 inject_on_first_request: Optional[bool] = Field( 1084 False, 1085 description="Using the `page number` with value defined by `start_from_page` during the first request", 1086 title="Inject Page Number on First Request", 1087 ) 1088 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1089 1090 1091class PrimaryKey(BaseModel): 1092 __root__: Union[str, List[str], List[List[str]]] = Field( 1093 ..., 1094 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1095 examples=["id", ["code", "type"]], 1096 title="Primary Key", 1097 ) 1098 1099 1100class PropertyLimitType(Enum): 1101 characters = "characters" 1102 property_count = "property_count" 1103 1104 1105class PropertyChunking(BaseModel): 1106 type: Literal["PropertyChunking"] 1107 property_limit_type: PropertyLimitType = Field( 1108 ..., 1109 description="The type used to determine the maximum number of properties per chunk", 1110 title="Property Limit Type", 1111 ) 1112 property_limit: Optional[int] = Field( 1113 None, 1114 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1115 title="Property Limit", 1116 ) 1117 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1118 None, 1119 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1120 title="Record Merge Strategy", 1121 ) 1122 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1123 1124 1125class RecordFilter(BaseModel): 1126 type: Literal["RecordFilter"] 1127 condition: Optional[str] = Field( 1128 "", 1129 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1130 examples=[ 1131 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1132 "{{ record.status in ['active', 'expired'] }}", 1133 ], 1134 title="Condition", 1135 ) 1136 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1137 1138 1139class SchemaNormalization(Enum): 1140 Default = "Default" 1141 None_ = "None" 1142 1143 1144class RemoveFields(BaseModel): 1145 type: Literal["RemoveFields"] 1146 condition: Optional[str] = Field( 1147 "", 1148 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1149 examples=[ 1150 "{{ property|string == '' }}", 1151 "{{ property is integer }}", 1152 "{{ property|length > 5 }}", 1153 "{{ property == 'some_string_to_match' }}", 1154 ], 1155 ) 1156 field_pointers: List[List[str]] = Field( 1157 ..., 1158 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1159 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1160 title="Field Paths", 1161 ) 1162 1163 1164class RequestPath(BaseModel): 1165 type: Literal["RequestPath"] 1166 1167 1168class InjectInto(Enum): 1169 request_parameter = "request_parameter" 1170 header = "header" 1171 body_data = "body_data" 1172 body_json = "body_json" 1173 1174 1175class RequestOption(BaseModel): 1176 type: Literal["RequestOption"] 1177 inject_into: InjectInto = Field( 1178 ..., 1179 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1180 examples=["request_parameter", "header", "body_data", "body_json"], 1181 title="Inject Into", 1182 ) 1183 field_name: Optional[str] = Field( 1184 None, 1185 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1186 examples=["segment_id"], 1187 title="Field Name", 1188 ) 1189 field_path: Optional[List[str]] = Field( 1190 None, 1191 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1192 examples=[["data", "viewer", "id"]], 1193 title="Field Path", 1194 ) 1195 1196 1197class Schemas(BaseModel): 1198 pass 1199 1200 class Config: 1201 extra = Extra.allow 1202 1203 1204class LegacySessionTokenAuthenticator(BaseModel): 1205 type: Literal["LegacySessionTokenAuthenticator"] 1206 header: str = Field( 1207 ..., 1208 description="The name of the session token header that will be injected in the request", 1209 examples=["X-Session"], 1210 title="Session Request Header", 1211 ) 1212 login_url: str = Field( 1213 ..., 1214 description="Path of the login URL (do not include the base URL)", 1215 examples=["session"], 1216 title="Login Path", 1217 ) 1218 session_token: Optional[str] = Field( 1219 None, 1220 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1221 example=["{{ config['session_token'] }}"], 1222 title="Session Token", 1223 ) 1224 session_token_response_key: str = Field( 1225 ..., 1226 description="Name of the key of the session token to be extracted from the response", 1227 examples=["id"], 1228 title="Response Token Response Key", 1229 ) 1230 username: Optional[str] = Field( 1231 None, 1232 description="Username used to authenticate and obtain a session token", 1233 examples=[" {{ config['username'] }}"], 1234 title="Username", 1235 ) 1236 password: Optional[str] = Field( 1237 "", 1238 description="Password used to authenticate and obtain a session token", 1239 examples=["{{ config['password'] }}", ""], 1240 title="Password", 1241 ) 1242 validate_session_url: str = Field( 1243 ..., 1244 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1245 examples=["user/current"], 1246 title="Validate Session Path", 1247 ) 1248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1249 1250 1251class Action1(Enum): 1252 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1253 RESET = "RESET" 1254 1255 1256class PaginationResetLimits(BaseModel): 1257 type: Literal["PaginationResetLimits"] 1258 number_of_records: Optional[int] = None 1259 1260 1261class CsvDecoder(BaseModel): 1262 type: Literal["CsvDecoder"] 1263 encoding: Optional[str] = "utf-8" 1264 delimiter: Optional[str] = "," 1265 set_values_to_none: Optional[List[str]] = None 1266 1267 1268class AsyncJobStatusMap(BaseModel): 1269 type: Optional[Literal["AsyncJobStatusMap"]] = None 1270 running: List[str] 1271 completed: List[str] 1272 failed: List[str] 1273 timeout: List[str] 1274 skipped: Optional[List[str]] = None 1275 1276 1277class ValueType(Enum): 1278 string = "string" 1279 number = "number" 1280 integer = "integer" 1281 boolean = "boolean" 1282 1283 1284class WaitTimeFromHeader(BaseModel): 1285 type: Literal["WaitTimeFromHeader"] 1286 header: str = Field( 1287 ..., 1288 description="The name of the response header defining how long to wait before retrying.", 1289 examples=["Retry-After"], 1290 title="Response Header Name", 1291 ) 1292 regex: Optional[str] = Field( 1293 None, 1294 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1295 examples=["([-+]?\\d+)"], 1296 title="Extraction Regex", 1297 ) 1298 max_waiting_time_in_seconds: Optional[float] = Field( 1299 None, 1300 description="Given the value extracted from the header is greater than this value, stop the stream.", 1301 examples=[3600], 1302 title="Max Waiting Time in Seconds", 1303 ) 1304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1305 1306 1307class WaitUntilTimeFromHeader(BaseModel): 1308 type: Literal["WaitUntilTimeFromHeader"] 1309 header: str = Field( 1310 ..., 1311 description="The name of the response header defining how long to wait before retrying.", 1312 examples=["wait_time"], 1313 title="Response Header", 1314 ) 1315 min_wait: Optional[Union[float, str]] = Field( 1316 None, 1317 description="Minimum time to wait before retrying.", 1318 examples=[10, "60"], 1319 title="Minimum Wait Time", 1320 ) 1321 regex: Optional[str] = Field( 1322 None, 1323 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1324 examples=["([-+]?\\d+)"], 1325 title="Extraction Regex", 1326 ) 1327 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1328 1329 1330class ComponentMappingDefinition(BaseModel): 1331 type: Literal["ComponentMappingDefinition"] 1332 field_path: List[str] = Field( 1333 ..., 1334 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1335 examples=[ 1336 ["name"], 1337 ["retriever", "requester", "url"], 1338 ["retriever", "requester", "{{ components_values.field }}"], 1339 ["*", "**", "name"], 1340 ], 1341 title="Field Path", 1342 ) 1343 value: str = Field( 1344 ..., 1345 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1346 examples=[ 1347 "{{ components_values['updates'] }}", 1348 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1349 "{{ config['segment_id'] }}", 1350 "{{ stream_slice['parent_id'] }}", 1351 "{{ stream_slice['extra_fields']['name'] }}", 1352 ], 1353 title="Value", 1354 ) 1355 value_type: Optional[ValueType] = Field( 1356 None, 1357 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1358 title="Value Type", 1359 ) 1360 create_or_update: Optional[bool] = Field( 1361 False, 1362 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1363 title="Create or Update", 1364 ) 1365 condition: Optional[str] = Field( 1366 None, 1367 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1368 examples=[ 1369 "{{ components_values.get('cursor_field', None) }}", 1370 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1371 ], 1372 title="Condition", 1373 ) 1374 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1375 1376 1377class StreamConfig(BaseModel): 1378 type: Literal["StreamConfig"] 1379 configs_pointer: List[str] = Field( 1380 ..., 1381 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1382 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1383 title="Configs Pointer", 1384 ) 1385 default_values: Optional[List[Dict[str, Any]]] = Field( 1386 None, 1387 description="A list of default values, each matching the structure expected from the parsed component value.", 1388 title="Default Values", 1389 ) 1390 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1391 1392 1393class ConfigComponentsResolver(BaseModel): 1394 type: Literal["ConfigComponentsResolver"] 1395 stream_config: Union[List[StreamConfig], StreamConfig] 1396 components_mapping: List[ComponentMappingDefinition] 1397 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1398 1399 1400class StreamParametersDefinition(BaseModel): 1401 type: Literal["StreamParametersDefinition"] 1402 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1403 ..., 1404 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1405 examples=[ 1406 [ 1407 { 1408 "name": "test stream", 1409 "$parameters": {"entity": "test entity"}, 1410 "primary_key": "test key", 1411 } 1412 ] 1413 ], 1414 title="Stream Parameters", 1415 ) 1416 1417 1418class ParametrizedComponentsResolver(BaseModel): 1419 type: Literal["ParametrizedComponentsResolver"] 1420 stream_parameters: StreamParametersDefinition 1421 components_mapping: List[ComponentMappingDefinition] 1422 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1423 1424 1425class RequestBodyPlainText(BaseModel): 1426 type: Literal["RequestBodyPlainText"] 1427 value: str 1428 1429 1430class RequestBodyUrlEncodedForm(BaseModel): 1431 type: Literal["RequestBodyUrlEncodedForm"] 1432 value: Dict[str, str] 1433 1434 1435class RequestBodyJsonObject(BaseModel): 1436 type: Literal["RequestBodyJsonObject"] 1437 value: Dict[str, Any] 1438 1439 1440class RequestBodyGraphQlQuery(BaseModel): 1441 class Config: 1442 extra = Extra.allow 1443 1444 query: str = Field(..., description="The GraphQL query to be executed") 1445 1446 1447class ValidateAdheresToSchema(BaseModel): 1448 type: Literal["ValidateAdheresToSchema"] 1449 base_schema: Union[str, Dict[str, Any]] = Field( 1450 ..., 1451 description="The base JSON schema against which the user-provided schema will be validated.", 1452 examples=[ 1453 "{{ config['report_validation_schema'] }}", 1454 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1455 { 1456 "$schema": "http://json-schema.org/draft-07/schema#", 1457 "title": "Person", 1458 "type": "object", 1459 "properties": { 1460 "name": {"type": "string", "description": "The person's name"}, 1461 "age": { 1462 "type": "integer", 1463 "minimum": 0, 1464 "description": "The person's age", 1465 }, 1466 }, 1467 "required": ["name", "age"], 1468 }, 1469 ], 1470 title="Base JSON Schema", 1471 ) 1472 1473 1474class CustomValidationStrategy(BaseModel): 1475 class Config: 1476 extra = Extra.allow 1477 1478 type: Literal["CustomValidationStrategy"] 1479 class_name: str = Field( 1480 ..., 1481 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1482 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1483 title="Class Name", 1484 ) 1485 1486 1487class ConfigRemapField(BaseModel): 1488 type: Literal["ConfigRemapField"] 1489 map: Union[Dict[str, Any], str] = Field( 1490 ..., 1491 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1492 examples=[ 1493 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1494 "{{ config['status_mapping'] }}", 1495 ], 1496 title="Value Mapping", 1497 ) 1498 field_path: List[str] = Field( 1499 ..., 1500 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1501 examples=[ 1502 ["status"], 1503 ["data", "status"], 1504 ["data", "{{ config.name }}", "status"], 1505 ["data", "*", "status"], 1506 ], 1507 title="Field Path", 1508 ) 1509 1510 1511class ConfigRemoveFields(BaseModel): 1512 type: Literal["ConfigRemoveFields"] 1513 field_pointers: List[List[str]] = Field( 1514 ..., 1515 description="A list of field pointers to be removed from the config.", 1516 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1517 title="Field Pointers", 1518 ) 1519 condition: Optional[str] = Field( 1520 "", 1521 description="Fields will be removed if expression is evaluated to True.", 1522 examples=[ 1523 "{{ config['environemnt'] == 'sandbox' }}", 1524 "{{ property is integer }}", 1525 "{{ property|length > 5 }}", 1526 "{{ property == 'some_string_to_match' }}", 1527 ], 1528 ) 1529 1530 1531class CustomConfigTransformation(BaseModel): 1532 type: Literal["CustomConfigTransformation"] 1533 class_name: str = Field( 1534 ..., 1535 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1536 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1537 ) 1538 parameters: Optional[Dict[str, Any]] = Field( 1539 None, 1540 alias="$parameters", 1541 description="Additional parameters to be passed to the custom config transformation.", 1542 ) 1543 1544 1545class AddedFieldDefinition(BaseModel): 1546 type: Literal["AddedFieldDefinition"] 1547 path: List[str] = Field( 1548 ..., 1549 description="List of strings defining the path where to add the value on the record.", 1550 examples=[["segment_id"], ["metadata", "segment_id"]], 1551 title="Path", 1552 ) 1553 value: str = Field( 1554 ..., 1555 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1556 examples=[ 1557 "{{ record['updates'] }}", 1558 "{{ record['MetaData']['LastUpdatedTime'] }}", 1559 "{{ stream_partition['segment_id'] }}", 1560 ], 1561 title="Value", 1562 ) 1563 value_type: Optional[ValueType] = Field( 1564 None, 1565 description="Type of the value. If not specified, the type will be inferred from the value.", 1566 title="Value Type", 1567 ) 1568 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1569 1570 1571class AddFields(BaseModel): 1572 type: Literal["AddFields"] 1573 fields: List[AddedFieldDefinition] = Field( 1574 ..., 1575 description="List of transformations (path and corresponding value) that will be added to the record.", 1576 title="Fields", 1577 ) 1578 condition: Optional[str] = Field( 1579 "", 1580 description="Fields will be added if expression is evaluated to True.", 1581 examples=[ 1582 "{{ property|string == '' }}", 1583 "{{ property is integer }}", 1584 "{{ property|length > 5 }}", 1585 "{{ property == 'some_string_to_match' }}", 1586 ], 1587 ) 1588 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1589 1590 1591class ApiKeyAuthenticator(BaseModel): 1592 type: Literal["ApiKeyAuthenticator"] 1593 api_token: Optional[str] = Field( 1594 None, 1595 description="The API key to inject in the request. Fill it in the user inputs.", 1596 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1597 title="API Key", 1598 ) 1599 header: Optional[str] = Field( 1600 None, 1601 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1602 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1603 title="Header Name", 1604 ) 1605 inject_into: Optional[RequestOption] = Field( 1606 None, 1607 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1608 examples=[ 1609 {"inject_into": "header", "field_name": "Authorization"}, 1610 {"inject_into": "request_parameter", "field_name": "authKey"}, 1611 ], 1612 title="Inject API Key Into Outgoing HTTP Request", 1613 ) 1614 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1615 1616 1617class AuthFlow(BaseModel): 1618 auth_flow_type: Optional[AuthFlowType] = Field( 1619 None, description="The type of auth to use", title="Auth flow type" 1620 ) 1621 predicate_key: Optional[List[str]] = Field( 1622 None, 1623 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1624 examples=[["credentials", "auth_type"]], 1625 title="Predicate key", 1626 ) 1627 predicate_value: Optional[str] = Field( 1628 None, 1629 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1630 examples=["Oauth"], 1631 title="Predicate value", 1632 ) 1633 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1634 1635 1636class CheckStream(BaseModel): 1637 type: Literal["CheckStream"] 1638 stream_names: Optional[List[str]] = Field( 1639 None, 1640 description="Names of the streams to try reading from when running a check operation.", 1641 examples=[["users"], ["users", "contacts"]], 1642 title="Stream Names", 1643 ) 1644 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1645 1646 1647class IncrementingCountCursor(BaseModel): 1648 type: Literal["IncrementingCountCursor"] 1649 cursor_field: str = Field( 1650 ..., 1651 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1652 examples=["created_at", "{{ config['record_cursor'] }}"], 1653 title="Cursor Field", 1654 ) 1655 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1656 None, 1657 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1658 title="Allow Catalog Defined Cursor Field", 1659 ) 1660 start_value: Optional[Union[str, int]] = Field( 1661 None, 1662 description="The value that determines the earliest record that should be synced.", 1663 examples=[0, "{{ config['start_value'] }}"], 1664 title="Start Value", 1665 ) 1666 start_value_option: Optional[RequestOption] = Field( 1667 None, 1668 description="Optionally configures how the start value will be sent in requests to the source API.", 1669 title="Inject Start Value Into Outgoing HTTP Request", 1670 ) 1671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1672 1673 1674class DatetimeBasedCursor(BaseModel): 1675 type: Literal["DatetimeBasedCursor"] 1676 clamping: Optional[Clamping] = Field( 1677 None, 1678 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1679 title="Date Range Clamping", 1680 ) 1681 cursor_field: str = Field( 1682 ..., 1683 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1684 examples=["created_at", "{{ config['record_cursor'] }}"], 1685 title="Cursor Field", 1686 ) 1687 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1688 None, 1689 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1690 title="Allow Catalog Defined Cursor Field", 1691 ) 1692 cursor_datetime_formats: Optional[List[str]] = Field( 1693 None, 1694 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1695 examples=[ 1696 "%Y-%m-%d", 1697 "%Y-%m-%d %H:%M:%S", 1698 "%Y-%m-%dT%H:%M:%S", 1699 "%Y-%m-%dT%H:%M:%SZ", 1700 "%Y-%m-%dT%H:%M:%S%z", 1701 "%Y-%m-%dT%H:%M:%S.%fZ", 1702 "%Y-%m-%dT%H:%M:%S.%f%z", 1703 "%Y-%m-%d %H:%M:%S.%f+00:00", 1704 "%s", 1705 "%ms", 1706 ], 1707 title="Cursor Datetime Formats", 1708 ) 1709 start_datetime: Union[MinMaxDatetime, str] = Field( 1710 ..., 1711 description="The datetime that determines the earliest record that should be synced.", 1712 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1713 title="Start Datetime", 1714 ) 1715 start_time_option: Optional[RequestOption] = Field( 1716 None, 1717 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1718 title="Inject Start Time Into Outgoing HTTP Request", 1719 ) 1720 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1721 None, 1722 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1723 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1724 title="End Datetime", 1725 ) 1726 end_time_option: Optional[RequestOption] = Field( 1727 None, 1728 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1729 title="Inject End Time Into Outgoing HTTP Request", 1730 ) 1731 datetime_format: str = Field( 1732 ..., 1733 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1734 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1735 title="Outgoing Datetime Format", 1736 ) 1737 cursor_granularity: Optional[str] = Field( 1738 None, 1739 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1740 examples=["PT1S"], 1741 title="Cursor Granularity", 1742 ) 1743 is_data_feed: Optional[bool] = Field( 1744 None, 1745 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1746 title="Data Feed API", 1747 ) 1748 is_client_side_incremental: Optional[bool] = Field( 1749 None, 1750 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1751 title="Client-side Incremental Filtering", 1752 ) 1753 is_compare_strictly: Optional[bool] = Field( 1754 False, 1755 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1756 title="Strict Start-End Time Comparison", 1757 ) 1758 global_substream_cursor: Optional[bool] = Field( 1759 False, 1760 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1761 title="Global Substream Cursor", 1762 ) 1763 lookback_window: Optional[str] = Field( 1764 None, 1765 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1766 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1767 title="Lookback Window", 1768 ) 1769 partition_field_end: Optional[str] = Field( 1770 None, 1771 description="Name of the partition start time field.", 1772 examples=["ending_time"], 1773 title="Partition Field End", 1774 ) 1775 partition_field_start: Optional[str] = Field( 1776 None, 1777 description="Name of the partition end time field.", 1778 examples=["starting_time"], 1779 title="Partition Field Start", 1780 ) 1781 step: Optional[str] = Field( 1782 None, 1783 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1784 examples=["P1W", "{{ config['step_increment'] }}"], 1785 title="Step", 1786 ) 1787 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1788 1789 1790class JwtAuthenticator(BaseModel): 1791 type: Literal["JwtAuthenticator"] 1792 secret_key: str = Field( 1793 ..., 1794 description="Secret used to sign the JSON web token.", 1795 examples=["{{ config['secret_key'] }}"], 1796 title="Secret Key", 1797 ) 1798 base64_encode_secret_key: Optional[bool] = Field( 1799 False, 1800 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1801 title="Base64-encode Secret Key", 1802 ) 1803 algorithm: Algorithm = Field( 1804 ..., 1805 description="Algorithm used to sign the JSON web token.", 1806 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1807 title="Algorithm", 1808 ) 1809 token_duration: Optional[int] = Field( 1810 1200, 1811 description="The amount of time in seconds a JWT token can be valid after being issued.", 1812 examples=[1200, 3600], 1813 title="Token Duration", 1814 ) 1815 header_prefix: Optional[str] = Field( 1816 None, 1817 description="The prefix to be used within the Authentication header.", 1818 examples=["Bearer", "Basic"], 1819 title="Header Prefix", 1820 ) 1821 jwt_headers: Optional[JwtHeaders] = Field( 1822 None, 1823 description="JWT headers used when signing JSON web token.", 1824 title="JWT Headers", 1825 ) 1826 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1827 None, 1828 description="Additional headers to be included with the JWT headers object.", 1829 title="Additional JWT Headers", 1830 ) 1831 jwt_payload: Optional[JwtPayload] = Field( 1832 None, 1833 description="JWT Payload used when signing JSON web token.", 1834 title="JWT Payload", 1835 ) 1836 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1837 None, 1838 description="Additional properties to be added to the JWT payload.", 1839 title="Additional JWT Payload Properties", 1840 ) 1841 passphrase: Optional[str] = Field( 1842 None, 1843 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1844 examples=["{{ config['passphrase'] }}"], 1845 title="Passphrase", 1846 ) 1847 request_option: Optional[RequestOption] = Field( 1848 None, 1849 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1850 title="Request Option", 1851 ) 1852 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1853 1854 1855class OAuthAuthenticator(BaseModel): 1856 type: Literal["OAuthAuthenticator"] 1857 client_id_name: Optional[str] = Field( 1858 "client_id", 1859 description="The name of the property to use to refresh the `access_token`.", 1860 examples=["custom_app_id"], 1861 title="Client ID Property Name", 1862 ) 1863 client_id: Optional[str] = Field( 1864 None, 1865 description="The OAuth client ID. Fill it in the user inputs.", 1866 examples=[ 1867 "{{ config['client_id'] }}", 1868 "{{ config['credentials']['client_id }}", 1869 ], 1870 title="Client ID", 1871 ) 1872 client_secret_name: Optional[str] = Field( 1873 "client_secret", 1874 description="The name of the property to use to refresh the `access_token`.", 1875 examples=["custom_app_secret"], 1876 title="Client Secret Property Name", 1877 ) 1878 client_secret: Optional[str] = Field( 1879 None, 1880 description="The OAuth client secret. Fill it in the user inputs.", 1881 examples=[ 1882 "{{ config['client_secret'] }}", 1883 "{{ config['credentials']['client_secret }}", 1884 ], 1885 title="Client Secret", 1886 ) 1887 refresh_token_name: Optional[str] = Field( 1888 "refresh_token", 1889 description="The name of the property to use to refresh the `access_token`.", 1890 examples=["custom_app_refresh_value"], 1891 title="Refresh Token Property Name", 1892 ) 1893 refresh_token: Optional[str] = Field( 1894 None, 1895 description="Credential artifact used to get a new access token.", 1896 examples=[ 1897 "{{ config['refresh_token'] }}", 1898 "{{ config['credentials]['refresh_token'] }}", 1899 ], 1900 title="Refresh Token", 1901 ) 1902 token_refresh_endpoint: Optional[str] = Field( 1903 None, 1904 description="The full URL to call to obtain a new access token.", 1905 examples=["https://connect.squareup.com/oauth2/token"], 1906 title="Token Refresh Endpoint", 1907 ) 1908 access_token_name: Optional[str] = Field( 1909 "access_token", 1910 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1911 examples=["access_token"], 1912 title="Access Token Property Name", 1913 ) 1914 access_token_value: Optional[str] = Field( 1915 None, 1916 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1917 examples=["secret_access_token_value"], 1918 title="Access Token Value", 1919 ) 1920 expires_in_name: Optional[str] = Field( 1921 "expires_in", 1922 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1923 examples=["expires_in"], 1924 title="Token Expiry Property Name", 1925 ) 1926 grant_type_name: Optional[str] = Field( 1927 "grant_type", 1928 description="The name of the property to use to refresh the `access_token`.", 1929 examples=["custom_grant_type"], 1930 title="Grant Type Property Name", 1931 ) 1932 grant_type: Optional[str] = Field( 1933 "refresh_token", 1934 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1935 examples=["refresh_token", "client_credentials"], 1936 title="Grant Type", 1937 ) 1938 refresh_request_body: Optional[Dict[str, Any]] = Field( 1939 None, 1940 description="Body of the request sent to get a new access token.", 1941 examples=[ 1942 { 1943 "applicationId": "{{ config['application_id'] }}", 1944 "applicationSecret": "{{ config['application_secret'] }}", 1945 "token": "{{ config['token'] }}", 1946 } 1947 ], 1948 title="Refresh Request Body", 1949 ) 1950 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1951 None, 1952 description="Headers of the request sent to get a new access token.", 1953 examples=[ 1954 { 1955 "Authorization": "<AUTH_TOKEN>", 1956 "Content-Type": "application/x-www-form-urlencoded", 1957 } 1958 ], 1959 title="Refresh Request Headers", 1960 ) 1961 send_refresh_request_as_query_params: Optional[bool] = Field( 1962 False, 1963 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 1964 examples=[True], 1965 title="Send Refresh Request As Query Params", 1966 ) 1967 scopes: Optional[List[str]] = Field( 1968 None, 1969 description="List of scopes that should be granted to the access token.", 1970 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1971 title="Scopes", 1972 ) 1973 token_expiry_date: Optional[str] = Field( 1974 None, 1975 description="The access token expiry date.", 1976 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1977 title="Token Expiry Date", 1978 ) 1979 token_expiry_date_format: Optional[str] = Field( 1980 None, 1981 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1982 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1983 title="Token Expiry Date Format", 1984 ) 1985 refresh_token_error_status_codes: Optional[List[int]] = Field( 1986 None, 1987 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1988 examples=[[400, 500]], 1989 title="Refresh Token Error Status Codes", 1990 ) 1991 refresh_token_error_key: Optional[str] = Field( 1992 None, 1993 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1994 examples=["error"], 1995 title="Refresh Token Error Key", 1996 ) 1997 refresh_token_error_values: Optional[List[str]] = Field( 1998 None, 1999 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2000 examples=[["invalid_grant", "invalid_permissions"]], 2001 title="Refresh Token Error Values", 2002 ) 2003 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2004 None, 2005 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2006 title="Refresh Token Updater", 2007 ) 2008 profile_assertion: Optional[JwtAuthenticator] = Field( 2009 None, 2010 description="The authenticator being used to authenticate the client authenticator.", 2011 title="Profile Assertion", 2012 ) 2013 use_profile_assertion: Optional[bool] = Field( 2014 False, 2015 description="Enable using profile assertion as a flow for OAuth authorization.", 2016 title="Use Profile Assertion", 2017 ) 2018 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2019 2020 2021class FixedWindowCallRatePolicy(BaseModel): 2022 class Config: 2023 extra = Extra.allow 2024 2025 type: Literal["FixedWindowCallRatePolicy"] 2026 period: str = Field( 2027 ..., description="The time interval for the rate limit window.", title="Period" 2028 ) 2029 call_limit: int = Field( 2030 ..., 2031 description="The maximum number of calls allowed within the period.", 2032 title="Call Limit", 2033 ) 2034 matchers: List[HttpRequestRegexMatcher] = Field( 2035 ..., 2036 description="List of matchers that define which requests this policy applies to.", 2037 title="Matchers", 2038 ) 2039 2040 2041class MovingWindowCallRatePolicy(BaseModel): 2042 class Config: 2043 extra = Extra.allow 2044 2045 type: Literal["MovingWindowCallRatePolicy"] 2046 rates: List[Rate] = Field( 2047 ..., 2048 description="List of rates that define the call limits for different time intervals.", 2049 title="Rates", 2050 ) 2051 matchers: List[HttpRequestRegexMatcher] = Field( 2052 ..., 2053 description="List of matchers that define which requests this policy applies to.", 2054 title="Matchers", 2055 ) 2056 2057 2058class UnlimitedCallRatePolicy(BaseModel): 2059 class Config: 2060 extra = Extra.allow 2061 2062 type: Literal["UnlimitedCallRatePolicy"] 2063 matchers: List[HttpRequestRegexMatcher] = Field( 2064 ..., 2065 description="List of matchers that define which requests this policy applies to.", 2066 title="Matchers", 2067 ) 2068 2069 2070class DefaultErrorHandler(BaseModel): 2071 type: Literal["DefaultErrorHandler"] 2072 backoff_strategies: Optional[ 2073 List[ 2074 Union[ 2075 ConstantBackoffStrategy, 2076 ExponentialBackoffStrategy, 2077 WaitTimeFromHeader, 2078 WaitUntilTimeFromHeader, 2079 CustomBackoffStrategy, 2080 ] 2081 ] 2082 ] = Field( 2083 None, 2084 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2085 title="Backoff Strategies", 2086 ) 2087 max_retries: Optional[Union[int, str]] = Field( 2088 5, 2089 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2090 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2091 title="Max Retry Count", 2092 ) 2093 response_filters: Optional[List[HttpResponseFilter]] = Field( 2094 None, 2095 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2096 title="Response Filters", 2097 ) 2098 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2099 2100 2101class DefaultPaginator(BaseModel): 2102 type: Literal["DefaultPaginator"] 2103 pagination_strategy: Union[ 2104 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2105 ] = Field( 2106 ..., 2107 description="Strategy defining how records are paginated.", 2108 title="Pagination Strategy", 2109 ) 2110 page_size_option: Optional[RequestOption] = Field( 2111 None, title="Inject Page Size Into Outgoing HTTP Request" 2112 ) 2113 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2114 None, 2115 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2116 title="Inject Page Token Into Outgoing HTTP Request", 2117 ) 2118 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2119 2120 2121class RecordExpander(BaseModel): 2122 type: Literal["RecordExpander"] 2123 expand_records_from_field: List[str] = Field( 2124 ..., 2125 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2126 examples=[ 2127 ["lines", "data"], 2128 ["items"], 2129 ["nested", "array"], 2130 ["sections", "*", "items"], 2131 ], 2132 title="Expand Records From Field", 2133 ) 2134 remain_original_record: Optional[bool] = Field( 2135 False, 2136 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2137 title="Remain Original Record", 2138 ) 2139 on_no_records: Optional[OnNoRecords] = Field( 2140 OnNoRecords.skip, 2141 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2142 title="On No Records", 2143 ) 2144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2145 2146 2147class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2148 type: Literal["ApiKey"] 2149 inject_into: RequestOption = Field( 2150 ..., 2151 description="Configure how the API Key will be sent in requests to the source API.", 2152 examples=[ 2153 {"inject_into": "header", "field_name": "Authorization"}, 2154 {"inject_into": "request_parameter", "field_name": "authKey"}, 2155 ], 2156 title="Inject API Key Into Outgoing HTTP Request", 2157 ) 2158 api_token: Optional[str] = Field( 2159 "{{ session_token }}", 2160 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2161 examples=[ 2162 "{{ session_token }}", 2163 "Token {{ session_token }}", 2164 "Bearer {{ session_token }}", 2165 ], 2166 title="API Token Template", 2167 ) 2168 2169 2170class JsonSchemaPropertySelector(BaseModel): 2171 type: Literal["JsonSchemaPropertySelector"] 2172 transformations: Optional[ 2173 List[ 2174 Union[ 2175 AddFields, 2176 RemoveFields, 2177 KeysToLower, 2178 KeysToSnakeCase, 2179 FlattenFields, 2180 DpathFlattenFields, 2181 KeysReplace, 2182 CustomTransformation, 2183 ] 2184 ] 2185 ] = Field( 2186 None, 2187 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2188 title="Transformations", 2189 ) 2190 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2191 2192 2193class ListPartitionRouter(BaseModel): 2194 type: Literal["ListPartitionRouter"] 2195 cursor_field: str = Field( 2196 ..., 2197 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2198 examples=["section", "{{ config['section_key'] }}"], 2199 title="Current Partition Value Identifier", 2200 ) 2201 values: Union[str, List[str]] = Field( 2202 ..., 2203 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2204 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2205 title="Partition Values", 2206 ) 2207 request_option: Optional[RequestOption] = Field( 2208 None, 2209 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2210 title="Inject Partition Value Into Outgoing HTTP Request", 2211 ) 2212 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2213 2214 2215class PaginationReset(BaseModel): 2216 type: Literal["PaginationReset"] 2217 action: Action1 2218 limits: Optional[PaginationResetLimits] = None 2219 2220 2221class GzipDecoder(BaseModel): 2222 type: Literal["GzipDecoder"] 2223 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2224 2225 2226class RequestBodyGraphQL(BaseModel): 2227 type: Literal["RequestBodyGraphQL"] 2228 value: RequestBodyGraphQlQuery 2229 2230 2231class DpathValidator(BaseModel): 2232 type: Literal["DpathValidator"] 2233 field_path: List[str] = Field( 2234 ..., 2235 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2236 examples=[ 2237 ["data"], 2238 ["data", "records"], 2239 ["data", "{{ parameters.name }}"], 2240 ["data", "*", "record"], 2241 ], 2242 title="Field Path", 2243 ) 2244 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2245 ..., 2246 description="The condition that the specified config value will be evaluated against", 2247 title="Validation Strategy", 2248 ) 2249 2250 2251class PredicateValidator(BaseModel): 2252 type: Literal["PredicateValidator"] 2253 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2254 ..., 2255 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2256 examples=[ 2257 "test-value", 2258 "{{ config['api_version'] }}", 2259 "{{ config['tenant_id'] }}", 2260 123, 2261 ], 2262 title="Value", 2263 ) 2264 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2265 ..., 2266 description="The validation strategy to apply to the value.", 2267 title="Validation Strategy", 2268 ) 2269 2270 2271class ConfigAddFields(BaseModel): 2272 type: Literal["ConfigAddFields"] 2273 fields: List[AddedFieldDefinition] = Field( 2274 ..., 2275 description="A list of transformations (path and corresponding value) that will be added to the config.", 2276 title="Fields", 2277 ) 2278 condition: Optional[str] = Field( 2279 "", 2280 description="Fields will be added if expression is evaluated to True.", 2281 examples=[ 2282 "{{ config['environemnt'] == 'sandbox' }}", 2283 "{{ property is integer }}", 2284 "{{ property|length > 5 }}", 2285 "{{ property == 'some_string_to_match' }}", 2286 ], 2287 ) 2288 2289 2290class CompositeErrorHandler(BaseModel): 2291 type: Literal["CompositeErrorHandler"] 2292 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2293 Field( 2294 ..., 2295 description="List of error handlers to iterate on to determine how to handle a failed response.", 2296 title="Error Handlers", 2297 ) 2298 ) 2299 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2300 2301 2302class HTTPAPIBudget(BaseModel): 2303 class Config: 2304 extra = Extra.allow 2305 2306 type: Literal["HTTPAPIBudget"] 2307 policies: List[ 2308 Union[ 2309 FixedWindowCallRatePolicy, 2310 MovingWindowCallRatePolicy, 2311 UnlimitedCallRatePolicy, 2312 ] 2313 ] = Field( 2314 ..., 2315 description="List of call rate policies that define how many calls are allowed.", 2316 title="Policies", 2317 ) 2318 ratelimit_reset_header: Optional[str] = Field( 2319 "ratelimit-reset", 2320 description="The HTTP response header name that indicates when the rate limit resets.", 2321 title="Rate Limit Reset Header", 2322 ) 2323 ratelimit_remaining_header: Optional[str] = Field( 2324 "ratelimit-remaining", 2325 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2326 title="Rate Limit Remaining Header", 2327 ) 2328 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2329 [429], 2330 description="List of HTTP status codes that indicate a rate limit has been hit.", 2331 title="Status Codes for Rate Limit Hit", 2332 ) 2333 2334 2335class DpathExtractor(BaseModel): 2336 type: Literal["DpathExtractor"] 2337 field_path: List[str] = Field( 2338 ..., 2339 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2340 examples=[ 2341 ["data"], 2342 ["data", "records"], 2343 ["data", "{{ parameters.name }}"], 2344 ["data", "*", "record"], 2345 ], 2346 title="Field Path", 2347 ) 2348 record_expander: Optional[RecordExpander] = Field( 2349 None, 2350 description="Optional component to expand records by extracting items from nested array fields.", 2351 title="Record Expander", 2352 ) 2353 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2354 2355 2356class ZipfileDecoder(BaseModel): 2357 class Config: 2358 extra = Extra.allow 2359 2360 type: Literal["ZipfileDecoder"] 2361 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2362 ..., 2363 description="Parser to parse the decompressed data from the zipfile(s).", 2364 title="Parser", 2365 ) 2366 2367 2368class RecordSelector(BaseModel): 2369 type: Literal["RecordSelector"] 2370 extractor: Union[DpathExtractor, CustomRecordExtractor] 2371 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2372 None, 2373 description="Responsible for filtering records to be emitted by the Source.", 2374 title="Record Filter", 2375 ) 2376 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2377 None, 2378 description="Responsible for normalization according to the schema.", 2379 title="Schema Normalization", 2380 ) 2381 transform_before_filtering: Optional[bool] = Field( 2382 None, 2383 description="If true, transformation will be applied before record filtering.", 2384 title="Transform Before Filtering", 2385 ) 2386 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2387 2388 2389class ConfigMigration(BaseModel): 2390 type: Literal["ConfigMigration"] 2391 description: Optional[str] = Field( 2392 None, description="The description/purpose of the config migration." 2393 ) 2394 transformations: List[ 2395 Union[ 2396 ConfigRemapField, 2397 ConfigAddFields, 2398 ConfigRemoveFields, 2399 CustomConfigTransformation, 2400 ] 2401 ] = Field( 2402 ..., 2403 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2404 title="Transformations", 2405 ) 2406 2407 2408class ConfigNormalizationRules(BaseModel): 2409 class Config: 2410 extra = Extra.forbid 2411 2412 type: Literal["ConfigNormalizationRules"] 2413 config_migrations: Optional[List[ConfigMigration]] = Field( 2414 [], 2415 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2416 title="Config Migrations", 2417 ) 2418 transformations: Optional[ 2419 List[ 2420 Union[ 2421 ConfigRemapField, 2422 ConfigAddFields, 2423 ConfigRemoveFields, 2424 CustomConfigTransformation, 2425 ] 2426 ] 2427 ] = Field( 2428 [], 2429 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2430 title="Transformations", 2431 ) 2432 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2433 [], 2434 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2435 title="Validations", 2436 ) 2437 2438 2439class Spec(BaseModel): 2440 type: Literal["Spec"] 2441 connection_specification: Dict[str, Any] = Field( 2442 ..., 2443 description="A connection specification describing how a the connector can be configured.", 2444 title="Connection Specification", 2445 ) 2446 documentation_url: Optional[str] = Field( 2447 None, 2448 description="URL of the connector's documentation page.", 2449 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2450 title="Documentation URL", 2451 ) 2452 advanced_auth: Optional[AuthFlow] = Field( 2453 None, 2454 description="Advanced specification for configuring the authentication flow.", 2455 title="Advanced Auth", 2456 ) 2457 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2458 None, title="Config Normalization Rules" 2459 ) 2460 2461 2462class DeclarativeSource1(BaseModel): 2463 class Config: 2464 extra = Extra.forbid 2465 2466 type: Literal["DeclarativeSource"] 2467 check: Union[CheckStream, CheckDynamicStream] 2468 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2469 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2470 version: str = Field( 2471 ..., 2472 description="The version of the Airbyte CDK used to build and test the source.", 2473 ) 2474 schemas: Optional[Schemas] = None 2475 definitions: Optional[Dict[str, Any]] = None 2476 spec: Optional[Spec] = None 2477 concurrency_level: Optional[ConcurrencyLevel] = None 2478 api_budget: Optional[HTTPAPIBudget] = None 2479 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2480 None, 2481 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2482 title="Stream Groups", 2483 ) 2484 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2485 None, 2486 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2487 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2488 title="Maximum Concurrent Asynchronous Jobs", 2489 ) 2490 metadata: Optional[Dict[str, Any]] = Field( 2491 None, 2492 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2493 ) 2494 description: Optional[str] = Field( 2495 None, 2496 description="A description of the connector. It will be presented on the Source documentation page.", 2497 ) 2498 2499 2500class DeclarativeSource2(BaseModel): 2501 class Config: 2502 extra = Extra.forbid 2503 2504 type: Literal["DeclarativeSource"] 2505 check: Union[CheckStream, CheckDynamicStream] 2506 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2507 None 2508 ) 2509 dynamic_streams: List[DynamicDeclarativeStream] 2510 version: str = Field( 2511 ..., 2512 description="The version of the Airbyte CDK used to build and test the source.", 2513 ) 2514 schemas: Optional[Schemas] = None 2515 definitions: Optional[Dict[str, Any]] = None 2516 spec: Optional[Spec] = None 2517 concurrency_level: Optional[ConcurrencyLevel] = None 2518 api_budget: Optional[HTTPAPIBudget] = None 2519 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2520 None, 2521 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2522 title="Stream Groups", 2523 ) 2524 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2525 None, 2526 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2527 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2528 title="Maximum Concurrent Asynchronous Jobs", 2529 ) 2530 metadata: Optional[Dict[str, Any]] = Field( 2531 None, 2532 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2533 ) 2534 description: Optional[str] = Field( 2535 None, 2536 description="A description of the connector. It will be presented on the Source documentation page.", 2537 ) 2538 2539 2540class DeclarativeSource(BaseModel): 2541 class Config: 2542 extra = Extra.forbid 2543 2544 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2545 ..., 2546 description="An API source that extracts data according to its declarative components.", 2547 title="DeclarativeSource", 2548 ) 2549 2550 2551class SelectiveAuthenticator(BaseModel): 2552 class Config: 2553 extra = Extra.allow 2554 2555 type: Literal["SelectiveAuthenticator"] 2556 authenticator_selection_path: List[str] = Field( 2557 ..., 2558 description="Path of the field in config with selected authenticator name", 2559 examples=[["auth"], ["auth", "type"]], 2560 title="Authenticator Selection Path", 2561 ) 2562 authenticators: Dict[ 2563 str, 2564 Union[ 2565 ApiKeyAuthenticator, 2566 BasicHttpAuthenticator, 2567 BearerAuthenticator, 2568 OAuthAuthenticator, 2569 JwtAuthenticator, 2570 SessionTokenAuthenticator, 2571 LegacySessionTokenAuthenticator, 2572 CustomAuthenticator, 2573 NoAuth, 2574 ], 2575 ] = Field( 2576 ..., 2577 description="Authenticators to select from.", 2578 examples=[ 2579 { 2580 "authenticators": { 2581 "token": "#/definitions/ApiKeyAuthenticator", 2582 "oauth": "#/definitions/OAuthAuthenticator", 2583 "jwt": "#/definitions/JwtAuthenticator", 2584 } 2585 } 2586 ], 2587 title="Authenticators", 2588 ) 2589 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2590 2591 2592class ConditionalStreams(BaseModel): 2593 type: Literal["ConditionalStreams"] 2594 condition: str = Field( 2595 ..., 2596 description="Condition that will be evaluated to determine if a set of streams should be available.", 2597 examples=["{{ config['is_sandbox'] }}"], 2598 title="Condition", 2599 ) 2600 streams: List[DeclarativeStream] = Field( 2601 ..., 2602 description="Streams that will be used during an operation based on the condition.", 2603 title="Streams", 2604 ) 2605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2606 2607 2608class FileUploader(BaseModel): 2609 type: Literal["FileUploader"] 2610 requester: Union[HttpRequester, CustomRequester] = Field( 2611 ..., 2612 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2613 ) 2614 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2615 ..., 2616 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2617 ) 2618 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2619 None, 2620 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2621 ) 2622 filename_extractor: Optional[str] = Field( 2623 None, 2624 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2625 examples=[ 2626 "{{ record.id }}/{{ record.file_name }}/", 2627 "{{ record.id }}_{{ record.file_name }}/", 2628 ], 2629 ) 2630 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2631 2632 2633class DeclarativeStream(BaseModel): 2634 class Config: 2635 extra = Extra.allow 2636 2637 type: Literal["DeclarativeStream"] 2638 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2639 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2640 ..., 2641 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2642 title="Retriever", 2643 ) 2644 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2645 None, 2646 description="Component used to fetch data incrementally based on a time field in the data.", 2647 title="Incremental Sync", 2648 ) 2649 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2650 schema_loader: Optional[ 2651 Union[ 2652 InlineSchemaLoader, 2653 DynamicSchemaLoader, 2654 JsonFileSchemaLoader, 2655 List[ 2656 Union[ 2657 InlineSchemaLoader, 2658 DynamicSchemaLoader, 2659 JsonFileSchemaLoader, 2660 CustomSchemaLoader, 2661 ] 2662 ], 2663 CustomSchemaLoader, 2664 ] 2665 ] = Field( 2666 None, 2667 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2668 title="Schema Loader", 2669 ) 2670 transformations: Optional[ 2671 List[ 2672 Union[ 2673 AddFields, 2674 RemoveFields, 2675 KeysToLower, 2676 KeysToSnakeCase, 2677 FlattenFields, 2678 DpathFlattenFields, 2679 KeysReplace, 2680 CustomTransformation, 2681 ] 2682 ] 2683 ] = Field( 2684 None, 2685 description="A list of transformations to be applied to each output record.", 2686 title="Transformations", 2687 ) 2688 state_migrations: Optional[ 2689 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2690 ] = Field( 2691 [], 2692 description="Array of state migrations to be applied on the input state", 2693 title="State Migrations", 2694 ) 2695 file_uploader: Optional[FileUploader] = Field( 2696 None, 2697 description="(experimental) Describes how to fetch a file", 2698 title="File Uploader", 2699 ) 2700 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2701 2702 2703class SessionTokenAuthenticator(BaseModel): 2704 type: Literal["SessionTokenAuthenticator"] 2705 login_requester: HttpRequester = Field( 2706 ..., 2707 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2708 examples=[ 2709 { 2710 "type": "HttpRequester", 2711 "url_base": "https://my_api.com", 2712 "path": "/login", 2713 "authenticator": { 2714 "type": "BasicHttpAuthenticator", 2715 "username": "{{ config.username }}", 2716 "password": "{{ config.password }}", 2717 }, 2718 } 2719 ], 2720 title="Login Requester", 2721 ) 2722 session_token_path: List[str] = Field( 2723 ..., 2724 description="The path in the response body returned from the login requester to the session token.", 2725 examples=[["access_token"], ["result", "token"]], 2726 title="Session Token Path", 2727 ) 2728 expiration_duration: Optional[str] = Field( 2729 None, 2730 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2731 examples=["PT1H", "P1D"], 2732 title="Expiration Duration", 2733 ) 2734 request_authentication: Union[ 2735 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2736 ] = Field( 2737 ..., 2738 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2739 title="Data Request Authentication", 2740 ) 2741 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2742 None, description="Component used to decode the response.", title="Decoder" 2743 ) 2744 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2745 2746 2747class HttpRequester(BaseModelWithDeprecations): 2748 type: Literal["HttpRequester"] 2749 url_base: Optional[str] = Field( 2750 None, 2751 deprecated=True, 2752 deprecation_message="Use `url` field instead.", 2753 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2754 examples=[ 2755 "https://connect.squareup.com/v2", 2756 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2757 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2758 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2759 ], 2760 title="API Base URL", 2761 ) 2762 url: Optional[str] = Field( 2763 None, 2764 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2765 examples=[ 2766 "https://connect.squareup.com/v2", 2767 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2768 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2769 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2770 ], 2771 title="API Endpoint URL", 2772 ) 2773 path: Optional[str] = Field( 2774 None, 2775 deprecated=True, 2776 deprecation_message="Use `url` field instead.", 2777 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2778 examples=[ 2779 "/products", 2780 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2781 "/trades/{{ config['symbol_id'] }}/history", 2782 ], 2783 title="URL Path", 2784 ) 2785 http_method: Optional[HttpMethod] = Field( 2786 HttpMethod.GET, 2787 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2788 examples=["GET", "POST"], 2789 title="HTTP Method", 2790 ) 2791 authenticator: Optional[ 2792 Union[ 2793 ApiKeyAuthenticator, 2794 BasicHttpAuthenticator, 2795 BearerAuthenticator, 2796 OAuthAuthenticator, 2797 JwtAuthenticator, 2798 SessionTokenAuthenticator, 2799 SelectiveAuthenticator, 2800 CustomAuthenticator, 2801 NoAuth, 2802 LegacySessionTokenAuthenticator, 2803 ] 2804 ] = Field( 2805 None, 2806 description="Authentication method to use for requests sent to the API.", 2807 title="Authenticator", 2808 ) 2809 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2810 None, 2811 deprecated=True, 2812 deprecation_message="Use `query_properties` field instead.", 2813 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2814 title="Fetch Properties from Endpoint", 2815 ) 2816 query_properties: Optional[QueryProperties] = Field( 2817 None, 2818 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2819 title="Query Properties", 2820 ) 2821 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2822 None, 2823 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2824 examples=[ 2825 {"unit": "day"}, 2826 { 2827 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2828 }, 2829 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2830 {"sort_by[asc]": "updated_at"}, 2831 ], 2832 title="Query Parameters", 2833 ) 2834 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2835 None, 2836 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2837 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2838 title="Request Headers", 2839 ) 2840 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2841 None, 2842 deprecated=True, 2843 deprecation_message="Use `request_body` field instead.", 2844 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2845 examples=[ 2846 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2847 ], 2848 title="Request Body Payload (Non-JSON)", 2849 ) 2850 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2851 None, 2852 deprecated=True, 2853 deprecation_message="Use `request_body` field instead.", 2854 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2855 examples=[ 2856 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2857 {"key": "{{ config['value'] }}"}, 2858 {"sort": {"field": "updated_at", "order": "ascending"}}, 2859 ], 2860 title="Request Body JSON Payload", 2861 ) 2862 request_body: Optional[ 2863 Union[ 2864 RequestBodyPlainText, 2865 RequestBodyUrlEncodedForm, 2866 RequestBodyJsonObject, 2867 RequestBodyGraphQL, 2868 ] 2869 ] = Field( 2870 None, 2871 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2872 title="Request Body", 2873 ) 2874 error_handler: Optional[ 2875 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2876 ] = Field( 2877 None, 2878 description="Error handler component that defines how to handle errors.", 2879 title="Error Handler", 2880 ) 2881 use_cache: Optional[bool] = Field( 2882 False, 2883 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2884 title="Use Cache", 2885 ) 2886 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2887 2888 2889class DynamicSchemaLoader(BaseModel): 2890 type: Literal["DynamicSchemaLoader"] 2891 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2892 ..., 2893 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2894 title="Retriever", 2895 ) 2896 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2897 None, 2898 description="Responsible for filtering fields to be added to json schema.", 2899 title="Schema Filter", 2900 ) 2901 schema_transformations: Optional[ 2902 List[ 2903 Union[ 2904 AddFields, 2905 RemoveFields, 2906 KeysToLower, 2907 KeysToSnakeCase, 2908 FlattenFields, 2909 DpathFlattenFields, 2910 KeysReplace, 2911 CustomTransformation, 2912 ] 2913 ] 2914 ] = Field( 2915 None, 2916 description="A list of transformations to be applied to the schema.", 2917 title="Schema Transformations", 2918 ) 2919 schema_type_identifier: SchemaTypeIdentifier 2920 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2921 2922 2923class ParentStreamConfig(BaseModel): 2924 type: Literal["ParentStreamConfig"] 2925 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2926 ..., description="Reference to the parent stream.", title="Parent Stream" 2927 ) 2928 parent_key: str = Field( 2929 ..., 2930 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2931 examples=["id", "{{ config['parent_record_id'] }}"], 2932 title="Parent Key", 2933 ) 2934 partition_field: str = Field( 2935 ..., 2936 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2937 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2938 title="Current Parent Key Value Identifier", 2939 ) 2940 request_option: Optional[RequestOption] = Field( 2941 None, 2942 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2943 title="Request Option", 2944 ) 2945 incremental_dependency: Optional[bool] = Field( 2946 False, 2947 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2948 title="Incremental Dependency", 2949 ) 2950 lazy_read_pointer: Optional[List[str]] = Field( 2951 [], 2952 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2953 title="Lazy Read Pointer", 2954 ) 2955 extra_fields: Optional[List[List[str]]] = Field( 2956 None, 2957 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2958 title="Extra Fields", 2959 ) 2960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2961 2962 2963class PropertiesFromEndpoint(BaseModel): 2964 type: Literal["PropertiesFromEndpoint"] 2965 property_field_path: List[str] = Field( 2966 ..., 2967 description="Describes the path to the field that should be extracted", 2968 examples=[["name"]], 2969 ) 2970 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2971 ..., 2972 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2973 ) 2974 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2975 2976 2977class QueryProperties(BaseModel): 2978 type: Literal["QueryProperties"] 2979 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2980 ..., 2981 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2982 title="Property List", 2983 ) 2984 always_include_properties: Optional[List[str]] = Field( 2985 None, 2986 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2987 title="Always Include Properties", 2988 ) 2989 property_chunking: Optional[PropertyChunking] = Field( 2990 None, 2991 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2992 title="Property Chunking", 2993 ) 2994 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2995 None, 2996 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2997 title="Property Selector", 2998 ) 2999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3000 3001 3002class StateDelegatingStream(BaseModel): 3003 type: Literal["StateDelegatingStream"] 3004 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3005 full_refresh_stream: DeclarativeStream = Field( 3006 ..., 3007 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3008 title="Full Refresh Stream", 3009 ) 3010 incremental_stream: DeclarativeStream = Field( 3011 ..., 3012 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3013 title="Incremental Stream", 3014 ) 3015 api_retention_period: Optional[str] = Field( 3016 None, 3017 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3018 examples=["P30D", "P90D", "P1Y"], 3019 title="API Retention Period", 3020 ) 3021 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3022 3023 3024class SimpleRetriever(BaseModel): 3025 type: Literal["SimpleRetriever"] 3026 requester: Union[HttpRequester, CustomRequester] = Field( 3027 ..., 3028 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3029 ) 3030 decoder: Optional[ 3031 Union[ 3032 JsonDecoder, 3033 JsonItemsDecoder, 3034 XmlDecoder, 3035 CsvDecoder, 3036 JsonlDecoder, 3037 GzipDecoder, 3038 IterableDecoder, 3039 ZipfileDecoder, 3040 CustomDecoder, 3041 ] 3042 ] = Field( 3043 None, 3044 description="Component decoding the response so records can be extracted.", 3045 title="HTTP Response Format", 3046 ) 3047 record_selector: RecordSelector = Field( 3048 ..., 3049 description="Component that describes how to extract records from a HTTP response.", 3050 ) 3051 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3052 None, 3053 description="Paginator component that describes how to navigate through the API's pages.", 3054 ) 3055 pagination_reset: Optional[PaginationReset] = Field( 3056 None, 3057 description="Describes what triggers pagination reset and how to handle it.", 3058 ) 3059 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3060 False, 3061 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3062 ) 3063 partition_router: Optional[ 3064 Union[ 3065 SubstreamPartitionRouter, 3066 ListPartitionRouter, 3067 GroupingPartitionRouter, 3068 CustomPartitionRouter, 3069 List[ 3070 Union[ 3071 SubstreamPartitionRouter, 3072 ListPartitionRouter, 3073 GroupingPartitionRouter, 3074 CustomPartitionRouter, 3075 ] 3076 ], 3077 ] 3078 ] = Field( 3079 None, 3080 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3081 title="Partition Router", 3082 ) 3083 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3084 3085 3086class AsyncRetriever(BaseModel): 3087 type: Literal["AsyncRetriever"] 3088 record_selector: RecordSelector = Field( 3089 ..., 3090 description="Component that describes how to extract records from a HTTP response.", 3091 ) 3092 status_mapping: AsyncJobStatusMap = Field( 3093 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3094 ) 3095 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3096 ..., description="Responsible for fetching the actual status of the async job." 3097 ) 3098 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3099 None, 3100 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3101 ) 3102 download_extractor: Optional[ 3103 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3104 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3105 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3106 ..., 3107 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3108 ) 3109 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3110 ..., 3111 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3112 ) 3113 polling_job_timeout: Optional[Union[int, str]] = Field( 3114 None, 3115 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3116 ) 3117 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3118 None, 3119 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3120 ge=1, 3121 ) 3122 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3123 None, 3124 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3125 ) 3126 download_requester: Union[HttpRequester, CustomRequester] = Field( 3127 ..., 3128 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3129 ) 3130 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3131 None, 3132 description="Paginator component that describes how to navigate through the API's pages during download.", 3133 ) 3134 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3135 None, 3136 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3137 ) 3138 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3139 None, 3140 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3141 ) 3142 partition_router: Optional[ 3143 Union[ 3144 ListPartitionRouter, 3145 SubstreamPartitionRouter, 3146 GroupingPartitionRouter, 3147 CustomPartitionRouter, 3148 List[ 3149 Union[ 3150 ListPartitionRouter, 3151 SubstreamPartitionRouter, 3152 GroupingPartitionRouter, 3153 CustomPartitionRouter, 3154 ] 3155 ], 3156 ] 3157 ] = Field( 3158 [], 3159 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3160 title="Partition Router", 3161 ) 3162 decoder: Optional[ 3163 Union[ 3164 CsvDecoder, 3165 GzipDecoder, 3166 JsonDecoder, 3167 JsonItemsDecoder, 3168 JsonlDecoder, 3169 IterableDecoder, 3170 XmlDecoder, 3171 ZipfileDecoder, 3172 CustomDecoder, 3173 ] 3174 ] = Field( 3175 None, 3176 description="Component decoding the response so records can be extracted.", 3177 title="HTTP Response Format", 3178 ) 3179 download_decoder: Optional[ 3180 Union[ 3181 CsvDecoder, 3182 GzipDecoder, 3183 JsonDecoder, 3184 JsonItemsDecoder, 3185 JsonlDecoder, 3186 IterableDecoder, 3187 XmlDecoder, 3188 ZipfileDecoder, 3189 CustomDecoder, 3190 ] 3191 ] = Field( 3192 None, 3193 description="Component decoding the download response so records can be extracted.", 3194 title="Download HTTP Response Format", 3195 ) 3196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3197 3198 3199class BlockSimultaneousSyncsAction(BaseModel): 3200 type: Literal["BlockSimultaneousSyncsAction"] 3201 3202 3203class StreamGroup(BaseModel): 3204 streams: List[str] = Field( 3205 ..., 3206 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3207 title="Streams", 3208 ) 3209 action: BlockSimultaneousSyncsAction = Field( 3210 ..., 3211 description="The action to apply to streams in this group.", 3212 title="Action", 3213 ) 3214 3215 3216class SubstreamPartitionRouter(BaseModel): 3217 type: Literal["SubstreamPartitionRouter"] 3218 parent_stream_configs: List[ParentStreamConfig] = Field( 3219 ..., 3220 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3221 title="Parent Stream Configs", 3222 ) 3223 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3224 3225 3226class GroupingPartitionRouter(BaseModel): 3227 type: Literal["GroupingPartitionRouter"] 3228 group_size: int = Field( 3229 ..., 3230 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3231 examples=[10, 50], 3232 title="Group Size", 3233 ) 3234 underlying_partition_router: Union[ 3235 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3236 ] = Field( 3237 ..., 3238 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3239 title="Underlying Partition Router", 3240 ) 3241 deduplicate: Optional[bool] = Field( 3242 True, 3243 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3244 title="Deduplicate Partitions", 3245 ) 3246 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3247 3248 3249class HttpComponentsResolver(BaseModel): 3250 type: Literal["HttpComponentsResolver"] 3251 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3252 ..., 3253 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3254 title="Retriever", 3255 ) 3256 components_mapping: List[ComponentMappingDefinition] 3257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3258 3259 3260class DynamicDeclarativeStream(BaseModel): 3261 type: Literal["DynamicDeclarativeStream"] 3262 name: Optional[str] = Field( 3263 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3264 ) 3265 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3266 ..., description="Reference to the stream template.", title="Stream Template" 3267 ) 3268 components_resolver: Union[ 3269 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3270 ] = Field( 3271 ..., 3272 description="Component resolve and populates stream templates with components values.", 3273 title="Components Resolver", 3274 ) 3275 use_parent_parameters: Optional[bool] = Field( 3276 True, 3277 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3278 title="Use Parent Parameters", 3279 ) 3280 3281 3282ComplexFieldType.update_forward_refs() 3283GzipDecoder.update_forward_refs() 3284CompositeErrorHandler.update_forward_refs() 3285DeclarativeSource1.update_forward_refs() 3286DeclarativeSource2.update_forward_refs() 3287SelectiveAuthenticator.update_forward_refs() 3288ConditionalStreams.update_forward_refs() 3289FileUploader.update_forward_refs() 3290DeclarativeStream.update_forward_refs() 3291SessionTokenAuthenticator.update_forward_refs() 3292HttpRequester.update_forward_refs() 3293DynamicSchemaLoader.update_forward_refs() 3294ParentStreamConfig.update_forward_refs() 3295PropertiesFromEndpoint.update_forward_refs() 3296SimpleRetriever.update_forward_refs() 3297AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 jitter_range_in_seconds: Optional[float] = Field( 109 None, 110 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 111 examples=[15], 112 ge=0, 113 title="Jitter Range", 114 ) 115 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
118class CursorPagination(BaseModel): 119 type: Literal["CursorPagination"] 120 cursor_value: str = Field( 121 ..., 122 description="Value of the cursor defining the next page to fetch.", 123 examples=[ 124 "{{ headers.link.next.cursor }}", 125 "{{ last_record['key'] }}", 126 "{{ response['nextPage'] }}", 127 ], 128 title="Cursor Value", 129 ) 130 page_size: Optional[Union[int, str]] = Field( 131 None, 132 description="The number of records to include in each pages.", 133 examples=[100, "{{ config['page_size'] }}"], 134 title="Page Size", 135 ) 136 stop_condition: Optional[str] = Field( 137 None, 138 description="Template string evaluating when to stop paginating.", 139 examples=[ 140 "{{ response.data.has_more is false }}", 141 "{{ 'next' not in headers['link'] }}", 142 ], 143 title="Stop Condition", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
148class CustomAuthenticator(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomAuthenticator"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
162class CustomBackoffStrategy(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomBackoffStrategy"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomBackoffStrategy"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
176class CustomErrorHandler(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomErrorHandler"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomErrorHandler"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
190class CustomPaginationStrategy(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomPaginationStrategy"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomPaginationStrategy"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
204class CustomRecordExtractor(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordExtractor"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomRecordExtractor"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
218class CustomRecordFilter(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRecordFilter"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomCustomRecordFilter"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
232class CustomRequester(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRequester"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRecordExtractor"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
246class CustomRetriever(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomRetriever"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomRetriever"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
260class CustomPartitionRouter(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomPartitionRouter"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomPartitionRouter"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
274class CustomSchemaLoader(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaLoader"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 282 examples=["source_railz.components.MyCustomSchemaLoader"], 283 title="Class Name", 284 ) 285 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
288class CustomSchemaNormalization(BaseModel): 289 class Config: 290 extra = Extra.allow 291 292 type: Literal["CustomSchemaNormalization"] 293 class_name: str = Field( 294 ..., 295 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 296 examples=[ 297 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 298 ], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
304class CustomStateMigration(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomStateMigration"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomStateMigration"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
318class CustomTransformation(BaseModel): 319 class Config: 320 extra = Extra.allow 321 322 type: Literal["CustomTransformation"] 323 class_name: str = Field( 324 ..., 325 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 326 examples=["source_railz.components.MyCustomTransformation"], 327 title="Class Name", 328 ) 329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
349class Algorithm(Enum): 350 HS256 = "HS256" 351 HS384 = "HS384" 352 HS512 = "HS512" 353 ES256 = "ES256" 354 ES256K = "ES256K" 355 ES384 = "ES384" 356 ES512 = "ES512" 357 RS256 = "RS256" 358 RS384 = "RS384" 359 RS512 = "RS512" 360 PS256 = "PS256" 361 PS384 = "PS384" 362 PS512 = "PS512" 363 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
366class JwtHeaders(BaseModel): 367 class Config: 368 extra = Extra.forbid 369 370 kid: Optional[str] = Field( 371 None, 372 description="Private key ID for user account.", 373 examples=["{{ config['kid'] }}"], 374 title="Key Identifier", 375 ) 376 typ: Optional[str] = Field( 377 "JWT", 378 description="The media type of the complete JWT.", 379 examples=["JWT"], 380 title="Type", 381 ) 382 cty: Optional[str] = Field( 383 None, 384 description="Content type of JWT header.", 385 examples=["JWT"], 386 title="Content Type", 387 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
390class JwtPayload(BaseModel): 391 class Config: 392 extra = Extra.forbid 393 394 iss: Optional[str] = Field( 395 None, 396 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 397 examples=["{{ config['iss'] }}"], 398 title="Issuer", 399 ) 400 sub: Optional[str] = Field( 401 None, 402 description="The subject of the JWT. Commonly defined by the API.", 403 title="Subject", 404 ) 405 aud: Optional[str] = Field( 406 None, 407 description="The recipient that the JWT is intended for. Commonly defined by the API.", 408 examples=["appstoreconnect-v1"], 409 title="Audience", 410 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
413class RefreshTokenUpdater(BaseModel): 414 refresh_token_name: Optional[str] = Field( 415 "refresh_token", 416 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 417 examples=["refresh_token"], 418 title="Refresh Token Property Name", 419 ) 420 access_token_config_path: Optional[List[str]] = Field( 421 ["credentials", "access_token"], 422 description="Config path to the access token. Make sure the field actually exists in the config.", 423 examples=[["credentials", "access_token"], ["access_token"]], 424 title="Config Path To Access Token", 425 ) 426 refresh_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "refresh_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "refresh_token"], ["refresh_token"]], 430 title="Config Path To Refresh Token", 431 ) 432 token_expiry_date_config_path: Optional[List[str]] = Field( 433 ["credentials", "token_expiry_date"], 434 description="Config path to the expiry date. Make sure actually exists in the config.", 435 examples=[["credentials", "token_expiry_date"]], 436 title="Config Path To Expiry Date", 437 ) 438 refresh_token_error_status_codes: Optional[List[int]] = Field( 439 [], 440 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 441 examples=[[400, 500]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 443 ) 444 refresh_token_error_key: Optional[str] = Field( 445 "", 446 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 447 examples=["error"], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 449 ) 450 refresh_token_error_values: Optional[List[str]] = Field( 451 [], 452 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 453 examples=[["invalid_grant", "invalid_permissions"]], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 455 )
class
Rate(pydantic.v1.main.BaseModel):
458class Rate(BaseModel): 459 class Config: 460 extra = Extra.allow 461 462 limit: Union[int, str] = Field( 463 ..., 464 description="The maximum number of calls allowed within the interval.", 465 title="Limit", 466 ) 467 interval: str = Field( 468 ..., 469 description="The time interval for the rate limit.", 470 examples=["PT1H", "P1D"], 471 title="Interval", 472 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
475class HttpRequestRegexMatcher(BaseModel): 476 class Config: 477 extra = Extra.allow 478 479 method: Optional[str] = Field( 480 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 481 ) 482 url_base: Optional[str] = Field( 483 None, 484 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 485 title="URL Base", 486 ) 487 url_path_pattern: Optional[str] = Field( 488 None, 489 description="A regular expression pattern to match the URL path.", 490 title="URL Path Pattern", 491 ) 492 params: Optional[Dict[str, Any]] = Field( 493 None, description="The query parameters to match.", title="Parameters" 494 ) 495 headers: Optional[Dict[str, Any]] = Field( 496 None, description="The headers to match.", title="Headers" 497 ) 498 weight: Optional[Union[int, str]] = Field( 499 None, 500 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 501 title="Weight", 502 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
505class ResponseToFileExtractor(BaseModel): 506 type: Literal["ResponseToFileExtractor"] 507 preserve_na_values: Optional[bool] = Field( 508 False, 509 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 510 title="Preserve NA Values", 511 ) 512 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
520class ExponentialBackoffStrategy(BaseModel): 521 type: Literal["ExponentialBackoffStrategy"] 522 factor: Optional[Union[float, str]] = Field( 523 5, 524 description="Multiplicative constant applied on each retry.", 525 examples=[5, 5.5, "10"], 526 title="Factor", 527 ) 528 jitter_range_in_seconds: Optional[float] = Field( 529 None, 530 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 531 examples=[2], 532 ge=0, 533 title="Jitter Range", 534 ) 535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
538class GroupByKeyMergeStrategy(BaseModel): 539 type: Literal["GroupByKeyMergeStrategy"] 540 key: Union[str, List[str]] = Field( 541 ..., 542 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 543 examples=["id", ["parent_id", "end_date"]], 544 title="Key", 545 ) 546 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
558class Action(Enum): 559 SUCCESS = "SUCCESS" 560 FAIL = "FAIL" 561 RETRY = "RETRY" 562 IGNORE = "IGNORE" 563 RESET_PAGINATION = "RESET_PAGINATION" 564 RATE_LIMITED = "RATE_LIMITED" 565 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
568class FailureType(Enum): 569 system_error = "system_error" 570 config_error = "config_error" 571 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
574class HttpResponseFilter(BaseModel): 575 type: Literal["HttpResponseFilter"] 576 action: Optional[Action] = Field( 577 None, 578 description="Action to execute if a response matches the filter.", 579 examples=[ 580 "SUCCESS", 581 "FAIL", 582 "RETRY", 583 "IGNORE", 584 "RESET_PAGINATION", 585 "RATE_LIMITED", 586 "REFRESH_TOKEN_THEN_RETRY", 587 ], 588 title="Action", 589 ) 590 failure_type: Optional[FailureType] = Field( 591 None, 592 description="Failure type of traced exception if a response matches the filter.", 593 examples=["system_error", "config_error", "transient_error"], 594 title="Failure Type", 595 ) 596 error_message: Optional[str] = Field( 597 None, 598 description="Error Message to display if the response matches the filter.", 599 title="Error Message", 600 ) 601 error_message_contains: Optional[str] = Field( 602 None, 603 description="Match the response if its error message contains the substring.", 604 example=["This API operation is not enabled for this site"], 605 title="Error Message Substring", 606 ) 607 http_codes: Optional[List[int]] = Field( 608 None, 609 description="Match the response if its HTTP code is included in this list.", 610 examples=[[420, 429], [500]], 611 title="HTTP Codes", 612 unique_items=True, 613 ) 614 predicate: Optional[str] = Field( 615 None, 616 description="Match the response if the predicate evaluates to true.", 617 examples=[ 618 "{{ 'Too much requests' in response }}", 619 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 620 ], 621 title="Predicate", 622 ) 623 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
626class ComplexFieldType(BaseModel): 627 field_type: str 628 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
631class TypesMap(BaseModel): 632 target_type: Union[str, List[str], ComplexFieldType] 633 current_type: Union[str, List[str]] 634 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
637class SchemaTypeIdentifier(BaseModel): 638 type: Optional[Literal["SchemaTypeIdentifier"]] = None 639 schema_pointer: Optional[List[str]] = Field( 640 [], 641 description="List of nested fields defining the schema field path to extract. Defaults to [].", 642 title="Schema Path", 643 ) 644 key_pointer: List[str] = Field( 645 ..., 646 description="List of potentially nested fields describing the full path of the field key to extract.", 647 title="Key Path", 648 ) 649 type_pointer: Optional[List[str]] = Field( 650 None, 651 description="List of potentially nested fields describing the full path of the field type to extract.", 652 title="Type Path", 653 ) 654 types_mapping: Optional[List[TypesMap]] = None 655 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
658class InlineSchemaLoader(BaseModel): 659 type: Literal["InlineSchemaLoader"] 660 schema_: Optional[Dict[str, Any]] = Field( 661 None, 662 alias="schema", 663 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 664 title="Schema", 665 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
668class JsonFileSchemaLoader(BaseModel): 669 type: Literal["JsonFileSchemaLoader"] 670 file_path: Optional[str] = Field( 671 None, 672 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 673 example=["./schemas/users.json"], 674 title="File Path", 675 ) 676 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
683class JsonItemsDecoder(BaseModel): 684 type: Literal["JsonItemsDecoder"] 685 items_path: str = Field( 686 ..., 687 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 688 title="Items Path", 689 ) 690 encoding: Optional[str] = Field( 691 "utf-8", 692 description="The character encoding of the JSON data. Defaults to UTF-8.", 693 title="Encoding", 694 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
711class FlattenFields(BaseModel): 712 type: Literal["FlattenFields"] 713 flatten_lists: Optional[bool] = Field( 714 True, 715 description="Whether to flatten lists or leave it as is. Default is True.", 716 title="Flatten Lists", 717 ) 718 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
721class KeyTransformation(BaseModel): 722 type: Literal["KeyTransformation"] 723 prefix: Optional[str] = Field( 724 None, 725 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 726 examples=["flattened_"], 727 title="Key Prefix", 728 ) 729 suffix: Optional[str] = Field( 730 None, 731 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 732 examples=["_flattened"], 733 title="Key Suffix", 734 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
737class DpathFlattenFields(BaseModel): 738 type: Literal["DpathFlattenFields"] 739 field_path: List[str] = Field( 740 ..., 741 description="A path to field that needs to be flattened.", 742 examples=[["data"], ["data", "*", "field"]], 743 title="Field Path", 744 ) 745 delete_origin_value: Optional[bool] = Field( 746 None, 747 description="Whether to delete the origin value or keep it. Default is False.", 748 title="Delete Origin Value", 749 ) 750 replace_record: Optional[bool] = Field( 751 None, 752 description="Whether to replace the origin record or not. Default is False.", 753 title="Replace Origin Record", 754 ) 755 key_transformation: Optional[KeyTransformation] = Field( 756 None, 757 description="Transformation for object keys. If not provided, original key will be used.", 758 title="Key transformation", 759 ) 760 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
763class KeysReplace(BaseModel): 764 type: Literal["KeysReplace"] 765 old: str = Field( 766 ..., 767 description="Old value to replace.", 768 examples=[ 769 " ", 770 "{{ record.id }}", 771 "{{ config['id'] }}", 772 "{{ stream_slice['id'] }}", 773 ], 774 title="Old value", 775 ) 776 new: str = Field( 777 ..., 778 description="New value to set.", 779 examples=[ 780 "_", 781 "{{ record.id }}", 782 "{{ config['id'] }}", 783 "{{ stream_slice['id'] }}", 784 ], 785 title="New value", 786 ) 787 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
798class CustomDecoder(BaseModel): 799 class Config: 800 extra = Extra.allow 801 802 type: Literal["CustomDecoder"] 803 class_name: str = Field( 804 ..., 805 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 806 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 807 title="Class Name", 808 ) 809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
812class MinMaxDatetime(BaseModel): 813 type: Literal["MinMaxDatetime"] 814 datetime: str = Field( 815 ..., 816 description="Datetime value.", 817 examples=[ 818 "2021-01-01", 819 "2021-01-01T00:00:00Z", 820 "{{ config['start_time'] }}", 821 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 822 ], 823 title="Datetime", 824 ) 825 datetime_format: Optional[str] = Field( 826 "", 827 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 828 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 829 title="Datetime Format", 830 ) 831 max_datetime: Optional[str] = Field( 832 None, 833 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 834 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 835 title="Max Datetime", 836 ) 837 min_datetime: Optional[str] = Field( 838 None, 839 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 840 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 841 title="Min Datetime", 842 ) 843 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
873class OauthConnectorInputSpecification(BaseModel): 874 class Config: 875 extra = Extra.allow 876 877 consent_url: str = Field( 878 ..., 879 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 880 examples=[ 881 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 882 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 883 ], 884 title="Consent URL", 885 ) 886 scope: Optional[str] = Field( 887 None, 888 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 889 examples=["user:read user:read_orders workspaces:read"], 890 title="Scopes", 891 ) 892 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 893 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 894 # The CDK schema defines the manifest contract; the platform reads these fields 895 # during the OAuth consent flow to build the authorization URL. 896 scopes: Optional[List[OAuthScope]] = Field( 897 None, 898 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 899 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 900 title="Scopes", 901 ) 902 optional_scopes: Optional[List[OAuthScope]] = Field( 903 None, 904 description="Optional OAuth scope objects that may or may not be granted.", 905 examples=[[{"scope": "admin:read"}]], 906 title="Optional Scopes", 907 ) 908 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 909 ScopesJoinStrategy.space, 910 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 911 title="Scopes Join Strategy", 912 ) 913 access_token_url: str = Field( 914 ..., 915 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 916 examples=[ 917 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 918 ], 919 title="Access Token URL", 920 ) 921 access_token_headers: Optional[Dict[str, Any]] = Field( 922 None, 923 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 924 examples=[ 925 { 926 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 927 } 928 ], 929 title="Access Token Headers", 930 ) 931 access_token_params: Optional[Dict[str, Any]] = Field( 932 None, 933 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 934 examples=[ 935 { 936 "{{ auth_code_key }}": "{{ auth_code_value }}", 937 "{{ client_id_key }}": "{{ client_id_value }}", 938 "{{ client_secret_key }}": "{{ client_secret_value }}", 939 } 940 ], 941 title="Access Token Query Params (Json Encoded)", 942 ) 943 extract_output: Optional[List[str]] = Field( 944 None, 945 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 946 examples=[["access_token", "refresh_token", "other_field"]], 947 title="Extract Output", 948 ) 949 state: Optional[State] = Field( 950 None, 951 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 952 examples=[{"min": 7, "max": 128}], 953 title="Configurable State Query Param", 954 ) 955 client_id_key: Optional[str] = Field( 956 None, 957 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 958 examples=["my_custom_client_id_key_name"], 959 title="Client ID Key Override", 960 ) 961 client_secret_key: Optional[str] = Field( 962 None, 963 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 964 examples=["my_custom_client_secret_key_name"], 965 title="Client Secret Key Override", 966 ) 967 scope_key: Optional[str] = Field( 968 None, 969 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 970 examples=["my_custom_scope_key_key_name"], 971 title="Scopes Key Override", 972 ) 973 state_key: Optional[str] = Field( 974 None, 975 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 976 examples=["my_custom_state_key_key_name"], 977 title="State Key Override", 978 ) 979 auth_code_key: Optional[str] = Field( 980 None, 981 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 982 examples=["my_custom_auth_code_key_name"], 983 title="Auth Code Key Override", 984 ) 985 redirect_uri_key: Optional[str] = Field( 986 None, 987 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 988 examples=["my_custom_redirect_uri_key_name"], 989 title="Redirect URI Key Override", 990 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
993class OAuthConfigSpecification(BaseModel): 994 class Config: 995 extra = Extra.allow 996 997 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 998 None, 999 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1000 examples=[ 1001 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1002 { 1003 "app_id": { 1004 "type": "string", 1005 "path_in_connector_config": ["info", "app_id"], 1006 } 1007 }, 1008 ], 1009 title="OAuth user input", 1010 ) 1011 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1012 None, 1013 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1014 title="DeclarativeOAuth Connector Specification", 1015 ) 1016 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1017 None, 1018 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1019 examples=[ 1020 { 1021 "refresh_token": { 1022 "type": "string,", 1023 "path_in_connector_config": ["credentials", "refresh_token"], 1024 } 1025 } 1026 ], 1027 title="OAuth output specification", 1028 ) 1029 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1030 None, 1031 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1032 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1033 title="OAuth input specification", 1034 ) 1035 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1036 None, 1037 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1038 examples=[ 1039 { 1040 "client_id": { 1041 "type": "string,", 1042 "path_in_connector_config": ["credentials", "client_id"], 1043 }, 1044 "client_secret": { 1045 "type": "string,", 1046 "path_in_connector_config": ["credentials", "client_secret"], 1047 }, 1048 } 1049 ], 1050 title="OAuth server output specification", 1051 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1054class OffsetIncrement(BaseModel): 1055 type: Literal["OffsetIncrement"] 1056 page_size: Optional[Union[int, str]] = Field( 1057 None, 1058 description="The number of records to include in each pages.", 1059 examples=[100, "{{ config['page_size'] }}"], 1060 title="Limit", 1061 ) 1062 inject_on_first_request: Optional[bool] = Field( 1063 False, 1064 description="Using the `offset` with value `0` during the first request", 1065 title="Inject Offset on First Request", 1066 ) 1067 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1070class PageIncrement(BaseModel): 1071 type: Literal["PageIncrement"] 1072 page_size: Optional[Union[int, str]] = Field( 1073 None, 1074 description="The number of records to include in each pages.", 1075 examples=[100, "100", "{{ config['page_size'] }}"], 1076 title="Page Size", 1077 ) 1078 start_from_page: Optional[int] = Field( 1079 0, 1080 description="Index of the first page to request.", 1081 examples=[0, 1], 1082 title="Start From Page", 1083 ) 1084 inject_on_first_request: Optional[bool] = Field( 1085 False, 1086 description="Using the `page number` with value defined by `start_from_page` during the first request", 1087 title="Inject Page Number on First Request", 1088 ) 1089 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1092class PrimaryKey(BaseModel): 1093 __root__: Union[str, List[str], List[List[str]]] = Field( 1094 ..., 1095 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1096 examples=["id", ["code", "type"]], 1097 title="Primary Key", 1098 )
class
PropertyLimitType(enum.Enum):
1101class PropertyLimitType(Enum): 1102 characters = "characters" 1103 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1106class PropertyChunking(BaseModel): 1107 type: Literal["PropertyChunking"] 1108 property_limit_type: PropertyLimitType = Field( 1109 ..., 1110 description="The type used to determine the maximum number of properties per chunk", 1111 title="Property Limit Type", 1112 ) 1113 property_limit: Optional[int] = Field( 1114 None, 1115 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1116 title="Property Limit", 1117 ) 1118 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1119 None, 1120 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1121 title="Record Merge Strategy", 1122 ) 1123 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1126class RecordFilter(BaseModel): 1127 type: Literal["RecordFilter"] 1128 condition: Optional[str] = Field( 1129 "", 1130 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1131 examples=[ 1132 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1133 "{{ record.status in ['active', 'expired'] }}", 1134 ], 1135 title="Condition", 1136 ) 1137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1145class RemoveFields(BaseModel): 1146 type: Literal["RemoveFields"] 1147 condition: Optional[str] = Field( 1148 "", 1149 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1150 examples=[ 1151 "{{ property|string == '' }}", 1152 "{{ property is integer }}", 1153 "{{ property|length > 5 }}", 1154 "{{ property == 'some_string_to_match' }}", 1155 ], 1156 ) 1157 field_pointers: List[List[str]] = Field( 1158 ..., 1159 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1160 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1161 title="Field Paths", 1162 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1169class InjectInto(Enum): 1170 request_parameter = "request_parameter" 1171 header = "header" 1172 body_data = "body_data" 1173 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1176class RequestOption(BaseModel): 1177 type: Literal["RequestOption"] 1178 inject_into: InjectInto = Field( 1179 ..., 1180 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1181 examples=["request_parameter", "header", "body_data", "body_json"], 1182 title="Inject Into", 1183 ) 1184 field_name: Optional[str] = Field( 1185 None, 1186 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1187 examples=["segment_id"], 1188 title="Field Name", 1189 ) 1190 field_path: Optional[List[str]] = Field( 1191 None, 1192 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1193 examples=[["data", "viewer", "id"]], 1194 title="Field Path", 1195 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1205class LegacySessionTokenAuthenticator(BaseModel): 1206 type: Literal["LegacySessionTokenAuthenticator"] 1207 header: str = Field( 1208 ..., 1209 description="The name of the session token header that will be injected in the request", 1210 examples=["X-Session"], 1211 title="Session Request Header", 1212 ) 1213 login_url: str = Field( 1214 ..., 1215 description="Path of the login URL (do not include the base URL)", 1216 examples=["session"], 1217 title="Login Path", 1218 ) 1219 session_token: Optional[str] = Field( 1220 None, 1221 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1222 example=["{{ config['session_token'] }}"], 1223 title="Session Token", 1224 ) 1225 session_token_response_key: str = Field( 1226 ..., 1227 description="Name of the key of the session token to be extracted from the response", 1228 examples=["id"], 1229 title="Response Token Response Key", 1230 ) 1231 username: Optional[str] = Field( 1232 None, 1233 description="Username used to authenticate and obtain a session token", 1234 examples=[" {{ config['username'] }}"], 1235 title="Username", 1236 ) 1237 password: Optional[str] = Field( 1238 "", 1239 description="Password used to authenticate and obtain a session token", 1240 examples=["{{ config['password'] }}", ""], 1241 title="Password", 1242 ) 1243 validate_session_url: str = Field( 1244 ..., 1245 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1246 examples=["user/current"], 1247 title="Validate Session Path", 1248 ) 1249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1278class ValueType(Enum): 1279 string = "string" 1280 number = "number" 1281 integer = "integer" 1282 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1285class WaitTimeFromHeader(BaseModel): 1286 type: Literal["WaitTimeFromHeader"] 1287 header: str = Field( 1288 ..., 1289 description="The name of the response header defining how long to wait before retrying.", 1290 examples=["Retry-After"], 1291 title="Response Header Name", 1292 ) 1293 regex: Optional[str] = Field( 1294 None, 1295 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1296 examples=["([-+]?\\d+)"], 1297 title="Extraction Regex", 1298 ) 1299 max_waiting_time_in_seconds: Optional[float] = Field( 1300 None, 1301 description="Given the value extracted from the header is greater than this value, stop the stream.", 1302 examples=[3600], 1303 title="Max Waiting Time in Seconds", 1304 ) 1305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1308class WaitUntilTimeFromHeader(BaseModel): 1309 type: Literal["WaitUntilTimeFromHeader"] 1310 header: str = Field( 1311 ..., 1312 description="The name of the response header defining how long to wait before retrying.", 1313 examples=["wait_time"], 1314 title="Response Header", 1315 ) 1316 min_wait: Optional[Union[float, str]] = Field( 1317 None, 1318 description="Minimum time to wait before retrying.", 1319 examples=[10, "60"], 1320 title="Minimum Wait Time", 1321 ) 1322 regex: Optional[str] = Field( 1323 None, 1324 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1325 examples=["([-+]?\\d+)"], 1326 title="Extraction Regex", 1327 ) 1328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1331class ComponentMappingDefinition(BaseModel): 1332 type: Literal["ComponentMappingDefinition"] 1333 field_path: List[str] = Field( 1334 ..., 1335 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1336 examples=[ 1337 ["name"], 1338 ["retriever", "requester", "url"], 1339 ["retriever", "requester", "{{ components_values.field }}"], 1340 ["*", "**", "name"], 1341 ], 1342 title="Field Path", 1343 ) 1344 value: str = Field( 1345 ..., 1346 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1347 examples=[ 1348 "{{ components_values['updates'] }}", 1349 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1350 "{{ config['segment_id'] }}", 1351 "{{ stream_slice['parent_id'] }}", 1352 "{{ stream_slice['extra_fields']['name'] }}", 1353 ], 1354 title="Value", 1355 ) 1356 value_type: Optional[ValueType] = Field( 1357 None, 1358 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1359 title="Value Type", 1360 ) 1361 create_or_update: Optional[bool] = Field( 1362 False, 1363 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1364 title="Create or Update", 1365 ) 1366 condition: Optional[str] = Field( 1367 None, 1368 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1369 examples=[ 1370 "{{ components_values.get('cursor_field', None) }}", 1371 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1372 ], 1373 title="Condition", 1374 ) 1375 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1378class StreamConfig(BaseModel): 1379 type: Literal["StreamConfig"] 1380 configs_pointer: List[str] = Field( 1381 ..., 1382 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1383 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1384 title="Configs Pointer", 1385 ) 1386 default_values: Optional[List[Dict[str, Any]]] = Field( 1387 None, 1388 description="A list of default values, each matching the structure expected from the parsed component value.", 1389 title="Default Values", 1390 ) 1391 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1394class ConfigComponentsResolver(BaseModel): 1395 type: Literal["ConfigComponentsResolver"] 1396 stream_config: Union[List[StreamConfig], StreamConfig] 1397 components_mapping: List[ComponentMappingDefinition] 1398 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1401class StreamParametersDefinition(BaseModel): 1402 type: Literal["StreamParametersDefinition"] 1403 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1404 ..., 1405 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1406 examples=[ 1407 [ 1408 { 1409 "name": "test stream", 1410 "$parameters": {"entity": "test entity"}, 1411 "primary_key": "test key", 1412 } 1413 ] 1414 ], 1415 title="Stream Parameters", 1416 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1419class ParametrizedComponentsResolver(BaseModel): 1420 type: Literal["ParametrizedComponentsResolver"] 1421 stream_parameters: StreamParametersDefinition 1422 components_mapping: List[ComponentMappingDefinition] 1423 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1448class ValidateAdheresToSchema(BaseModel): 1449 type: Literal["ValidateAdheresToSchema"] 1450 base_schema: Union[str, Dict[str, Any]] = Field( 1451 ..., 1452 description="The base JSON schema against which the user-provided schema will be validated.", 1453 examples=[ 1454 "{{ config['report_validation_schema'] }}", 1455 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1456 { 1457 "$schema": "http://json-schema.org/draft-07/schema#", 1458 "title": "Person", 1459 "type": "object", 1460 "properties": { 1461 "name": {"type": "string", "description": "The person's name"}, 1462 "age": { 1463 "type": "integer", 1464 "minimum": 0, 1465 "description": "The person's age", 1466 }, 1467 }, 1468 "required": ["name", "age"], 1469 }, 1470 ], 1471 title="Base JSON Schema", 1472 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1475class CustomValidationStrategy(BaseModel): 1476 class Config: 1477 extra = Extra.allow 1478 1479 type: Literal["CustomValidationStrategy"] 1480 class_name: str = Field( 1481 ..., 1482 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1483 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1484 title="Class Name", 1485 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1488class ConfigRemapField(BaseModel): 1489 type: Literal["ConfigRemapField"] 1490 map: Union[Dict[str, Any], str] = Field( 1491 ..., 1492 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1493 examples=[ 1494 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1495 "{{ config['status_mapping'] }}", 1496 ], 1497 title="Value Mapping", 1498 ) 1499 field_path: List[str] = Field( 1500 ..., 1501 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1502 examples=[ 1503 ["status"], 1504 ["data", "status"], 1505 ["data", "{{ config.name }}", "status"], 1506 ["data", "*", "status"], 1507 ], 1508 title="Field Path", 1509 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1512class ConfigRemoveFields(BaseModel): 1513 type: Literal["ConfigRemoveFields"] 1514 field_pointers: List[List[str]] = Field( 1515 ..., 1516 description="A list of field pointers to be removed from the config.", 1517 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1518 title="Field Pointers", 1519 ) 1520 condition: Optional[str] = Field( 1521 "", 1522 description="Fields will be removed if expression is evaluated to True.", 1523 examples=[ 1524 "{{ config['environemnt'] == 'sandbox' }}", 1525 "{{ property is integer }}", 1526 "{{ property|length > 5 }}", 1527 "{{ property == 'some_string_to_match' }}", 1528 ], 1529 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1532class CustomConfigTransformation(BaseModel): 1533 type: Literal["CustomConfigTransformation"] 1534 class_name: str = Field( 1535 ..., 1536 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1537 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1538 ) 1539 parameters: Optional[Dict[str, Any]] = Field( 1540 None, 1541 alias="$parameters", 1542 description="Additional parameters to be passed to the custom config transformation.", 1543 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1546class AddedFieldDefinition(BaseModel): 1547 type: Literal["AddedFieldDefinition"] 1548 path: List[str] = Field( 1549 ..., 1550 description="List of strings defining the path where to add the value on the record.", 1551 examples=[["segment_id"], ["metadata", "segment_id"]], 1552 title="Path", 1553 ) 1554 value: str = Field( 1555 ..., 1556 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1557 examples=[ 1558 "{{ record['updates'] }}", 1559 "{{ record['MetaData']['LastUpdatedTime'] }}", 1560 "{{ stream_partition['segment_id'] }}", 1561 ], 1562 title="Value", 1563 ) 1564 value_type: Optional[ValueType] = Field( 1565 None, 1566 description="Type of the value. If not specified, the type will be inferred from the value.", 1567 title="Value Type", 1568 ) 1569 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1572class AddFields(BaseModel): 1573 type: Literal["AddFields"] 1574 fields: List[AddedFieldDefinition] = Field( 1575 ..., 1576 description="List of transformations (path and corresponding value) that will be added to the record.", 1577 title="Fields", 1578 ) 1579 condition: Optional[str] = Field( 1580 "", 1581 description="Fields will be added if expression is evaluated to True.", 1582 examples=[ 1583 "{{ property|string == '' }}", 1584 "{{ property is integer }}", 1585 "{{ property|length > 5 }}", 1586 "{{ property == 'some_string_to_match' }}", 1587 ], 1588 ) 1589 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1592class ApiKeyAuthenticator(BaseModel): 1593 type: Literal["ApiKeyAuthenticator"] 1594 api_token: Optional[str] = Field( 1595 None, 1596 description="The API key to inject in the request. Fill it in the user inputs.", 1597 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1598 title="API Key", 1599 ) 1600 header: Optional[str] = Field( 1601 None, 1602 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1603 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1604 title="Header Name", 1605 ) 1606 inject_into: Optional[RequestOption] = Field( 1607 None, 1608 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1609 examples=[ 1610 {"inject_into": "header", "field_name": "Authorization"}, 1611 {"inject_into": "request_parameter", "field_name": "authKey"}, 1612 ], 1613 title="Inject API Key Into Outgoing HTTP Request", 1614 ) 1615 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1618class AuthFlow(BaseModel): 1619 auth_flow_type: Optional[AuthFlowType] = Field( 1620 None, description="The type of auth to use", title="Auth flow type" 1621 ) 1622 predicate_key: Optional[List[str]] = Field( 1623 None, 1624 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1625 examples=[["credentials", "auth_type"]], 1626 title="Predicate key", 1627 ) 1628 predicate_value: Optional[str] = Field( 1629 None, 1630 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1631 examples=["Oauth"], 1632 title="Predicate value", 1633 ) 1634 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1637class CheckStream(BaseModel): 1638 type: Literal["CheckStream"] 1639 stream_names: Optional[List[str]] = Field( 1640 None, 1641 description="Names of the streams to try reading from when running a check operation.", 1642 examples=[["users"], ["users", "contacts"]], 1643 title="Stream Names", 1644 ) 1645 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1648class IncrementingCountCursor(BaseModel): 1649 type: Literal["IncrementingCountCursor"] 1650 cursor_field: str = Field( 1651 ..., 1652 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1653 examples=["created_at", "{{ config['record_cursor'] }}"], 1654 title="Cursor Field", 1655 ) 1656 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1657 None, 1658 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1659 title="Allow Catalog Defined Cursor Field", 1660 ) 1661 start_value: Optional[Union[str, int]] = Field( 1662 None, 1663 description="The value that determines the earliest record that should be synced.", 1664 examples=[0, "{{ config['start_value'] }}"], 1665 title="Start Value", 1666 ) 1667 start_value_option: Optional[RequestOption] = Field( 1668 None, 1669 description="Optionally configures how the start value will be sent in requests to the source API.", 1670 title="Inject Start Value Into Outgoing HTTP Request", 1671 ) 1672 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1675class DatetimeBasedCursor(BaseModel): 1676 type: Literal["DatetimeBasedCursor"] 1677 clamping: Optional[Clamping] = Field( 1678 None, 1679 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1680 title="Date Range Clamping", 1681 ) 1682 cursor_field: str = Field( 1683 ..., 1684 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1685 examples=["created_at", "{{ config['record_cursor'] }}"], 1686 title="Cursor Field", 1687 ) 1688 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1689 None, 1690 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1691 title="Allow Catalog Defined Cursor Field", 1692 ) 1693 cursor_datetime_formats: Optional[List[str]] = Field( 1694 None, 1695 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1696 examples=[ 1697 "%Y-%m-%d", 1698 "%Y-%m-%d %H:%M:%S", 1699 "%Y-%m-%dT%H:%M:%S", 1700 "%Y-%m-%dT%H:%M:%SZ", 1701 "%Y-%m-%dT%H:%M:%S%z", 1702 "%Y-%m-%dT%H:%M:%S.%fZ", 1703 "%Y-%m-%dT%H:%M:%S.%f%z", 1704 "%Y-%m-%d %H:%M:%S.%f+00:00", 1705 "%s", 1706 "%ms", 1707 ], 1708 title="Cursor Datetime Formats", 1709 ) 1710 start_datetime: Union[MinMaxDatetime, str] = Field( 1711 ..., 1712 description="The datetime that determines the earliest record that should be synced.", 1713 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1714 title="Start Datetime", 1715 ) 1716 start_time_option: Optional[RequestOption] = Field( 1717 None, 1718 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1719 title="Inject Start Time Into Outgoing HTTP Request", 1720 ) 1721 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1722 None, 1723 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1724 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1725 title="End Datetime", 1726 ) 1727 end_time_option: Optional[RequestOption] = Field( 1728 None, 1729 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1730 title="Inject End Time Into Outgoing HTTP Request", 1731 ) 1732 datetime_format: str = Field( 1733 ..., 1734 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1735 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1736 title="Outgoing Datetime Format", 1737 ) 1738 cursor_granularity: Optional[str] = Field( 1739 None, 1740 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1741 examples=["PT1S"], 1742 title="Cursor Granularity", 1743 ) 1744 is_data_feed: Optional[bool] = Field( 1745 None, 1746 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1747 title="Data Feed API", 1748 ) 1749 is_client_side_incremental: Optional[bool] = Field( 1750 None, 1751 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1752 title="Client-side Incremental Filtering", 1753 ) 1754 is_compare_strictly: Optional[bool] = Field( 1755 False, 1756 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1757 title="Strict Start-End Time Comparison", 1758 ) 1759 global_substream_cursor: Optional[bool] = Field( 1760 False, 1761 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1762 title="Global Substream Cursor", 1763 ) 1764 lookback_window: Optional[str] = Field( 1765 None, 1766 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1767 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1768 title="Lookback Window", 1769 ) 1770 partition_field_end: Optional[str] = Field( 1771 None, 1772 description="Name of the partition start time field.", 1773 examples=["ending_time"], 1774 title="Partition Field End", 1775 ) 1776 partition_field_start: Optional[str] = Field( 1777 None, 1778 description="Name of the partition end time field.", 1779 examples=["starting_time"], 1780 title="Partition Field Start", 1781 ) 1782 step: Optional[str] = Field( 1783 None, 1784 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1785 examples=["P1W", "{{ config['step_increment'] }}"], 1786 title="Step", 1787 ) 1788 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1791class JwtAuthenticator(BaseModel): 1792 type: Literal["JwtAuthenticator"] 1793 secret_key: str = Field( 1794 ..., 1795 description="Secret used to sign the JSON web token.", 1796 examples=["{{ config['secret_key'] }}"], 1797 title="Secret Key", 1798 ) 1799 base64_encode_secret_key: Optional[bool] = Field( 1800 False, 1801 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1802 title="Base64-encode Secret Key", 1803 ) 1804 algorithm: Algorithm = Field( 1805 ..., 1806 description="Algorithm used to sign the JSON web token.", 1807 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1808 title="Algorithm", 1809 ) 1810 token_duration: Optional[int] = Field( 1811 1200, 1812 description="The amount of time in seconds a JWT token can be valid after being issued.", 1813 examples=[1200, 3600], 1814 title="Token Duration", 1815 ) 1816 header_prefix: Optional[str] = Field( 1817 None, 1818 description="The prefix to be used within the Authentication header.", 1819 examples=["Bearer", "Basic"], 1820 title="Header Prefix", 1821 ) 1822 jwt_headers: Optional[JwtHeaders] = Field( 1823 None, 1824 description="JWT headers used when signing JSON web token.", 1825 title="JWT Headers", 1826 ) 1827 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1828 None, 1829 description="Additional headers to be included with the JWT headers object.", 1830 title="Additional JWT Headers", 1831 ) 1832 jwt_payload: Optional[JwtPayload] = Field( 1833 None, 1834 description="JWT Payload used when signing JSON web token.", 1835 title="JWT Payload", 1836 ) 1837 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1838 None, 1839 description="Additional properties to be added to the JWT payload.", 1840 title="Additional JWT Payload Properties", 1841 ) 1842 passphrase: Optional[str] = Field( 1843 None, 1844 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1845 examples=["{{ config['passphrase'] }}"], 1846 title="Passphrase", 1847 ) 1848 request_option: Optional[RequestOption] = Field( 1849 None, 1850 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1851 title="Request Option", 1852 ) 1853 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1856class OAuthAuthenticator(BaseModel): 1857 type: Literal["OAuthAuthenticator"] 1858 client_id_name: Optional[str] = Field( 1859 "client_id", 1860 description="The name of the property to use to refresh the `access_token`.", 1861 examples=["custom_app_id"], 1862 title="Client ID Property Name", 1863 ) 1864 client_id: Optional[str] = Field( 1865 None, 1866 description="The OAuth client ID. Fill it in the user inputs.", 1867 examples=[ 1868 "{{ config['client_id'] }}", 1869 "{{ config['credentials']['client_id }}", 1870 ], 1871 title="Client ID", 1872 ) 1873 client_secret_name: Optional[str] = Field( 1874 "client_secret", 1875 description="The name of the property to use to refresh the `access_token`.", 1876 examples=["custom_app_secret"], 1877 title="Client Secret Property Name", 1878 ) 1879 client_secret: Optional[str] = Field( 1880 None, 1881 description="The OAuth client secret. Fill it in the user inputs.", 1882 examples=[ 1883 "{{ config['client_secret'] }}", 1884 "{{ config['credentials']['client_secret }}", 1885 ], 1886 title="Client Secret", 1887 ) 1888 refresh_token_name: Optional[str] = Field( 1889 "refresh_token", 1890 description="The name of the property to use to refresh the `access_token`.", 1891 examples=["custom_app_refresh_value"], 1892 title="Refresh Token Property Name", 1893 ) 1894 refresh_token: Optional[str] = Field( 1895 None, 1896 description="Credential artifact used to get a new access token.", 1897 examples=[ 1898 "{{ config['refresh_token'] }}", 1899 "{{ config['credentials]['refresh_token'] }}", 1900 ], 1901 title="Refresh Token", 1902 ) 1903 token_refresh_endpoint: Optional[str] = Field( 1904 None, 1905 description="The full URL to call to obtain a new access token.", 1906 examples=["https://connect.squareup.com/oauth2/token"], 1907 title="Token Refresh Endpoint", 1908 ) 1909 access_token_name: Optional[str] = Field( 1910 "access_token", 1911 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1912 examples=["access_token"], 1913 title="Access Token Property Name", 1914 ) 1915 access_token_value: Optional[str] = Field( 1916 None, 1917 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1918 examples=["secret_access_token_value"], 1919 title="Access Token Value", 1920 ) 1921 expires_in_name: Optional[str] = Field( 1922 "expires_in", 1923 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1924 examples=["expires_in"], 1925 title="Token Expiry Property Name", 1926 ) 1927 grant_type_name: Optional[str] = Field( 1928 "grant_type", 1929 description="The name of the property to use to refresh the `access_token`.", 1930 examples=["custom_grant_type"], 1931 title="Grant Type Property Name", 1932 ) 1933 grant_type: Optional[str] = Field( 1934 "refresh_token", 1935 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1936 examples=["refresh_token", "client_credentials"], 1937 title="Grant Type", 1938 ) 1939 refresh_request_body: Optional[Dict[str, Any]] = Field( 1940 None, 1941 description="Body of the request sent to get a new access token.", 1942 examples=[ 1943 { 1944 "applicationId": "{{ config['application_id'] }}", 1945 "applicationSecret": "{{ config['application_secret'] }}", 1946 "token": "{{ config['token'] }}", 1947 } 1948 ], 1949 title="Refresh Request Body", 1950 ) 1951 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1952 None, 1953 description="Headers of the request sent to get a new access token.", 1954 examples=[ 1955 { 1956 "Authorization": "<AUTH_TOKEN>", 1957 "Content-Type": "application/x-www-form-urlencoded", 1958 } 1959 ], 1960 title="Refresh Request Headers", 1961 ) 1962 send_refresh_request_as_query_params: Optional[bool] = Field( 1963 False, 1964 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 1965 examples=[True], 1966 title="Send Refresh Request As Query Params", 1967 ) 1968 scopes: Optional[List[str]] = Field( 1969 None, 1970 description="List of scopes that should be granted to the access token.", 1971 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1972 title="Scopes", 1973 ) 1974 token_expiry_date: Optional[str] = Field( 1975 None, 1976 description="The access token expiry date.", 1977 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1978 title="Token Expiry Date", 1979 ) 1980 token_expiry_date_format: Optional[str] = Field( 1981 None, 1982 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1983 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1984 title="Token Expiry Date Format", 1985 ) 1986 refresh_token_error_status_codes: Optional[List[int]] = Field( 1987 None, 1988 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1989 examples=[[400, 500]], 1990 title="Refresh Token Error Status Codes", 1991 ) 1992 refresh_token_error_key: Optional[str] = Field( 1993 None, 1994 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1995 examples=["error"], 1996 title="Refresh Token Error Key", 1997 ) 1998 refresh_token_error_values: Optional[List[str]] = Field( 1999 None, 2000 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2001 examples=[["invalid_grant", "invalid_permissions"]], 2002 title="Refresh Token Error Values", 2003 ) 2004 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2005 None, 2006 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2007 title="Refresh Token Updater", 2008 ) 2009 profile_assertion: Optional[JwtAuthenticator] = Field( 2010 None, 2011 description="The authenticator being used to authenticate the client authenticator.", 2012 title="Profile Assertion", 2013 ) 2014 use_profile_assertion: Optional[bool] = Field( 2015 False, 2016 description="Enable using profile assertion as a flow for OAuth authorization.", 2017 title="Use Profile Assertion", 2018 ) 2019 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2022class FixedWindowCallRatePolicy(BaseModel): 2023 class Config: 2024 extra = Extra.allow 2025 2026 type: Literal["FixedWindowCallRatePolicy"] 2027 period: str = Field( 2028 ..., description="The time interval for the rate limit window.", title="Period" 2029 ) 2030 call_limit: int = Field( 2031 ..., 2032 description="The maximum number of calls allowed within the period.", 2033 title="Call Limit", 2034 ) 2035 matchers: List[HttpRequestRegexMatcher] = Field( 2036 ..., 2037 description="List of matchers that define which requests this policy applies to.", 2038 title="Matchers", 2039 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2042class MovingWindowCallRatePolicy(BaseModel): 2043 class Config: 2044 extra = Extra.allow 2045 2046 type: Literal["MovingWindowCallRatePolicy"] 2047 rates: List[Rate] = Field( 2048 ..., 2049 description="List of rates that define the call limits for different time intervals.", 2050 title="Rates", 2051 ) 2052 matchers: List[HttpRequestRegexMatcher] = Field( 2053 ..., 2054 description="List of matchers that define which requests this policy applies to.", 2055 title="Matchers", 2056 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2059class UnlimitedCallRatePolicy(BaseModel): 2060 class Config: 2061 extra = Extra.allow 2062 2063 type: Literal["UnlimitedCallRatePolicy"] 2064 matchers: List[HttpRequestRegexMatcher] = Field( 2065 ..., 2066 description="List of matchers that define which requests this policy applies to.", 2067 title="Matchers", 2068 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2071class DefaultErrorHandler(BaseModel): 2072 type: Literal["DefaultErrorHandler"] 2073 backoff_strategies: Optional[ 2074 List[ 2075 Union[ 2076 ConstantBackoffStrategy, 2077 ExponentialBackoffStrategy, 2078 WaitTimeFromHeader, 2079 WaitUntilTimeFromHeader, 2080 CustomBackoffStrategy, 2081 ] 2082 ] 2083 ] = Field( 2084 None, 2085 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2086 title="Backoff Strategies", 2087 ) 2088 max_retries: Optional[Union[int, str]] = Field( 2089 5, 2090 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2091 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2092 title="Max Retry Count", 2093 ) 2094 response_filters: Optional[List[HttpResponseFilter]] = Field( 2095 None, 2096 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2097 title="Response Filters", 2098 ) 2099 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2102class DefaultPaginator(BaseModel): 2103 type: Literal["DefaultPaginator"] 2104 pagination_strategy: Union[ 2105 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2106 ] = Field( 2107 ..., 2108 description="Strategy defining how records are paginated.", 2109 title="Pagination Strategy", 2110 ) 2111 page_size_option: Optional[RequestOption] = Field( 2112 None, title="Inject Page Size Into Outgoing HTTP Request" 2113 ) 2114 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2115 None, 2116 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2117 title="Inject Page Token Into Outgoing HTTP Request", 2118 ) 2119 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2122class RecordExpander(BaseModel): 2123 type: Literal["RecordExpander"] 2124 expand_records_from_field: List[str] = Field( 2125 ..., 2126 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2127 examples=[ 2128 ["lines", "data"], 2129 ["items"], 2130 ["nested", "array"], 2131 ["sections", "*", "items"], 2132 ], 2133 title="Expand Records From Field", 2134 ) 2135 remain_original_record: Optional[bool] = Field( 2136 False, 2137 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2138 title="Remain Original Record", 2139 ) 2140 on_no_records: Optional[OnNoRecords] = Field( 2141 OnNoRecords.skip, 2142 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2143 title="On No Records", 2144 ) 2145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2148class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2149 type: Literal["ApiKey"] 2150 inject_into: RequestOption = Field( 2151 ..., 2152 description="Configure how the API Key will be sent in requests to the source API.", 2153 examples=[ 2154 {"inject_into": "header", "field_name": "Authorization"}, 2155 {"inject_into": "request_parameter", "field_name": "authKey"}, 2156 ], 2157 title="Inject API Key Into Outgoing HTTP Request", 2158 ) 2159 api_token: Optional[str] = Field( 2160 "{{ session_token }}", 2161 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2162 examples=[ 2163 "{{ session_token }}", 2164 "Token {{ session_token }}", 2165 "Bearer {{ session_token }}", 2166 ], 2167 title="API Token Template", 2168 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2171class JsonSchemaPropertySelector(BaseModel): 2172 type: Literal["JsonSchemaPropertySelector"] 2173 transformations: Optional[ 2174 List[ 2175 Union[ 2176 AddFields, 2177 RemoveFields, 2178 KeysToLower, 2179 KeysToSnakeCase, 2180 FlattenFields, 2181 DpathFlattenFields, 2182 KeysReplace, 2183 CustomTransformation, 2184 ] 2185 ] 2186 ] = Field( 2187 None, 2188 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2189 title="Transformations", 2190 ) 2191 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2194class ListPartitionRouter(BaseModel): 2195 type: Literal["ListPartitionRouter"] 2196 cursor_field: str = Field( 2197 ..., 2198 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2199 examples=["section", "{{ config['section_key'] }}"], 2200 title="Current Partition Value Identifier", 2201 ) 2202 values: Union[str, List[str]] = Field( 2203 ..., 2204 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2205 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2206 title="Partition Values", 2207 ) 2208 request_option: Optional[RequestOption] = Field( 2209 None, 2210 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2211 title="Inject Partition Value Into Outgoing HTTP Request", 2212 ) 2213 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2216class PaginationReset(BaseModel): 2217 type: Literal["PaginationReset"] 2218 action: Action1 2219 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2222class GzipDecoder(BaseModel): 2223 type: Literal["GzipDecoder"] 2224 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2227class RequestBodyGraphQL(BaseModel): 2228 type: Literal["RequestBodyGraphQL"] 2229 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2232class DpathValidator(BaseModel): 2233 type: Literal["DpathValidator"] 2234 field_path: List[str] = Field( 2235 ..., 2236 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2237 examples=[ 2238 ["data"], 2239 ["data", "records"], 2240 ["data", "{{ parameters.name }}"], 2241 ["data", "*", "record"], 2242 ], 2243 title="Field Path", 2244 ) 2245 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2246 ..., 2247 description="The condition that the specified config value will be evaluated against", 2248 title="Validation Strategy", 2249 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2252class PredicateValidator(BaseModel): 2253 type: Literal["PredicateValidator"] 2254 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2255 ..., 2256 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2257 examples=[ 2258 "test-value", 2259 "{{ config['api_version'] }}", 2260 "{{ config['tenant_id'] }}", 2261 123, 2262 ], 2263 title="Value", 2264 ) 2265 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2266 ..., 2267 description="The validation strategy to apply to the value.", 2268 title="Validation Strategy", 2269 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2272class ConfigAddFields(BaseModel): 2273 type: Literal["ConfigAddFields"] 2274 fields: List[AddedFieldDefinition] = Field( 2275 ..., 2276 description="A list of transformations (path and corresponding value) that will be added to the config.", 2277 title="Fields", 2278 ) 2279 condition: Optional[str] = Field( 2280 "", 2281 description="Fields will be added if expression is evaluated to True.", 2282 examples=[ 2283 "{{ config['environemnt'] == 'sandbox' }}", 2284 "{{ property is integer }}", 2285 "{{ property|length > 5 }}", 2286 "{{ property == 'some_string_to_match' }}", 2287 ], 2288 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2291class CompositeErrorHandler(BaseModel): 2292 type: Literal["CompositeErrorHandler"] 2293 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2294 Field( 2295 ..., 2296 description="List of error handlers to iterate on to determine how to handle a failed response.", 2297 title="Error Handlers", 2298 ) 2299 ) 2300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2303class HTTPAPIBudget(BaseModel): 2304 class Config: 2305 extra = Extra.allow 2306 2307 type: Literal["HTTPAPIBudget"] 2308 policies: List[ 2309 Union[ 2310 FixedWindowCallRatePolicy, 2311 MovingWindowCallRatePolicy, 2312 UnlimitedCallRatePolicy, 2313 ] 2314 ] = Field( 2315 ..., 2316 description="List of call rate policies that define how many calls are allowed.", 2317 title="Policies", 2318 ) 2319 ratelimit_reset_header: Optional[str] = Field( 2320 "ratelimit-reset", 2321 description="The HTTP response header name that indicates when the rate limit resets.", 2322 title="Rate Limit Reset Header", 2323 ) 2324 ratelimit_remaining_header: Optional[str] = Field( 2325 "ratelimit-remaining", 2326 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2327 title="Rate Limit Remaining Header", 2328 ) 2329 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2330 [429], 2331 description="List of HTTP status codes that indicate a rate limit has been hit.", 2332 title="Status Codes for Rate Limit Hit", 2333 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2336class DpathExtractor(BaseModel): 2337 type: Literal["DpathExtractor"] 2338 field_path: List[str] = Field( 2339 ..., 2340 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2341 examples=[ 2342 ["data"], 2343 ["data", "records"], 2344 ["data", "{{ parameters.name }}"], 2345 ["data", "*", "record"], 2346 ], 2347 title="Field Path", 2348 ) 2349 record_expander: Optional[RecordExpander] = Field( 2350 None, 2351 description="Optional component to expand records by extracting items from nested array fields.", 2352 title="Record Expander", 2353 ) 2354 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2357class ZipfileDecoder(BaseModel): 2358 class Config: 2359 extra = Extra.allow 2360 2361 type: Literal["ZipfileDecoder"] 2362 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2363 ..., 2364 description="Parser to parse the decompressed data from the zipfile(s).", 2365 title="Parser", 2366 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2369class RecordSelector(BaseModel): 2370 type: Literal["RecordSelector"] 2371 extractor: Union[DpathExtractor, CustomRecordExtractor] 2372 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2373 None, 2374 description="Responsible for filtering records to be emitted by the Source.", 2375 title="Record Filter", 2376 ) 2377 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2378 None, 2379 description="Responsible for normalization according to the schema.", 2380 title="Schema Normalization", 2381 ) 2382 transform_before_filtering: Optional[bool] = Field( 2383 None, 2384 description="If true, transformation will be applied before record filtering.", 2385 title="Transform Before Filtering", 2386 ) 2387 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2390class ConfigMigration(BaseModel): 2391 type: Literal["ConfigMigration"] 2392 description: Optional[str] = Field( 2393 None, description="The description/purpose of the config migration." 2394 ) 2395 transformations: List[ 2396 Union[ 2397 ConfigRemapField, 2398 ConfigAddFields, 2399 ConfigRemoveFields, 2400 CustomConfigTransformation, 2401 ] 2402 ] = Field( 2403 ..., 2404 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2405 title="Transformations", 2406 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2409class ConfigNormalizationRules(BaseModel): 2410 class Config: 2411 extra = Extra.forbid 2412 2413 type: Literal["ConfigNormalizationRules"] 2414 config_migrations: Optional[List[ConfigMigration]] = Field( 2415 [], 2416 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2417 title="Config Migrations", 2418 ) 2419 transformations: Optional[ 2420 List[ 2421 Union[ 2422 ConfigRemapField, 2423 ConfigAddFields, 2424 ConfigRemoveFields, 2425 CustomConfigTransformation, 2426 ] 2427 ] 2428 ] = Field( 2429 [], 2430 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2431 title="Transformations", 2432 ) 2433 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2434 [], 2435 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2436 title="Validations", 2437 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2440class Spec(BaseModel): 2441 type: Literal["Spec"] 2442 connection_specification: Dict[str, Any] = Field( 2443 ..., 2444 description="A connection specification describing how a the connector can be configured.", 2445 title="Connection Specification", 2446 ) 2447 documentation_url: Optional[str] = Field( 2448 None, 2449 description="URL of the connector's documentation page.", 2450 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2451 title="Documentation URL", 2452 ) 2453 advanced_auth: Optional[AuthFlow] = Field( 2454 None, 2455 description="Advanced specification for configuring the authentication flow.", 2456 title="Advanced Auth", 2457 ) 2458 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2459 None, title="Config Normalization Rules" 2460 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2463class DeclarativeSource1(BaseModel): 2464 class Config: 2465 extra = Extra.forbid 2466 2467 type: Literal["DeclarativeSource"] 2468 check: Union[CheckStream, CheckDynamicStream] 2469 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2470 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2471 version: str = Field( 2472 ..., 2473 description="The version of the Airbyte CDK used to build and test the source.", 2474 ) 2475 schemas: Optional[Schemas] = None 2476 definitions: Optional[Dict[str, Any]] = None 2477 spec: Optional[Spec] = None 2478 concurrency_level: Optional[ConcurrencyLevel] = None 2479 api_budget: Optional[HTTPAPIBudget] = None 2480 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2481 None, 2482 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2483 title="Stream Groups", 2484 ) 2485 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2486 None, 2487 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2488 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2489 title="Maximum Concurrent Asynchronous Jobs", 2490 ) 2491 metadata: Optional[Dict[str, Any]] = Field( 2492 None, 2493 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2494 ) 2495 description: Optional[str] = Field( 2496 None, 2497 description="A description of the connector. It will be presented on the Source documentation page.", 2498 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2501class DeclarativeSource2(BaseModel): 2502 class Config: 2503 extra = Extra.forbid 2504 2505 type: Literal["DeclarativeSource"] 2506 check: Union[CheckStream, CheckDynamicStream] 2507 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2508 None 2509 ) 2510 dynamic_streams: List[DynamicDeclarativeStream] 2511 version: str = Field( 2512 ..., 2513 description="The version of the Airbyte CDK used to build and test the source.", 2514 ) 2515 schemas: Optional[Schemas] = None 2516 definitions: Optional[Dict[str, Any]] = None 2517 spec: Optional[Spec] = None 2518 concurrency_level: Optional[ConcurrencyLevel] = None 2519 api_budget: Optional[HTTPAPIBudget] = None 2520 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2521 None, 2522 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2523 title="Stream Groups", 2524 ) 2525 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2526 None, 2527 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2528 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2529 title="Maximum Concurrent Asynchronous Jobs", 2530 ) 2531 metadata: Optional[Dict[str, Any]] = Field( 2532 None, 2533 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2534 ) 2535 description: Optional[str] = Field( 2536 None, 2537 description="A description of the connector. It will be presented on the Source documentation page.", 2538 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2541class DeclarativeSource(BaseModel): 2542 class Config: 2543 extra = Extra.forbid 2544 2545 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2546 ..., 2547 description="An API source that extracts data according to its declarative components.", 2548 title="DeclarativeSource", 2549 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2552class SelectiveAuthenticator(BaseModel): 2553 class Config: 2554 extra = Extra.allow 2555 2556 type: Literal["SelectiveAuthenticator"] 2557 authenticator_selection_path: List[str] = Field( 2558 ..., 2559 description="Path of the field in config with selected authenticator name", 2560 examples=[["auth"], ["auth", "type"]], 2561 title="Authenticator Selection Path", 2562 ) 2563 authenticators: Dict[ 2564 str, 2565 Union[ 2566 ApiKeyAuthenticator, 2567 BasicHttpAuthenticator, 2568 BearerAuthenticator, 2569 OAuthAuthenticator, 2570 JwtAuthenticator, 2571 SessionTokenAuthenticator, 2572 LegacySessionTokenAuthenticator, 2573 CustomAuthenticator, 2574 NoAuth, 2575 ], 2576 ] = Field( 2577 ..., 2578 description="Authenticators to select from.", 2579 examples=[ 2580 { 2581 "authenticators": { 2582 "token": "#/definitions/ApiKeyAuthenticator", 2583 "oauth": "#/definitions/OAuthAuthenticator", 2584 "jwt": "#/definitions/JwtAuthenticator", 2585 } 2586 } 2587 ], 2588 title="Authenticators", 2589 ) 2590 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2593class ConditionalStreams(BaseModel): 2594 type: Literal["ConditionalStreams"] 2595 condition: str = Field( 2596 ..., 2597 description="Condition that will be evaluated to determine if a set of streams should be available.", 2598 examples=["{{ config['is_sandbox'] }}"], 2599 title="Condition", 2600 ) 2601 streams: List[DeclarativeStream] = Field( 2602 ..., 2603 description="Streams that will be used during an operation based on the condition.", 2604 title="Streams", 2605 ) 2606 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2609class FileUploader(BaseModel): 2610 type: Literal["FileUploader"] 2611 requester: Union[HttpRequester, CustomRequester] = Field( 2612 ..., 2613 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2614 ) 2615 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2616 ..., 2617 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2618 ) 2619 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2620 None, 2621 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2622 ) 2623 filename_extractor: Optional[str] = Field( 2624 None, 2625 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2626 examples=[ 2627 "{{ record.id }}/{{ record.file_name }}/", 2628 "{{ record.id }}_{{ record.file_name }}/", 2629 ], 2630 ) 2631 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2634class DeclarativeStream(BaseModel): 2635 class Config: 2636 extra = Extra.allow 2637 2638 type: Literal["DeclarativeStream"] 2639 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2640 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2641 ..., 2642 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2643 title="Retriever", 2644 ) 2645 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2646 None, 2647 description="Component used to fetch data incrementally based on a time field in the data.", 2648 title="Incremental Sync", 2649 ) 2650 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2651 schema_loader: Optional[ 2652 Union[ 2653 InlineSchemaLoader, 2654 DynamicSchemaLoader, 2655 JsonFileSchemaLoader, 2656 List[ 2657 Union[ 2658 InlineSchemaLoader, 2659 DynamicSchemaLoader, 2660 JsonFileSchemaLoader, 2661 CustomSchemaLoader, 2662 ] 2663 ], 2664 CustomSchemaLoader, 2665 ] 2666 ] = Field( 2667 None, 2668 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2669 title="Schema Loader", 2670 ) 2671 transformations: Optional[ 2672 List[ 2673 Union[ 2674 AddFields, 2675 RemoveFields, 2676 KeysToLower, 2677 KeysToSnakeCase, 2678 FlattenFields, 2679 DpathFlattenFields, 2680 KeysReplace, 2681 CustomTransformation, 2682 ] 2683 ] 2684 ] = Field( 2685 None, 2686 description="A list of transformations to be applied to each output record.", 2687 title="Transformations", 2688 ) 2689 state_migrations: Optional[ 2690 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2691 ] = Field( 2692 [], 2693 description="Array of state migrations to be applied on the input state", 2694 title="State Migrations", 2695 ) 2696 file_uploader: Optional[FileUploader] = Field( 2697 None, 2698 description="(experimental) Describes how to fetch a file", 2699 title="File Uploader", 2700 ) 2701 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2704class SessionTokenAuthenticator(BaseModel): 2705 type: Literal["SessionTokenAuthenticator"] 2706 login_requester: HttpRequester = Field( 2707 ..., 2708 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2709 examples=[ 2710 { 2711 "type": "HttpRequester", 2712 "url_base": "https://my_api.com", 2713 "path": "/login", 2714 "authenticator": { 2715 "type": "BasicHttpAuthenticator", 2716 "username": "{{ config.username }}", 2717 "password": "{{ config.password }}", 2718 }, 2719 } 2720 ], 2721 title="Login Requester", 2722 ) 2723 session_token_path: List[str] = Field( 2724 ..., 2725 description="The path in the response body returned from the login requester to the session token.", 2726 examples=[["access_token"], ["result", "token"]], 2727 title="Session Token Path", 2728 ) 2729 expiration_duration: Optional[str] = Field( 2730 None, 2731 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2732 examples=["PT1H", "P1D"], 2733 title="Expiration Duration", 2734 ) 2735 request_authentication: Union[ 2736 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2737 ] = Field( 2738 ..., 2739 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2740 title="Data Request Authentication", 2741 ) 2742 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2743 None, description="Component used to decode the response.", title="Decoder" 2744 ) 2745 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2748class HttpRequester(BaseModelWithDeprecations): 2749 type: Literal["HttpRequester"] 2750 url_base: Optional[str] = Field( 2751 None, 2752 deprecated=True, 2753 deprecation_message="Use `url` field instead.", 2754 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2755 examples=[ 2756 "https://connect.squareup.com/v2", 2757 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2758 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2759 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2760 ], 2761 title="API Base URL", 2762 ) 2763 url: Optional[str] = Field( 2764 None, 2765 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2766 examples=[ 2767 "https://connect.squareup.com/v2", 2768 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2769 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2770 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2771 ], 2772 title="API Endpoint URL", 2773 ) 2774 path: Optional[str] = Field( 2775 None, 2776 deprecated=True, 2777 deprecation_message="Use `url` field instead.", 2778 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2779 examples=[ 2780 "/products", 2781 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2782 "/trades/{{ config['symbol_id'] }}/history", 2783 ], 2784 title="URL Path", 2785 ) 2786 http_method: Optional[HttpMethod] = Field( 2787 HttpMethod.GET, 2788 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2789 examples=["GET", "POST"], 2790 title="HTTP Method", 2791 ) 2792 authenticator: Optional[ 2793 Union[ 2794 ApiKeyAuthenticator, 2795 BasicHttpAuthenticator, 2796 BearerAuthenticator, 2797 OAuthAuthenticator, 2798 JwtAuthenticator, 2799 SessionTokenAuthenticator, 2800 SelectiveAuthenticator, 2801 CustomAuthenticator, 2802 NoAuth, 2803 LegacySessionTokenAuthenticator, 2804 ] 2805 ] = Field( 2806 None, 2807 description="Authentication method to use for requests sent to the API.", 2808 title="Authenticator", 2809 ) 2810 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2811 None, 2812 deprecated=True, 2813 deprecation_message="Use `query_properties` field instead.", 2814 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2815 title="Fetch Properties from Endpoint", 2816 ) 2817 query_properties: Optional[QueryProperties] = Field( 2818 None, 2819 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2820 title="Query Properties", 2821 ) 2822 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2823 None, 2824 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2825 examples=[ 2826 {"unit": "day"}, 2827 { 2828 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2829 }, 2830 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2831 {"sort_by[asc]": "updated_at"}, 2832 ], 2833 title="Query Parameters", 2834 ) 2835 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2836 None, 2837 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2838 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2839 title="Request Headers", 2840 ) 2841 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2842 None, 2843 deprecated=True, 2844 deprecation_message="Use `request_body` field instead.", 2845 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2846 examples=[ 2847 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2848 ], 2849 title="Request Body Payload (Non-JSON)", 2850 ) 2851 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2852 None, 2853 deprecated=True, 2854 deprecation_message="Use `request_body` field instead.", 2855 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2856 examples=[ 2857 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2858 {"key": "{{ config['value'] }}"}, 2859 {"sort": {"field": "updated_at", "order": "ascending"}}, 2860 ], 2861 title="Request Body JSON Payload", 2862 ) 2863 request_body: Optional[ 2864 Union[ 2865 RequestBodyPlainText, 2866 RequestBodyUrlEncodedForm, 2867 RequestBodyJsonObject, 2868 RequestBodyGraphQL, 2869 ] 2870 ] = Field( 2871 None, 2872 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2873 title="Request Body", 2874 ) 2875 error_handler: Optional[ 2876 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2877 ] = Field( 2878 None, 2879 description="Error handler component that defines how to handle errors.", 2880 title="Error Handler", 2881 ) 2882 use_cache: Optional[bool] = Field( 2883 False, 2884 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 2885 title="Use Cache", 2886 ) 2887 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2890class DynamicSchemaLoader(BaseModel): 2891 type: Literal["DynamicSchemaLoader"] 2892 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2893 ..., 2894 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2895 title="Retriever", 2896 ) 2897 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2898 None, 2899 description="Responsible for filtering fields to be added to json schema.", 2900 title="Schema Filter", 2901 ) 2902 schema_transformations: Optional[ 2903 List[ 2904 Union[ 2905 AddFields, 2906 RemoveFields, 2907 KeysToLower, 2908 KeysToSnakeCase, 2909 FlattenFields, 2910 DpathFlattenFields, 2911 KeysReplace, 2912 CustomTransformation, 2913 ] 2914 ] 2915 ] = Field( 2916 None, 2917 description="A list of transformations to be applied to the schema.", 2918 title="Schema Transformations", 2919 ) 2920 schema_type_identifier: SchemaTypeIdentifier 2921 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2924class ParentStreamConfig(BaseModel): 2925 type: Literal["ParentStreamConfig"] 2926 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2927 ..., description="Reference to the parent stream.", title="Parent Stream" 2928 ) 2929 parent_key: str = Field( 2930 ..., 2931 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2932 examples=["id", "{{ config['parent_record_id'] }}"], 2933 title="Parent Key", 2934 ) 2935 partition_field: str = Field( 2936 ..., 2937 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2938 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2939 title="Current Parent Key Value Identifier", 2940 ) 2941 request_option: Optional[RequestOption] = Field( 2942 None, 2943 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2944 title="Request Option", 2945 ) 2946 incremental_dependency: Optional[bool] = Field( 2947 False, 2948 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2949 title="Incremental Dependency", 2950 ) 2951 lazy_read_pointer: Optional[List[str]] = Field( 2952 [], 2953 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2954 title="Lazy Read Pointer", 2955 ) 2956 extra_fields: Optional[List[List[str]]] = Field( 2957 None, 2958 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2959 title="Extra Fields", 2960 ) 2961 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2964class PropertiesFromEndpoint(BaseModel): 2965 type: Literal["PropertiesFromEndpoint"] 2966 property_field_path: List[str] = Field( 2967 ..., 2968 description="Describes the path to the field that should be extracted", 2969 examples=[["name"]], 2970 ) 2971 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2972 ..., 2973 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2974 ) 2975 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2978class QueryProperties(BaseModel): 2979 type: Literal["QueryProperties"] 2980 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2981 ..., 2982 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2983 title="Property List", 2984 ) 2985 always_include_properties: Optional[List[str]] = Field( 2986 None, 2987 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2988 title="Always Include Properties", 2989 ) 2990 property_chunking: Optional[PropertyChunking] = Field( 2991 None, 2992 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2993 title="Property Chunking", 2994 ) 2995 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2996 None, 2997 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2998 title="Property Selector", 2999 ) 3000 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3003class StateDelegatingStream(BaseModel): 3004 type: Literal["StateDelegatingStream"] 3005 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3006 full_refresh_stream: DeclarativeStream = Field( 3007 ..., 3008 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3009 title="Full Refresh Stream", 3010 ) 3011 incremental_stream: DeclarativeStream = Field( 3012 ..., 3013 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3014 title="Incremental Stream", 3015 ) 3016 api_retention_period: Optional[str] = Field( 3017 None, 3018 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3019 examples=["P30D", "P90D", "P1Y"], 3020 title="API Retention Period", 3021 ) 3022 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3025class SimpleRetriever(BaseModel): 3026 type: Literal["SimpleRetriever"] 3027 requester: Union[HttpRequester, CustomRequester] = Field( 3028 ..., 3029 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3030 ) 3031 decoder: Optional[ 3032 Union[ 3033 JsonDecoder, 3034 JsonItemsDecoder, 3035 XmlDecoder, 3036 CsvDecoder, 3037 JsonlDecoder, 3038 GzipDecoder, 3039 IterableDecoder, 3040 ZipfileDecoder, 3041 CustomDecoder, 3042 ] 3043 ] = Field( 3044 None, 3045 description="Component decoding the response so records can be extracted.", 3046 title="HTTP Response Format", 3047 ) 3048 record_selector: RecordSelector = Field( 3049 ..., 3050 description="Component that describes how to extract records from a HTTP response.", 3051 ) 3052 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3053 None, 3054 description="Paginator component that describes how to navigate through the API's pages.", 3055 ) 3056 pagination_reset: Optional[PaginationReset] = Field( 3057 None, 3058 description="Describes what triggers pagination reset and how to handle it.", 3059 ) 3060 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3061 False, 3062 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3063 ) 3064 partition_router: Optional[ 3065 Union[ 3066 SubstreamPartitionRouter, 3067 ListPartitionRouter, 3068 GroupingPartitionRouter, 3069 CustomPartitionRouter, 3070 List[ 3071 Union[ 3072 SubstreamPartitionRouter, 3073 ListPartitionRouter, 3074 GroupingPartitionRouter, 3075 CustomPartitionRouter, 3076 ] 3077 ], 3078 ] 3079 ] = Field( 3080 None, 3081 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3082 title="Partition Router", 3083 ) 3084 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3087class AsyncRetriever(BaseModel): 3088 type: Literal["AsyncRetriever"] 3089 record_selector: RecordSelector = Field( 3090 ..., 3091 description="Component that describes how to extract records from a HTTP response.", 3092 ) 3093 status_mapping: AsyncJobStatusMap = Field( 3094 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3095 ) 3096 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3097 ..., description="Responsible for fetching the actual status of the async job." 3098 ) 3099 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3100 None, 3101 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3102 ) 3103 download_extractor: Optional[ 3104 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3105 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3106 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3107 ..., 3108 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3109 ) 3110 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3111 ..., 3112 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3113 ) 3114 polling_job_timeout: Optional[Union[int, str]] = Field( 3115 None, 3116 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3117 ) 3118 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3119 None, 3120 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3121 ge=1, 3122 ) 3123 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3124 None, 3125 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3126 ) 3127 download_requester: Union[HttpRequester, CustomRequester] = Field( 3128 ..., 3129 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3130 ) 3131 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3132 None, 3133 description="Paginator component that describes how to navigate through the API's pages during download.", 3134 ) 3135 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3136 None, 3137 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3138 ) 3139 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3140 None, 3141 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3142 ) 3143 partition_router: Optional[ 3144 Union[ 3145 ListPartitionRouter, 3146 SubstreamPartitionRouter, 3147 GroupingPartitionRouter, 3148 CustomPartitionRouter, 3149 List[ 3150 Union[ 3151 ListPartitionRouter, 3152 SubstreamPartitionRouter, 3153 GroupingPartitionRouter, 3154 CustomPartitionRouter, 3155 ] 3156 ], 3157 ] 3158 ] = Field( 3159 [], 3160 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3161 title="Partition Router", 3162 ) 3163 decoder: Optional[ 3164 Union[ 3165 CsvDecoder, 3166 GzipDecoder, 3167 JsonDecoder, 3168 JsonItemsDecoder, 3169 JsonlDecoder, 3170 IterableDecoder, 3171 XmlDecoder, 3172 ZipfileDecoder, 3173 CustomDecoder, 3174 ] 3175 ] = Field( 3176 None, 3177 description="Component decoding the response so records can be extracted.", 3178 title="HTTP Response Format", 3179 ) 3180 download_decoder: Optional[ 3181 Union[ 3182 CsvDecoder, 3183 GzipDecoder, 3184 JsonDecoder, 3185 JsonItemsDecoder, 3186 JsonlDecoder, 3187 IterableDecoder, 3188 XmlDecoder, 3189 ZipfileDecoder, 3190 CustomDecoder, 3191 ] 3192 ] = Field( 3193 None, 3194 description="Component decoding the download response so records can be extracted.", 3195 title="Download HTTP Response Format", 3196 ) 3197 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3204class StreamGroup(BaseModel): 3205 streams: List[str] = Field( 3206 ..., 3207 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3208 title="Streams", 3209 ) 3210 action: BlockSimultaneousSyncsAction = Field( 3211 ..., 3212 description="The action to apply to streams in this group.", 3213 title="Action", 3214 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3217class SubstreamPartitionRouter(BaseModel): 3218 type: Literal["SubstreamPartitionRouter"] 3219 parent_stream_configs: List[ParentStreamConfig] = Field( 3220 ..., 3221 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3222 title="Parent Stream Configs", 3223 ) 3224 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3227class GroupingPartitionRouter(BaseModel): 3228 type: Literal["GroupingPartitionRouter"] 3229 group_size: int = Field( 3230 ..., 3231 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3232 examples=[10, 50], 3233 title="Group Size", 3234 ) 3235 underlying_partition_router: Union[ 3236 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3237 ] = Field( 3238 ..., 3239 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3240 title="Underlying Partition Router", 3241 ) 3242 deduplicate: Optional[bool] = Field( 3243 True, 3244 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3245 title="Deduplicate Partitions", 3246 ) 3247 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3250class HttpComponentsResolver(BaseModel): 3251 type: Literal["HttpComponentsResolver"] 3252 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3253 ..., 3254 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3255 title="Retriever", 3256 ) 3257 components_mapping: List[ComponentMappingDefinition] 3258 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3261class DynamicDeclarativeStream(BaseModel): 3262 type: Literal["DynamicDeclarativeStream"] 3263 name: Optional[str] = Field( 3264 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3265 ) 3266 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3267 ..., description="Reference to the stream template.", title="Stream Template" 3268 ) 3269 components_resolver: Union[ 3270 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3271 ] = Field( 3272 ..., 3273 description="Component resolve and populates stream templates with components values.", 3274 title="Components Resolver", 3275 ) 3276 use_parent_parameters: Optional[bool] = Field( 3277 True, 3278 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3279 title="Use Parent Parameters", 3280 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]