airbyte_cdk.sources.declarative.models.declarative_component_schema
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2 3# generated by datamodel-codegen: 4# filename: declarative_component_schema.yaml 5 6from __future__ import annotations 7 8from enum import Enum 9from typing import Any, Dict, List, Literal, Optional, Union 10 11from pydantic.v1 import BaseModel, Extra, Field 12 13from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 14 BaseModelWithDeprecations, 15) 16 17 18class AuthFlowType(Enum): 19 oauth2_0 = "oauth2.0" 20 oauth1_0 = "oauth1.0" 21 22 23class BasicHttpAuthenticator(BaseModel): 24 type: Literal["BasicHttpAuthenticator"] 25 username: str = Field( 26 ..., 27 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 28 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 29 title="Username", 30 ) 31 password: Optional[str] = Field( 32 "", 33 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 34 examples=["{{ config['password'] }}", ""], 35 title="Password", 36 ) 37 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 38 39 40class BearerAuthenticator(BaseModel): 41 type: Literal["BearerAuthenticator"] 42 api_token: str = Field( 43 ..., 44 description="Token to inject as request header for authenticating with the API.", 45 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 46 title="Bearer Token", 47 ) 48 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 49 50 51class DynamicStreamCheckConfig(BaseModel): 52 type: Literal["DynamicStreamCheckConfig"] 53 dynamic_stream_name: str = Field( 54 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 55 ) 56 stream_count: Optional[int] = Field( 57 0, 58 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 59 title="Stream Count", 60 ) 61 62 63class CheckDynamicStream(BaseModel): 64 type: Literal["CheckDynamicStream"] 65 stream_count: int = Field( 66 ..., 67 description="Numbers of the streams to try reading from when running a check operation.", 68 title="Stream Count", 69 ) 70 use_check_availability: Optional[bool] = Field( 71 True, 72 description="Enables stream check availability. This field is automatically set by the CDK.", 73 title="Use Check Availability", 74 ) 75 76 77class ConcurrencyLevel(BaseModel): 78 type: Optional[Literal["ConcurrencyLevel"]] = None 79 default_concurrency: Union[int, str] = Field( 80 ..., 81 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 82 examples=[10, "{{ config['num_workers'] or 10 }}"], 83 title="Default Concurrency", 84 ) 85 max_concurrency: Optional[int] = Field( 86 None, 87 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 88 examples=[20, 100], 89 title="Max Concurrency", 90 ) 91 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 92 93 94class ConstantBackoffStrategy(BaseModel): 95 type: Literal["ConstantBackoffStrategy"] 96 backoff_time_in_seconds: Union[float, str] = Field( 97 ..., 98 description="Backoff time in seconds.", 99 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 100 title="Backoff Time", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class CursorPagination(BaseModel): 106 type: Literal["CursorPagination"] 107 cursor_value: str = Field( 108 ..., 109 description="Value of the cursor defining the next page to fetch.", 110 examples=[ 111 "{{ headers.link.next.cursor }}", 112 "{{ last_record['key'] }}", 113 "{{ response['nextPage'] }}", 114 ], 115 title="Cursor Value", 116 ) 117 page_size: Optional[int] = Field( 118 None, 119 description="The number of records to include in each pages.", 120 examples=[100], 121 title="Page Size", 122 ) 123 stop_condition: Optional[str] = Field( 124 None, 125 description="Template string evaluating when to stop paginating.", 126 examples=[ 127 "{{ response.data.has_more is false }}", 128 "{{ 'next' not in headers['link'] }}", 129 ], 130 title="Stop Condition", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class CustomAuthenticator(BaseModel): 136 class Config: 137 extra = Extra.allow 138 139 type: Literal["CustomAuthenticator"] 140 class_name: str = Field( 141 ..., 142 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 143 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 144 title="Class Name", 145 ) 146 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 147 148 149class CustomBackoffStrategy(BaseModel): 150 class Config: 151 extra = Extra.allow 152 153 type: Literal["CustomBackoffStrategy"] 154 class_name: str = Field( 155 ..., 156 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 157 examples=["source_railz.components.MyCustomBackoffStrategy"], 158 title="Class Name", 159 ) 160 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 161 162 163class CustomErrorHandler(BaseModel): 164 class Config: 165 extra = Extra.allow 166 167 type: Literal["CustomErrorHandler"] 168 class_name: str = Field( 169 ..., 170 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 171 examples=["source_railz.components.MyCustomErrorHandler"], 172 title="Class Name", 173 ) 174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 175 176 177class CustomPaginationStrategy(BaseModel): 178 class Config: 179 extra = Extra.allow 180 181 type: Literal["CustomPaginationStrategy"] 182 class_name: str = Field( 183 ..., 184 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 185 examples=["source_railz.components.MyCustomPaginationStrategy"], 186 title="Class Name", 187 ) 188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 189 190 191class CustomRecordExtractor(BaseModel): 192 class Config: 193 extra = Extra.allow 194 195 type: Literal["CustomRecordExtractor"] 196 class_name: str = Field( 197 ..., 198 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 199 examples=["source_railz.components.MyCustomRecordExtractor"], 200 title="Class Name", 201 ) 202 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 203 204 205class CustomRecordFilter(BaseModel): 206 class Config: 207 extra = Extra.allow 208 209 type: Literal["CustomRecordFilter"] 210 class_name: str = Field( 211 ..., 212 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 213 examples=["source_railz.components.MyCustomCustomRecordFilter"], 214 title="Class Name", 215 ) 216 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 217 218 219class CustomRequester(BaseModel): 220 class Config: 221 extra = Extra.allow 222 223 type: Literal["CustomRequester"] 224 class_name: str = Field( 225 ..., 226 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 227 examples=["source_railz.components.MyCustomRecordExtractor"], 228 title="Class Name", 229 ) 230 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 231 232 233class CustomRetriever(BaseModel): 234 class Config: 235 extra = Extra.allow 236 237 type: Literal["CustomRetriever"] 238 class_name: str = Field( 239 ..., 240 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 241 examples=["source_railz.components.MyCustomRetriever"], 242 title="Class Name", 243 ) 244 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 245 246 247class CustomPartitionRouter(BaseModel): 248 class Config: 249 extra = Extra.allow 250 251 type: Literal["CustomPartitionRouter"] 252 class_name: str = Field( 253 ..., 254 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 255 examples=["source_railz.components.MyCustomPartitionRouter"], 256 title="Class Name", 257 ) 258 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 259 260 261class CustomSchemaLoader(BaseModel): 262 class Config: 263 extra = Extra.allow 264 265 type: Literal["CustomSchemaLoader"] 266 class_name: str = Field( 267 ..., 268 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 269 examples=["source_railz.components.MyCustomSchemaLoader"], 270 title="Class Name", 271 ) 272 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 273 274 275class CustomSchemaNormalization(BaseModel): 276 class Config: 277 extra = Extra.allow 278 279 type: Literal["CustomSchemaNormalization"] 280 class_name: str = Field( 281 ..., 282 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 283 examples=[ 284 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 285 ], 286 title="Class Name", 287 ) 288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 289 290 291class CustomStateMigration(BaseModel): 292 class Config: 293 extra = Extra.allow 294 295 type: Literal["CustomStateMigration"] 296 class_name: str = Field( 297 ..., 298 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 299 examples=["source_railz.components.MyCustomStateMigration"], 300 title="Class Name", 301 ) 302 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 303 304 305class CustomTransformation(BaseModel): 306 class Config: 307 extra = Extra.allow 308 309 type: Literal["CustomTransformation"] 310 class_name: str = Field( 311 ..., 312 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 313 examples=["source_railz.components.MyCustomTransformation"], 314 title="Class Name", 315 ) 316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 317 318 319class LegacyToPerPartitionStateMigration(BaseModel): 320 class Config: 321 extra = Extra.allow 322 323 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 324 325 326class Clamping(BaseModel): 327 target: str = Field( 328 ..., 329 description="The period of time that datetime windows will be clamped by", 330 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 331 title="Target", 332 ) 333 target_details: Optional[Dict[str, Any]] = None 334 335 336class Algorithm(Enum): 337 HS256 = "HS256" 338 HS384 = "HS384" 339 HS512 = "HS512" 340 ES256 = "ES256" 341 ES256K = "ES256K" 342 ES384 = "ES384" 343 ES512 = "ES512" 344 RS256 = "RS256" 345 RS384 = "RS384" 346 RS512 = "RS512" 347 PS256 = "PS256" 348 PS384 = "PS384" 349 PS512 = "PS512" 350 EdDSA = "EdDSA" 351 352 353class JwtHeaders(BaseModel): 354 class Config: 355 extra = Extra.forbid 356 357 kid: Optional[str] = Field( 358 None, 359 description="Private key ID for user account.", 360 examples=["{{ config['kid'] }}"], 361 title="Key Identifier", 362 ) 363 typ: Optional[str] = Field( 364 "JWT", 365 description="The media type of the complete JWT.", 366 examples=["JWT"], 367 title="Type", 368 ) 369 cty: Optional[str] = Field( 370 None, 371 description="Content type of JWT header.", 372 examples=["JWT"], 373 title="Content Type", 374 ) 375 376 377class JwtPayload(BaseModel): 378 class Config: 379 extra = Extra.forbid 380 381 iss: Optional[str] = Field( 382 None, 383 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 384 examples=["{{ config['iss'] }}"], 385 title="Issuer", 386 ) 387 sub: Optional[str] = Field( 388 None, 389 description="The subject of the JWT. Commonly defined by the API.", 390 title="Subject", 391 ) 392 aud: Optional[str] = Field( 393 None, 394 description="The recipient that the JWT is intended for. Commonly defined by the API.", 395 examples=["appstoreconnect-v1"], 396 title="Audience", 397 ) 398 399 400class RefreshTokenUpdater(BaseModel): 401 refresh_token_name: Optional[str] = Field( 402 "refresh_token", 403 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 404 examples=["refresh_token"], 405 title="Refresh Token Property Name", 406 ) 407 access_token_config_path: Optional[List[str]] = Field( 408 ["credentials", "access_token"], 409 description="Config path to the access token. Make sure the field actually exists in the config.", 410 examples=[["credentials", "access_token"], ["access_token"]], 411 title="Config Path To Access Token", 412 ) 413 refresh_token_config_path: Optional[List[str]] = Field( 414 ["credentials", "refresh_token"], 415 description="Config path to the access token. Make sure the field actually exists in the config.", 416 examples=[["credentials", "refresh_token"], ["refresh_token"]], 417 title="Config Path To Refresh Token", 418 ) 419 token_expiry_date_config_path: Optional[List[str]] = Field( 420 ["credentials", "token_expiry_date"], 421 description="Config path to the expiry date. Make sure actually exists in the config.", 422 examples=[["credentials", "token_expiry_date"]], 423 title="Config Path To Expiry Date", 424 ) 425 refresh_token_error_status_codes: Optional[List[int]] = Field( 426 [], 427 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 428 examples=[[400, 500]], 429 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 430 ) 431 refresh_token_error_key: Optional[str] = Field( 432 "", 433 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 434 examples=["error"], 435 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 436 ) 437 refresh_token_error_values: Optional[List[str]] = Field( 438 [], 439 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 440 examples=[["invalid_grant", "invalid_permissions"]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 442 ) 443 444 445class Rate(BaseModel): 446 class Config: 447 extra = Extra.allow 448 449 limit: Union[int, str] = Field( 450 ..., 451 description="The maximum number of calls allowed within the interval.", 452 title="Limit", 453 ) 454 interval: str = Field( 455 ..., 456 description="The time interval for the rate limit.", 457 examples=["PT1H", "P1D"], 458 title="Interval", 459 ) 460 461 462class HttpRequestRegexMatcher(BaseModel): 463 class Config: 464 extra = Extra.allow 465 466 method: Optional[str] = Field( 467 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 468 ) 469 url_base: Optional[str] = Field( 470 None, 471 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 472 title="URL Base", 473 ) 474 url_path_pattern: Optional[str] = Field( 475 None, 476 description="A regular expression pattern to match the URL path.", 477 title="URL Path Pattern", 478 ) 479 params: Optional[Dict[str, Any]] = Field( 480 None, description="The query parameters to match.", title="Parameters" 481 ) 482 headers: Optional[Dict[str, Any]] = Field( 483 None, description="The headers to match.", title="Headers" 484 ) 485 486 487class DpathExtractor(BaseModel): 488 type: Literal["DpathExtractor"] 489 field_path: List[str] = Field( 490 ..., 491 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 492 examples=[ 493 ["data"], 494 ["data", "records"], 495 ["data", "{{ parameters.name }}"], 496 ["data", "*", "record"], 497 ], 498 title="Field Path", 499 ) 500 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 501 502 503class ResponseToFileExtractor(BaseModel): 504 type: Literal["ResponseToFileExtractor"] 505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 506 507 508class ExponentialBackoffStrategy(BaseModel): 509 type: Literal["ExponentialBackoffStrategy"] 510 factor: Optional[Union[float, str]] = Field( 511 5, 512 description="Multiplicative constant applied on each retry.", 513 examples=[5, 5.5, "10"], 514 title="Factor", 515 ) 516 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 517 518 519class GroupByKeyMergeStrategy(BaseModel): 520 type: Literal["GroupByKeyMergeStrategy"] 521 key: Union[str, List[str]] = Field( 522 ..., 523 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 524 examples=["id", ["parent_id", "end_date"]], 525 title="Key", 526 ) 527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 528 529 530class SessionTokenRequestBearerAuthenticator(BaseModel): 531 type: Literal["Bearer"] 532 533 534class HttpMethod(Enum): 535 GET = "GET" 536 POST = "POST" 537 538 539class Action(Enum): 540 SUCCESS = "SUCCESS" 541 FAIL = "FAIL" 542 RETRY = "RETRY" 543 IGNORE = "IGNORE" 544 RESET_PAGINATION = "RESET_PAGINATION" 545 RATE_LIMITED = "RATE_LIMITED" 546 547 548class FailureType(Enum): 549 system_error = "system_error" 550 config_error = "config_error" 551 transient_error = "transient_error" 552 553 554class HttpResponseFilter(BaseModel): 555 type: Literal["HttpResponseFilter"] 556 action: Optional[Action] = Field( 557 None, 558 description="Action to execute if a response matches the filter.", 559 examples=[ 560 "SUCCESS", 561 "FAIL", 562 "RETRY", 563 "IGNORE", 564 "RESET_PAGINATION", 565 "RATE_LIMITED", 566 ], 567 title="Action", 568 ) 569 failure_type: Optional[FailureType] = Field( 570 None, 571 description="Failure type of traced exception if a response matches the filter.", 572 examples=["system_error", "config_error", "transient_error"], 573 title="Failure Type", 574 ) 575 error_message: Optional[str] = Field( 576 None, 577 description="Error Message to display if the response matches the filter.", 578 title="Error Message", 579 ) 580 error_message_contains: Optional[str] = Field( 581 None, 582 description="Match the response if its error message contains the substring.", 583 example=["This API operation is not enabled for this site"], 584 title="Error Message Substring", 585 ) 586 http_codes: Optional[List[int]] = Field( 587 None, 588 description="Match the response if its HTTP code is included in this list.", 589 examples=[[420, 429], [500]], 590 title="HTTP Codes", 591 unique_items=True, 592 ) 593 predicate: Optional[str] = Field( 594 None, 595 description="Match the response if the predicate evaluates to true.", 596 examples=[ 597 "{{ 'Too much requests' in response }}", 598 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 599 ], 600 title="Predicate", 601 ) 602 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 603 604 605class ComplexFieldType(BaseModel): 606 field_type: str 607 items: Optional[Union[str, ComplexFieldType]] = None 608 609 610class TypesMap(BaseModel): 611 target_type: Union[str, List[str], ComplexFieldType] 612 current_type: Union[str, List[str]] 613 condition: Optional[str] = None 614 615 616class SchemaTypeIdentifier(BaseModel): 617 type: Optional[Literal["SchemaTypeIdentifier"]] = None 618 schema_pointer: Optional[List[str]] = Field( 619 [], 620 description="List of nested fields defining the schema field path to extract. Defaults to [].", 621 title="Schema Path", 622 ) 623 key_pointer: List[str] = Field( 624 ..., 625 description="List of potentially nested fields describing the full path of the field key to extract.", 626 title="Key Path", 627 ) 628 type_pointer: Optional[List[str]] = Field( 629 None, 630 description="List of potentially nested fields describing the full path of the field type to extract.", 631 title="Type Path", 632 ) 633 types_mapping: Optional[List[TypesMap]] = None 634 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 635 636 637class InlineSchemaLoader(BaseModel): 638 type: Literal["InlineSchemaLoader"] 639 schema_: Optional[Dict[str, Any]] = Field( 640 None, 641 alias="schema", 642 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 643 title="Schema", 644 ) 645 646 647class JsonFileSchemaLoader(BaseModel): 648 type: Literal["JsonFileSchemaLoader"] 649 file_path: Optional[str] = Field( 650 None, 651 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 652 example=["./schemas/users.json"], 653 title="File Path", 654 ) 655 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 656 657 658class JsonDecoder(BaseModel): 659 type: Literal["JsonDecoder"] 660 661 662class JsonlDecoder(BaseModel): 663 type: Literal["JsonlDecoder"] 664 665 666class KeysToLower(BaseModel): 667 type: Literal["KeysToLower"] 668 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 669 670 671class KeysToSnakeCase(BaseModel): 672 type: Literal["KeysToSnakeCase"] 673 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 674 675 676class FlattenFields(BaseModel): 677 type: Literal["FlattenFields"] 678 flatten_lists: Optional[bool] = Field( 679 True, 680 description="Whether to flatten lists or leave it as is. Default is True.", 681 title="Flatten Lists", 682 ) 683 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 684 685 686class KeyTransformation(BaseModel): 687 type: Literal["KeyTransformation"] 688 prefix: Optional[str] = Field( 689 None, 690 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 691 examples=["flattened_"], 692 title="Key Prefix", 693 ) 694 suffix: Optional[str] = Field( 695 None, 696 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 697 examples=["_flattened"], 698 title="Key Suffix", 699 ) 700 701 702class DpathFlattenFields(BaseModel): 703 type: Literal["DpathFlattenFields"] 704 field_path: List[str] = Field( 705 ..., 706 description="A path to field that needs to be flattened.", 707 examples=[["data"], ["data", "*", "field"]], 708 title="Field Path", 709 ) 710 delete_origin_value: Optional[bool] = Field( 711 None, 712 description="Whether to delete the origin value or keep it. Default is False.", 713 title="Delete Origin Value", 714 ) 715 replace_record: Optional[bool] = Field( 716 None, 717 description="Whether to replace the origin record or not. Default is False.", 718 title="Replace Origin Record", 719 ) 720 key_transformation: Optional[KeyTransformation] = Field( 721 None, 722 description="Transformation for object keys. If not provided, original key will be used.", 723 title="Key transformation", 724 ) 725 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 726 727 728class KeysReplace(BaseModel): 729 type: Literal["KeysReplace"] 730 old: str = Field( 731 ..., 732 description="Old value to replace.", 733 examples=[ 734 " ", 735 "{{ record.id }}", 736 "{{ config['id'] }}", 737 "{{ stream_slice['id'] }}", 738 ], 739 title="Old value", 740 ) 741 new: str = Field( 742 ..., 743 description="New value to set.", 744 examples=[ 745 "_", 746 "{{ record.id }}", 747 "{{ config['id'] }}", 748 "{{ stream_slice['id'] }}", 749 ], 750 title="New value", 751 ) 752 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 753 754 755class IterableDecoder(BaseModel): 756 type: Literal["IterableDecoder"] 757 758 759class XmlDecoder(BaseModel): 760 type: Literal["XmlDecoder"] 761 762 763class CustomDecoder(BaseModel): 764 class Config: 765 extra = Extra.allow 766 767 type: Literal["CustomDecoder"] 768 class_name: str = Field( 769 ..., 770 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 771 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 772 title="Class Name", 773 ) 774 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 775 776 777class MinMaxDatetime(BaseModel): 778 type: Literal["MinMaxDatetime"] 779 datetime: str = Field( 780 ..., 781 description="Datetime value.", 782 examples=[ 783 "2021-01-01", 784 "2021-01-01T00:00:00Z", 785 "{{ config['start_time'] }}", 786 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 787 ], 788 title="Datetime", 789 ) 790 datetime_format: Optional[str] = Field( 791 "", 792 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 793 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 794 title="Datetime Format", 795 ) 796 max_datetime: Optional[str] = Field( 797 None, 798 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 799 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 800 title="Max Datetime", 801 ) 802 min_datetime: Optional[str] = Field( 803 None, 804 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 805 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 806 title="Min Datetime", 807 ) 808 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 809 810 811class NoAuth(BaseModel): 812 type: Literal["NoAuth"] 813 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 814 815 816class NoPagination(BaseModel): 817 type: Literal["NoPagination"] 818 819 820class State(BaseModel): 821 class Config: 822 extra = Extra.allow 823 824 min: int 825 max: int 826 827 828class OauthConnectorInputSpecification(BaseModel): 829 class Config: 830 extra = Extra.allow 831 832 consent_url: str = Field( 833 ..., 834 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 835 examples=[ 836 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 837 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 838 ], 839 title="Consent URL", 840 ) 841 scope: Optional[str] = Field( 842 None, 843 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 844 examples=["user:read user:read_orders workspaces:read"], 845 title="Scopes", 846 ) 847 access_token_url: str = Field( 848 ..., 849 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 850 examples=[ 851 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 852 ], 853 title="Access Token URL", 854 ) 855 access_token_headers: Optional[Dict[str, Any]] = Field( 856 None, 857 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 858 examples=[ 859 { 860 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 861 } 862 ], 863 title="Access Token Headers", 864 ) 865 access_token_params: Optional[Dict[str, Any]] = Field( 866 None, 867 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 868 examples=[ 869 { 870 "{{ auth_code_key }}": "{{ auth_code_value }}", 871 "{{ client_id_key }}": "{{ client_id_value }}", 872 "{{ client_secret_key }}": "{{ client_secret_value }}", 873 } 874 ], 875 title="Access Token Query Params (Json Encoded)", 876 ) 877 extract_output: Optional[List[str]] = Field( 878 None, 879 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 880 examples=[["access_token", "refresh_token", "other_field"]], 881 title="Extract Output", 882 ) 883 state: Optional[State] = Field( 884 None, 885 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 886 examples=[{"min": 7, "max": 128}], 887 title="Configurable State Query Param", 888 ) 889 client_id_key: Optional[str] = Field( 890 None, 891 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 892 examples=["my_custom_client_id_key_name"], 893 title="Client ID Key Override", 894 ) 895 client_secret_key: Optional[str] = Field( 896 None, 897 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 898 examples=["my_custom_client_secret_key_name"], 899 title="Client Secret Key Override", 900 ) 901 scope_key: Optional[str] = Field( 902 None, 903 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 904 examples=["my_custom_scope_key_key_name"], 905 title="Scopes Key Override", 906 ) 907 state_key: Optional[str] = Field( 908 None, 909 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 910 examples=["my_custom_state_key_key_name"], 911 title="State Key Override", 912 ) 913 auth_code_key: Optional[str] = Field( 914 None, 915 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 916 examples=["my_custom_auth_code_key_name"], 917 title="Auth Code Key Override", 918 ) 919 redirect_uri_key: Optional[str] = Field( 920 None, 921 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 922 examples=["my_custom_redirect_uri_key_name"], 923 title="Redirect URI Key Override", 924 ) 925 926 927class OAuthConfigSpecification(BaseModel): 928 class Config: 929 extra = Extra.allow 930 931 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 932 None, 933 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 934 examples=[ 935 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 936 { 937 "app_id": { 938 "type": "string", 939 "path_in_connector_config": ["info", "app_id"], 940 } 941 }, 942 ], 943 title="OAuth user input", 944 ) 945 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 946 None, 947 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 948 title="DeclarativeOAuth Connector Specification", 949 ) 950 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 951 None, 952 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 953 examples=[ 954 { 955 "refresh_token": { 956 "type": "string,", 957 "path_in_connector_config": ["credentials", "refresh_token"], 958 } 959 } 960 ], 961 title="OAuth output specification", 962 ) 963 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 964 None, 965 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 966 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 967 title="OAuth input specification", 968 ) 969 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 970 None, 971 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 972 examples=[ 973 { 974 "client_id": { 975 "type": "string,", 976 "path_in_connector_config": ["credentials", "client_id"], 977 }, 978 "client_secret": { 979 "type": "string,", 980 "path_in_connector_config": ["credentials", "client_secret"], 981 }, 982 } 983 ], 984 title="OAuth server output specification", 985 ) 986 987 988class OffsetIncrement(BaseModel): 989 type: Literal["OffsetIncrement"] 990 page_size: Optional[Union[int, str]] = Field( 991 None, 992 description="The number of records to include in each pages.", 993 examples=[100, "{{ config['page_size'] }}"], 994 title="Limit", 995 ) 996 inject_on_first_request: Optional[bool] = Field( 997 False, 998 description="Using the `offset` with value `0` during the first request", 999 title="Inject Offset on First Request", 1000 ) 1001 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1002 1003 1004class PageIncrement(BaseModel): 1005 type: Literal["PageIncrement"] 1006 page_size: Optional[Union[int, str]] = Field( 1007 None, 1008 description="The number of records to include in each pages.", 1009 examples=[100, "100", "{{ config['page_size'] }}"], 1010 title="Page Size", 1011 ) 1012 start_from_page: Optional[int] = Field( 1013 0, 1014 description="Index of the first page to request.", 1015 examples=[0, 1], 1016 title="Start From Page", 1017 ) 1018 inject_on_first_request: Optional[bool] = Field( 1019 False, 1020 description="Using the `page number` with value defined by `start_from_page` during the first request", 1021 title="Inject Page Number on First Request", 1022 ) 1023 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1024 1025 1026class PrimaryKey(BaseModel): 1027 __root__: Union[str, List[str], List[List[str]]] = Field( 1028 ..., 1029 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1030 examples=["id", ["code", "type"]], 1031 title="Primary Key", 1032 ) 1033 1034 1035class PropertyLimitType(Enum): 1036 characters = "characters" 1037 property_count = "property_count" 1038 1039 1040class PropertyChunking(BaseModel): 1041 type: Literal["PropertyChunking"] 1042 property_limit_type: PropertyLimitType = Field( 1043 ..., 1044 description="The type used to determine the maximum number of properties per chunk", 1045 title="Property Limit Type", 1046 ) 1047 property_limit: Optional[int] = Field( 1048 None, 1049 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1050 title="Property Limit", 1051 ) 1052 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1053 None, 1054 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1055 title="Record Merge Strategy", 1056 ) 1057 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1058 1059 1060class RecordFilter(BaseModel): 1061 type: Literal["RecordFilter"] 1062 condition: Optional[str] = Field( 1063 "", 1064 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1065 examples=[ 1066 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1067 "{{ record.status in ['active', 'expired'] }}", 1068 ], 1069 title="Condition", 1070 ) 1071 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1072 1073 1074class SchemaNormalization(Enum): 1075 Default = "Default" 1076 None_ = "None" 1077 1078 1079class RemoveFields(BaseModel): 1080 type: Literal["RemoveFields"] 1081 condition: Optional[str] = Field( 1082 "", 1083 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1084 examples=[ 1085 "{{ property|string == '' }}", 1086 "{{ property is integer }}", 1087 "{{ property|length > 5 }}", 1088 "{{ property == 'some_string_to_match' }}", 1089 ], 1090 ) 1091 field_pointers: List[List[str]] = Field( 1092 ..., 1093 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1094 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1095 title="Field Paths", 1096 ) 1097 1098 1099class RequestPath(BaseModel): 1100 type: Literal["RequestPath"] 1101 1102 1103class InjectInto(Enum): 1104 request_parameter = "request_parameter" 1105 header = "header" 1106 body_data = "body_data" 1107 body_json = "body_json" 1108 1109 1110class RequestOption(BaseModel): 1111 type: Literal["RequestOption"] 1112 inject_into: InjectInto = Field( 1113 ..., 1114 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1115 examples=["request_parameter", "header", "body_data", "body_json"], 1116 title="Inject Into", 1117 ) 1118 field_name: Optional[str] = Field( 1119 None, 1120 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1121 examples=["segment_id"], 1122 title="Field Name", 1123 ) 1124 field_path: Optional[List[str]] = Field( 1125 None, 1126 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1127 examples=[["data", "viewer", "id"]], 1128 title="Field Path", 1129 ) 1130 1131 1132class Schemas(BaseModel): 1133 pass 1134 1135 class Config: 1136 extra = Extra.allow 1137 1138 1139class LegacySessionTokenAuthenticator(BaseModel): 1140 type: Literal["LegacySessionTokenAuthenticator"] 1141 header: str = Field( 1142 ..., 1143 description="The name of the session token header that will be injected in the request", 1144 examples=["X-Session"], 1145 title="Session Request Header", 1146 ) 1147 login_url: str = Field( 1148 ..., 1149 description="Path of the login URL (do not include the base URL)", 1150 examples=["session"], 1151 title="Login Path", 1152 ) 1153 session_token: Optional[str] = Field( 1154 None, 1155 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1156 example=["{{ config['session_token'] }}"], 1157 title="Session Token", 1158 ) 1159 session_token_response_key: str = Field( 1160 ..., 1161 description="Name of the key of the session token to be extracted from the response", 1162 examples=["id"], 1163 title="Response Token Response Key", 1164 ) 1165 username: Optional[str] = Field( 1166 None, 1167 description="Username used to authenticate and obtain a session token", 1168 examples=[" {{ config['username'] }}"], 1169 title="Username", 1170 ) 1171 password: Optional[str] = Field( 1172 "", 1173 description="Password used to authenticate and obtain a session token", 1174 examples=["{{ config['password'] }}", ""], 1175 title="Password", 1176 ) 1177 validate_session_url: str = Field( 1178 ..., 1179 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1180 examples=["user/current"], 1181 title="Validate Session Path", 1182 ) 1183 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1184 1185 1186class Action1(Enum): 1187 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1188 RESET = "RESET" 1189 1190 1191class PaginationResetLimits(BaseModel): 1192 type: Literal["PaginationResetLimits"] 1193 number_of_records: Optional[int] = None 1194 1195 1196class CsvDecoder(BaseModel): 1197 type: Literal["CsvDecoder"] 1198 encoding: Optional[str] = "utf-8" 1199 delimiter: Optional[str] = "," 1200 set_values_to_none: Optional[List[str]] = None 1201 1202 1203class AsyncJobStatusMap(BaseModel): 1204 type: Optional[Literal["AsyncJobStatusMap"]] = None 1205 running: List[str] 1206 completed: List[str] 1207 failed: List[str] 1208 timeout: List[str] 1209 1210 1211class ValueType(Enum): 1212 string = "string" 1213 number = "number" 1214 integer = "integer" 1215 boolean = "boolean" 1216 1217 1218class WaitTimeFromHeader(BaseModel): 1219 type: Literal["WaitTimeFromHeader"] 1220 header: str = Field( 1221 ..., 1222 description="The name of the response header defining how long to wait before retrying.", 1223 examples=["Retry-After"], 1224 title="Response Header Name", 1225 ) 1226 regex: Optional[str] = Field( 1227 None, 1228 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1229 examples=["([-+]?\\d+)"], 1230 title="Extraction Regex", 1231 ) 1232 max_waiting_time_in_seconds: Optional[float] = Field( 1233 None, 1234 description="Given the value extracted from the header is greater than this value, stop the stream.", 1235 examples=[3600], 1236 title="Max Waiting Time in Seconds", 1237 ) 1238 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1239 1240 1241class WaitUntilTimeFromHeader(BaseModel): 1242 type: Literal["WaitUntilTimeFromHeader"] 1243 header: str = Field( 1244 ..., 1245 description="The name of the response header defining how long to wait before retrying.", 1246 examples=["wait_time"], 1247 title="Response Header", 1248 ) 1249 min_wait: Optional[Union[float, str]] = Field( 1250 None, 1251 description="Minimum time to wait before retrying.", 1252 examples=[10, "60"], 1253 title="Minimum Wait Time", 1254 ) 1255 regex: Optional[str] = Field( 1256 None, 1257 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1258 examples=["([-+]?\\d+)"], 1259 title="Extraction Regex", 1260 ) 1261 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1262 1263 1264class ComponentMappingDefinition(BaseModel): 1265 type: Literal["ComponentMappingDefinition"] 1266 field_path: List[str] = Field( 1267 ..., 1268 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1269 examples=[ 1270 ["name"], 1271 ["retriever", "requester", "url"], 1272 ["retriever", "requester", "{{ components_values.field }}"], 1273 ["*", "**", "name"], 1274 ], 1275 title="Field Path", 1276 ) 1277 value: str = Field( 1278 ..., 1279 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1280 examples=[ 1281 "{{ components_values['updates'] }}", 1282 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1283 "{{ config['segment_id'] }}", 1284 "{{ stream_slice['parent_id'] }}", 1285 "{{ stream_slice['extra_fields']['name'] }}", 1286 ], 1287 title="Value", 1288 ) 1289 value_type: Optional[ValueType] = Field( 1290 None, 1291 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1292 title="Value Type", 1293 ) 1294 create_or_update: Optional[bool] = Field( 1295 False, 1296 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1297 title="Create or Update", 1298 ) 1299 condition: Optional[str] = Field( 1300 None, 1301 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1302 examples=[ 1303 "{{ components_values.get('cursor_field', None) }}", 1304 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1305 ], 1306 title="Condition", 1307 ) 1308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1309 1310 1311class StreamConfig(BaseModel): 1312 type: Literal["StreamConfig"] 1313 configs_pointer: List[str] = Field( 1314 ..., 1315 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1316 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1317 title="Configs Pointer", 1318 ) 1319 default_values: Optional[List[Dict[str, Any]]] = Field( 1320 None, 1321 description="A list of default values, each matching the structure expected from the parsed component value.", 1322 title="Default Values", 1323 ) 1324 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1325 1326 1327class ConfigComponentsResolver(BaseModel): 1328 type: Literal["ConfigComponentsResolver"] 1329 stream_config: Union[List[StreamConfig], StreamConfig] 1330 components_mapping: List[ComponentMappingDefinition] 1331 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1332 1333 1334class StreamParametersDefinition(BaseModel): 1335 type: Literal["StreamParametersDefinition"] 1336 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1337 ..., 1338 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1339 examples=[ 1340 [ 1341 { 1342 "name": "test stream", 1343 "$parameters": {"entity": "test entity"}, 1344 "primary_key": "test key", 1345 } 1346 ] 1347 ], 1348 title="Stream Parameters", 1349 ) 1350 1351 1352class ParametrizedComponentsResolver(BaseModel): 1353 type: Literal["ParametrizedComponentsResolver"] 1354 stream_parameters: StreamParametersDefinition 1355 components_mapping: List[ComponentMappingDefinition] 1356 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1357 1358 1359class RequestBodyPlainText(BaseModel): 1360 type: Literal["RequestBodyPlainText"] 1361 value: str 1362 1363 1364class RequestBodyUrlEncodedForm(BaseModel): 1365 type: Literal["RequestBodyUrlEncodedForm"] 1366 value: Dict[str, str] 1367 1368 1369class RequestBodyJsonObject(BaseModel): 1370 type: Literal["RequestBodyJsonObject"] 1371 value: Dict[str, Any] 1372 1373 1374class RequestBodyGraphQlQuery(BaseModel): 1375 class Config: 1376 extra = Extra.allow 1377 1378 query: str = Field(..., description="The GraphQL query to be executed") 1379 1380 1381class ValidateAdheresToSchema(BaseModel): 1382 type: Literal["ValidateAdheresToSchema"] 1383 base_schema: Union[str, Dict[str, Any]] = Field( 1384 ..., 1385 description="The base JSON schema against which the user-provided schema will be validated.", 1386 examples=[ 1387 "{{ config['report_validation_schema'] }}", 1388 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1389 { 1390 "$schema": "http://json-schema.org/draft-07/schema#", 1391 "title": "Person", 1392 "type": "object", 1393 "properties": { 1394 "name": {"type": "string", "description": "The person's name"}, 1395 "age": { 1396 "type": "integer", 1397 "minimum": 0, 1398 "description": "The person's age", 1399 }, 1400 }, 1401 "required": ["name", "age"], 1402 }, 1403 ], 1404 title="Base JSON Schema", 1405 ) 1406 1407 1408class CustomValidationStrategy(BaseModel): 1409 class Config: 1410 extra = Extra.allow 1411 1412 type: Literal["CustomValidationStrategy"] 1413 class_name: str = Field( 1414 ..., 1415 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1416 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1417 title="Class Name", 1418 ) 1419 1420 1421class ConfigRemapField(BaseModel): 1422 type: Literal["ConfigRemapField"] 1423 map: Union[Dict[str, Any], str] = Field( 1424 ..., 1425 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1426 examples=[ 1427 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1428 "{{ config['status_mapping'] }}", 1429 ], 1430 title="Value Mapping", 1431 ) 1432 field_path: List[str] = Field( 1433 ..., 1434 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1435 examples=[ 1436 ["status"], 1437 ["data", "status"], 1438 ["data", "{{ config.name }}", "status"], 1439 ["data", "*", "status"], 1440 ], 1441 title="Field Path", 1442 ) 1443 1444 1445class ConfigRemoveFields(BaseModel): 1446 type: Literal["ConfigRemoveFields"] 1447 field_pointers: List[List[str]] = Field( 1448 ..., 1449 description="A list of field pointers to be removed from the config.", 1450 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1451 title="Field Pointers", 1452 ) 1453 condition: Optional[str] = Field( 1454 "", 1455 description="Fields will be removed if expression is evaluated to True.", 1456 examples=[ 1457 "{{ config['environemnt'] == 'sandbox' }}", 1458 "{{ property is integer }}", 1459 "{{ property|length > 5 }}", 1460 "{{ property == 'some_string_to_match' }}", 1461 ], 1462 ) 1463 1464 1465class CustomConfigTransformation(BaseModel): 1466 type: Literal["CustomConfigTransformation"] 1467 class_name: str = Field( 1468 ..., 1469 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1470 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1471 ) 1472 parameters: Optional[Dict[str, Any]] = Field( 1473 None, 1474 alias="$parameters", 1475 description="Additional parameters to be passed to the custom config transformation.", 1476 ) 1477 1478 1479class AddedFieldDefinition(BaseModel): 1480 type: Literal["AddedFieldDefinition"] 1481 path: List[str] = Field( 1482 ..., 1483 description="List of strings defining the path where to add the value on the record.", 1484 examples=[["segment_id"], ["metadata", "segment_id"]], 1485 title="Path", 1486 ) 1487 value: str = Field( 1488 ..., 1489 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1490 examples=[ 1491 "{{ record['updates'] }}", 1492 "{{ record['MetaData']['LastUpdatedTime'] }}", 1493 "{{ stream_partition['segment_id'] }}", 1494 ], 1495 title="Value", 1496 ) 1497 value_type: Optional[ValueType] = Field( 1498 None, 1499 description="Type of the value. If not specified, the type will be inferred from the value.", 1500 title="Value Type", 1501 ) 1502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1503 1504 1505class AddFields(BaseModel): 1506 type: Literal["AddFields"] 1507 fields: List[AddedFieldDefinition] = Field( 1508 ..., 1509 description="List of transformations (path and corresponding value) that will be added to the record.", 1510 title="Fields", 1511 ) 1512 condition: Optional[str] = Field( 1513 "", 1514 description="Fields will be added if expression is evaluated to True.", 1515 examples=[ 1516 "{{ property|string == '' }}", 1517 "{{ property is integer }}", 1518 "{{ property|length > 5 }}", 1519 "{{ property == 'some_string_to_match' }}", 1520 ], 1521 ) 1522 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1523 1524 1525class ApiKeyAuthenticator(BaseModel): 1526 type: Literal["ApiKeyAuthenticator"] 1527 api_token: Optional[str] = Field( 1528 None, 1529 description="The API key to inject in the request. Fill it in the user inputs.", 1530 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1531 title="API Key", 1532 ) 1533 header: Optional[str] = Field( 1534 None, 1535 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1536 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1537 title="Header Name", 1538 ) 1539 inject_into: Optional[RequestOption] = Field( 1540 None, 1541 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1542 examples=[ 1543 {"inject_into": "header", "field_name": "Authorization"}, 1544 {"inject_into": "request_parameter", "field_name": "authKey"}, 1545 ], 1546 title="Inject API Key Into Outgoing HTTP Request", 1547 ) 1548 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1549 1550 1551class AuthFlow(BaseModel): 1552 auth_flow_type: Optional[AuthFlowType] = Field( 1553 None, description="The type of auth to use", title="Auth flow type" 1554 ) 1555 predicate_key: Optional[List[str]] = Field( 1556 None, 1557 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1558 examples=[["credentials", "auth_type"]], 1559 title="Predicate key", 1560 ) 1561 predicate_value: Optional[str] = Field( 1562 None, 1563 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1564 examples=["Oauth"], 1565 title="Predicate value", 1566 ) 1567 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1568 1569 1570class CheckStream(BaseModel): 1571 type: Literal["CheckStream"] 1572 stream_names: Optional[List[str]] = Field( 1573 None, 1574 description="Names of the streams to try reading from when running a check operation.", 1575 examples=[["users"], ["users", "contacts"]], 1576 title="Stream Names", 1577 ) 1578 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1579 1580 1581class IncrementingCountCursor(BaseModel): 1582 type: Literal["IncrementingCountCursor"] 1583 cursor_field: str = Field( 1584 ..., 1585 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1586 examples=["created_at", "{{ config['record_cursor'] }}"], 1587 title="Cursor Field", 1588 ) 1589 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1590 None, 1591 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1592 title="Allow Catalog Defined Cursor Field", 1593 ) 1594 start_value: Optional[Union[str, int]] = Field( 1595 None, 1596 description="The value that determines the earliest record that should be synced.", 1597 examples=[0, "{{ config['start_value'] }}"], 1598 title="Start Value", 1599 ) 1600 start_value_option: Optional[RequestOption] = Field( 1601 None, 1602 description="Optionally configures how the start value will be sent in requests to the source API.", 1603 title="Inject Start Value Into Outgoing HTTP Request", 1604 ) 1605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1606 1607 1608class DatetimeBasedCursor(BaseModel): 1609 type: Literal["DatetimeBasedCursor"] 1610 clamping: Optional[Clamping] = Field( 1611 None, 1612 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1613 title="Date Range Clamping", 1614 ) 1615 cursor_field: str = Field( 1616 ..., 1617 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1618 examples=["created_at", "{{ config['record_cursor'] }}"], 1619 title="Cursor Field", 1620 ) 1621 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1622 None, 1623 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1624 title="Allow Catalog Defined Cursor Field", 1625 ) 1626 cursor_datetime_formats: Optional[List[str]] = Field( 1627 None, 1628 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1629 examples=[ 1630 "%Y-%m-%d", 1631 "%Y-%m-%d %H:%M:%S", 1632 "%Y-%m-%dT%H:%M:%S", 1633 "%Y-%m-%dT%H:%M:%SZ", 1634 "%Y-%m-%dT%H:%M:%S%z", 1635 "%Y-%m-%dT%H:%M:%S.%fZ", 1636 "%Y-%m-%dT%H:%M:%S.%f%z", 1637 "%Y-%m-%d %H:%M:%S.%f+00:00", 1638 "%s", 1639 "%ms", 1640 ], 1641 title="Cursor Datetime Formats", 1642 ) 1643 start_datetime: Union[MinMaxDatetime, str] = Field( 1644 ..., 1645 description="The datetime that determines the earliest record that should be synced.", 1646 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1647 title="Start Datetime", 1648 ) 1649 start_time_option: Optional[RequestOption] = Field( 1650 None, 1651 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1652 title="Inject Start Time Into Outgoing HTTP Request", 1653 ) 1654 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1655 None, 1656 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1657 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1658 title="End Datetime", 1659 ) 1660 end_time_option: Optional[RequestOption] = Field( 1661 None, 1662 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1663 title="Inject End Time Into Outgoing HTTP Request", 1664 ) 1665 datetime_format: str = Field( 1666 ..., 1667 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1668 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1669 title="Outgoing Datetime Format", 1670 ) 1671 cursor_granularity: Optional[str] = Field( 1672 None, 1673 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1674 examples=["PT1S"], 1675 title="Cursor Granularity", 1676 ) 1677 is_data_feed: Optional[bool] = Field( 1678 None, 1679 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1680 title="Data Feed API", 1681 ) 1682 is_client_side_incremental: Optional[bool] = Field( 1683 None, 1684 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1685 title="Client-side Incremental Filtering", 1686 ) 1687 is_compare_strictly: Optional[bool] = Field( 1688 False, 1689 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1690 title="Strict Start-End Time Comparison", 1691 ) 1692 global_substream_cursor: Optional[bool] = Field( 1693 False, 1694 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1695 title="Global Substream Cursor", 1696 ) 1697 lookback_window: Optional[str] = Field( 1698 None, 1699 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1700 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1701 title="Lookback Window", 1702 ) 1703 partition_field_end: Optional[str] = Field( 1704 None, 1705 description="Name of the partition start time field.", 1706 examples=["ending_time"], 1707 title="Partition Field End", 1708 ) 1709 partition_field_start: Optional[str] = Field( 1710 None, 1711 description="Name of the partition end time field.", 1712 examples=["starting_time"], 1713 title="Partition Field Start", 1714 ) 1715 step: Optional[str] = Field( 1716 None, 1717 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1718 examples=["P1W", "{{ config['step_increment'] }}"], 1719 title="Step", 1720 ) 1721 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1722 1723 1724class JwtAuthenticator(BaseModel): 1725 type: Literal["JwtAuthenticator"] 1726 secret_key: str = Field( 1727 ..., 1728 description="Secret used to sign the JSON web token.", 1729 examples=["{{ config['secret_key'] }}"], 1730 title="Secret Key", 1731 ) 1732 base64_encode_secret_key: Optional[bool] = Field( 1733 False, 1734 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1735 title="Base64-encode Secret Key", 1736 ) 1737 algorithm: Algorithm = Field( 1738 ..., 1739 description="Algorithm used to sign the JSON web token.", 1740 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1741 title="Algorithm", 1742 ) 1743 token_duration: Optional[int] = Field( 1744 1200, 1745 description="The amount of time in seconds a JWT token can be valid after being issued.", 1746 examples=[1200, 3600], 1747 title="Token Duration", 1748 ) 1749 header_prefix: Optional[str] = Field( 1750 None, 1751 description="The prefix to be used within the Authentication header.", 1752 examples=["Bearer", "Basic"], 1753 title="Header Prefix", 1754 ) 1755 jwt_headers: Optional[JwtHeaders] = Field( 1756 None, 1757 description="JWT headers used when signing JSON web token.", 1758 title="JWT Headers", 1759 ) 1760 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1761 None, 1762 description="Additional headers to be included with the JWT headers object.", 1763 title="Additional JWT Headers", 1764 ) 1765 jwt_payload: Optional[JwtPayload] = Field( 1766 None, 1767 description="JWT Payload used when signing JSON web token.", 1768 title="JWT Payload", 1769 ) 1770 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1771 None, 1772 description="Additional properties to be added to the JWT payload.", 1773 title="Additional JWT Payload Properties", 1774 ) 1775 passphrase: Optional[str] = Field( 1776 None, 1777 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1778 examples=["{{ config['passphrase'] }}"], 1779 title="Passphrase", 1780 ) 1781 request_option: Optional[RequestOption] = Field( 1782 None, 1783 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1784 title="Request Option", 1785 ) 1786 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1787 1788 1789class OAuthAuthenticator(BaseModel): 1790 type: Literal["OAuthAuthenticator"] 1791 client_id_name: Optional[str] = Field( 1792 "client_id", 1793 description="The name of the property to use to refresh the `access_token`.", 1794 examples=["custom_app_id"], 1795 title="Client ID Property Name", 1796 ) 1797 client_id: Optional[str] = Field( 1798 None, 1799 description="The OAuth client ID. Fill it in the user inputs.", 1800 examples=[ 1801 "{{ config['client_id'] }}", 1802 "{{ config['credentials']['client_id }}", 1803 ], 1804 title="Client ID", 1805 ) 1806 client_secret_name: Optional[str] = Field( 1807 "client_secret", 1808 description="The name of the property to use to refresh the `access_token`.", 1809 examples=["custom_app_secret"], 1810 title="Client Secret Property Name", 1811 ) 1812 client_secret: Optional[str] = Field( 1813 None, 1814 description="The OAuth client secret. Fill it in the user inputs.", 1815 examples=[ 1816 "{{ config['client_secret'] }}", 1817 "{{ config['credentials']['client_secret }}", 1818 ], 1819 title="Client Secret", 1820 ) 1821 refresh_token_name: Optional[str] = Field( 1822 "refresh_token", 1823 description="The name of the property to use to refresh the `access_token`.", 1824 examples=["custom_app_refresh_value"], 1825 title="Refresh Token Property Name", 1826 ) 1827 refresh_token: Optional[str] = Field( 1828 None, 1829 description="Credential artifact used to get a new access token.", 1830 examples=[ 1831 "{{ config['refresh_token'] }}", 1832 "{{ config['credentials]['refresh_token'] }}", 1833 ], 1834 title="Refresh Token", 1835 ) 1836 token_refresh_endpoint: Optional[str] = Field( 1837 None, 1838 description="The full URL to call to obtain a new access token.", 1839 examples=["https://connect.squareup.com/oauth2/token"], 1840 title="Token Refresh Endpoint", 1841 ) 1842 access_token_name: Optional[str] = Field( 1843 "access_token", 1844 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1845 examples=["access_token"], 1846 title="Access Token Property Name", 1847 ) 1848 access_token_value: Optional[str] = Field( 1849 None, 1850 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1851 examples=["secret_access_token_value"], 1852 title="Access Token Value", 1853 ) 1854 expires_in_name: Optional[str] = Field( 1855 "expires_in", 1856 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1857 examples=["expires_in"], 1858 title="Token Expiry Property Name", 1859 ) 1860 grant_type_name: Optional[str] = Field( 1861 "grant_type", 1862 description="The name of the property to use to refresh the `access_token`.", 1863 examples=["custom_grant_type"], 1864 title="Grant Type Property Name", 1865 ) 1866 grant_type: Optional[str] = Field( 1867 "refresh_token", 1868 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1869 examples=["refresh_token", "client_credentials"], 1870 title="Grant Type", 1871 ) 1872 refresh_request_body: Optional[Dict[str, Any]] = Field( 1873 None, 1874 description="Body of the request sent to get a new access token.", 1875 examples=[ 1876 { 1877 "applicationId": "{{ config['application_id'] }}", 1878 "applicationSecret": "{{ config['application_secret'] }}", 1879 "token": "{{ config['token'] }}", 1880 } 1881 ], 1882 title="Refresh Request Body", 1883 ) 1884 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1885 None, 1886 description="Headers of the request sent to get a new access token.", 1887 examples=[ 1888 { 1889 "Authorization": "<AUTH_TOKEN>", 1890 "Content-Type": "application/x-www-form-urlencoded", 1891 } 1892 ], 1893 title="Refresh Request Headers", 1894 ) 1895 scopes: Optional[List[str]] = Field( 1896 None, 1897 description="List of scopes that should be granted to the access token.", 1898 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1899 title="Scopes", 1900 ) 1901 token_expiry_date: Optional[str] = Field( 1902 None, 1903 description="The access token expiry date.", 1904 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1905 title="Token Expiry Date", 1906 ) 1907 token_expiry_date_format: Optional[str] = Field( 1908 None, 1909 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1910 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1911 title="Token Expiry Date Format", 1912 ) 1913 refresh_token_error_status_codes: Optional[List[int]] = Field( 1914 None, 1915 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1916 examples=[[400, 500]], 1917 title="Refresh Token Error Status Codes", 1918 ) 1919 refresh_token_error_key: Optional[str] = Field( 1920 None, 1921 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1922 examples=["error"], 1923 title="Refresh Token Error Key", 1924 ) 1925 refresh_token_error_values: Optional[List[str]] = Field( 1926 None, 1927 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1928 examples=[["invalid_grant", "invalid_permissions"]], 1929 title="Refresh Token Error Values", 1930 ) 1931 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1932 None, 1933 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1934 title="Refresh Token Updater", 1935 ) 1936 profile_assertion: Optional[JwtAuthenticator] = Field( 1937 None, 1938 description="The authenticator being used to authenticate the client authenticator.", 1939 title="Profile Assertion", 1940 ) 1941 use_profile_assertion: Optional[bool] = Field( 1942 False, 1943 description="Enable using profile assertion as a flow for OAuth authorization.", 1944 title="Use Profile Assertion", 1945 ) 1946 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1947 1948 1949class FixedWindowCallRatePolicy(BaseModel): 1950 class Config: 1951 extra = Extra.allow 1952 1953 type: Literal["FixedWindowCallRatePolicy"] 1954 period: str = Field( 1955 ..., description="The time interval for the rate limit window.", title="Period" 1956 ) 1957 call_limit: int = Field( 1958 ..., 1959 description="The maximum number of calls allowed within the period.", 1960 title="Call Limit", 1961 ) 1962 matchers: List[HttpRequestRegexMatcher] = Field( 1963 ..., 1964 description="List of matchers that define which requests this policy applies to.", 1965 title="Matchers", 1966 ) 1967 1968 1969class MovingWindowCallRatePolicy(BaseModel): 1970 class Config: 1971 extra = Extra.allow 1972 1973 type: Literal["MovingWindowCallRatePolicy"] 1974 rates: List[Rate] = Field( 1975 ..., 1976 description="List of rates that define the call limits for different time intervals.", 1977 title="Rates", 1978 ) 1979 matchers: List[HttpRequestRegexMatcher] = Field( 1980 ..., 1981 description="List of matchers that define which requests this policy applies to.", 1982 title="Matchers", 1983 ) 1984 1985 1986class UnlimitedCallRatePolicy(BaseModel): 1987 class Config: 1988 extra = Extra.allow 1989 1990 type: Literal["UnlimitedCallRatePolicy"] 1991 matchers: List[HttpRequestRegexMatcher] = Field( 1992 ..., 1993 description="List of matchers that define which requests this policy applies to.", 1994 title="Matchers", 1995 ) 1996 1997 1998class DefaultErrorHandler(BaseModel): 1999 type: Literal["DefaultErrorHandler"] 2000 backoff_strategies: Optional[ 2001 List[ 2002 Union[ 2003 ConstantBackoffStrategy, 2004 ExponentialBackoffStrategy, 2005 WaitTimeFromHeader, 2006 WaitUntilTimeFromHeader, 2007 CustomBackoffStrategy, 2008 ] 2009 ] 2010 ] = Field( 2011 None, 2012 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2013 title="Backoff Strategies", 2014 ) 2015 max_retries: Optional[int] = Field( 2016 5, 2017 description="The maximum number of time to retry a retryable request before giving up and failing.", 2018 examples=[5, 0, 10], 2019 title="Max Retry Count", 2020 ) 2021 response_filters: Optional[List[HttpResponseFilter]] = Field( 2022 None, 2023 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2024 title="Response Filters", 2025 ) 2026 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2027 2028 2029class DefaultPaginator(BaseModel): 2030 type: Literal["DefaultPaginator"] 2031 pagination_strategy: Union[ 2032 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2033 ] = Field( 2034 ..., 2035 description="Strategy defining how records are paginated.", 2036 title="Pagination Strategy", 2037 ) 2038 page_size_option: Optional[RequestOption] = Field( 2039 None, title="Inject Page Size Into Outgoing HTTP Request" 2040 ) 2041 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2042 None, 2043 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2044 title="Inject Page Token Into Outgoing HTTP Request", 2045 ) 2046 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2047 2048 2049class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2050 type: Literal["ApiKey"] 2051 inject_into: RequestOption = Field( 2052 ..., 2053 description="Configure how the API Key will be sent in requests to the source API.", 2054 examples=[ 2055 {"inject_into": "header", "field_name": "Authorization"}, 2056 {"inject_into": "request_parameter", "field_name": "authKey"}, 2057 ], 2058 title="Inject API Key Into Outgoing HTTP Request", 2059 ) 2060 2061 2062class JsonSchemaPropertySelector(BaseModel): 2063 type: Literal["JsonSchemaPropertySelector"] 2064 transformations: Optional[ 2065 List[ 2066 Union[ 2067 AddFields, 2068 RemoveFields, 2069 KeysToLower, 2070 KeysToSnakeCase, 2071 FlattenFields, 2072 DpathFlattenFields, 2073 KeysReplace, 2074 CustomTransformation, 2075 ] 2076 ] 2077 ] = Field( 2078 None, 2079 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2080 title="Transformations", 2081 ) 2082 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2083 2084 2085class ListPartitionRouter(BaseModel): 2086 type: Literal["ListPartitionRouter"] 2087 cursor_field: str = Field( 2088 ..., 2089 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2090 examples=["section", "{{ config['section_key'] }}"], 2091 title="Current Partition Value Identifier", 2092 ) 2093 values: Union[str, List[str]] = Field( 2094 ..., 2095 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2096 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2097 title="Partition Values", 2098 ) 2099 request_option: Optional[RequestOption] = Field( 2100 None, 2101 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2102 title="Inject Partition Value Into Outgoing HTTP Request", 2103 ) 2104 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2105 2106 2107class RecordSelector(BaseModel): 2108 type: Literal["RecordSelector"] 2109 extractor: Union[DpathExtractor, CustomRecordExtractor] 2110 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2111 None, 2112 description="Responsible for filtering records to be emitted by the Source.", 2113 title="Record Filter", 2114 ) 2115 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2116 None, 2117 description="Responsible for normalization according to the schema.", 2118 title="Schema Normalization", 2119 ) 2120 transform_before_filtering: Optional[bool] = Field( 2121 None, 2122 description="If true, transformation will be applied before record filtering.", 2123 title="Transform Before Filtering", 2124 ) 2125 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2126 2127 2128class PaginationReset(BaseModel): 2129 type: Literal["PaginationReset"] 2130 action: Action1 2131 limits: Optional[PaginationResetLimits] = None 2132 2133 2134class GzipDecoder(BaseModel): 2135 type: Literal["GzipDecoder"] 2136 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2137 2138 2139class RequestBodyGraphQL(BaseModel): 2140 type: Literal["RequestBodyGraphQL"] 2141 value: RequestBodyGraphQlQuery 2142 2143 2144class DpathValidator(BaseModel): 2145 type: Literal["DpathValidator"] 2146 field_path: List[str] = Field( 2147 ..., 2148 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2149 examples=[ 2150 ["data"], 2151 ["data", "records"], 2152 ["data", "{{ parameters.name }}"], 2153 ["data", "*", "record"], 2154 ], 2155 title="Field Path", 2156 ) 2157 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2158 ..., 2159 description="The condition that the specified config value will be evaluated against", 2160 title="Validation Strategy", 2161 ) 2162 2163 2164class PredicateValidator(BaseModel): 2165 type: Literal["PredicateValidator"] 2166 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2167 ..., 2168 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2169 examples=[ 2170 "test-value", 2171 "{{ config['api_version'] }}", 2172 "{{ config['tenant_id'] }}", 2173 123, 2174 ], 2175 title="Value", 2176 ) 2177 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2178 ..., 2179 description="The validation strategy to apply to the value.", 2180 title="Validation Strategy", 2181 ) 2182 2183 2184class ConfigAddFields(BaseModel): 2185 type: Literal["ConfigAddFields"] 2186 fields: List[AddedFieldDefinition] = Field( 2187 ..., 2188 description="A list of transformations (path and corresponding value) that will be added to the config.", 2189 title="Fields", 2190 ) 2191 condition: Optional[str] = Field( 2192 "", 2193 description="Fields will be added if expression is evaluated to True.", 2194 examples=[ 2195 "{{ config['environemnt'] == 'sandbox' }}", 2196 "{{ property is integer }}", 2197 "{{ property|length > 5 }}", 2198 "{{ property == 'some_string_to_match' }}", 2199 ], 2200 ) 2201 2202 2203class CompositeErrorHandler(BaseModel): 2204 type: Literal["CompositeErrorHandler"] 2205 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2206 Field( 2207 ..., 2208 description="List of error handlers to iterate on to determine how to handle a failed response.", 2209 title="Error Handlers", 2210 ) 2211 ) 2212 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2213 2214 2215class HTTPAPIBudget(BaseModel): 2216 class Config: 2217 extra = Extra.allow 2218 2219 type: Literal["HTTPAPIBudget"] 2220 policies: List[ 2221 Union[ 2222 FixedWindowCallRatePolicy, 2223 MovingWindowCallRatePolicy, 2224 UnlimitedCallRatePolicy, 2225 ] 2226 ] = Field( 2227 ..., 2228 description="List of call rate policies that define how many calls are allowed.", 2229 title="Policies", 2230 ) 2231 ratelimit_reset_header: Optional[str] = Field( 2232 "ratelimit-reset", 2233 description="The HTTP response header name that indicates when the rate limit resets.", 2234 title="Rate Limit Reset Header", 2235 ) 2236 ratelimit_remaining_header: Optional[str] = Field( 2237 "ratelimit-remaining", 2238 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2239 title="Rate Limit Remaining Header", 2240 ) 2241 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2242 [429], 2243 description="List of HTTP status codes that indicate a rate limit has been hit.", 2244 title="Status Codes for Rate Limit Hit", 2245 ) 2246 2247 2248class ZipfileDecoder(BaseModel): 2249 class Config: 2250 extra = Extra.allow 2251 2252 type: Literal["ZipfileDecoder"] 2253 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2254 ..., 2255 description="Parser to parse the decompressed data from the zipfile(s).", 2256 title="Parser", 2257 ) 2258 2259 2260class ConfigMigration(BaseModel): 2261 type: Literal["ConfigMigration"] 2262 description: Optional[str] = Field( 2263 None, description="The description/purpose of the config migration." 2264 ) 2265 transformations: List[ 2266 Union[ 2267 ConfigRemapField, 2268 ConfigAddFields, 2269 ConfigRemoveFields, 2270 CustomConfigTransformation, 2271 ] 2272 ] = Field( 2273 ..., 2274 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2275 title="Transformations", 2276 ) 2277 2278 2279class ConfigNormalizationRules(BaseModel): 2280 class Config: 2281 extra = Extra.forbid 2282 2283 type: Literal["ConfigNormalizationRules"] 2284 config_migrations: Optional[List[ConfigMigration]] = Field( 2285 [], 2286 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2287 title="Config Migrations", 2288 ) 2289 transformations: Optional[ 2290 List[ 2291 Union[ 2292 ConfigRemapField, 2293 ConfigAddFields, 2294 ConfigRemoveFields, 2295 CustomConfigTransformation, 2296 ] 2297 ] 2298 ] = Field( 2299 [], 2300 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2301 title="Transformations", 2302 ) 2303 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2304 [], 2305 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2306 title="Validations", 2307 ) 2308 2309 2310class Spec(BaseModel): 2311 type: Literal["Spec"] 2312 connection_specification: Dict[str, Any] = Field( 2313 ..., 2314 description="A connection specification describing how a the connector can be configured.", 2315 title="Connection Specification", 2316 ) 2317 documentation_url: Optional[str] = Field( 2318 None, 2319 description="URL of the connector's documentation page.", 2320 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2321 title="Documentation URL", 2322 ) 2323 advanced_auth: Optional[AuthFlow] = Field( 2324 None, 2325 description="Advanced specification for configuring the authentication flow.", 2326 title="Advanced Auth", 2327 ) 2328 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2329 None, title="Config Normalization Rules" 2330 ) 2331 2332 2333class DeclarativeSource1(BaseModel): 2334 class Config: 2335 extra = Extra.forbid 2336 2337 type: Literal["DeclarativeSource"] 2338 check: Union[CheckStream, CheckDynamicStream] 2339 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2340 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2341 version: str = Field( 2342 ..., 2343 description="The version of the Airbyte CDK used to build and test the source.", 2344 ) 2345 schemas: Optional[Schemas] = None 2346 definitions: Optional[Dict[str, Any]] = None 2347 spec: Optional[Spec] = None 2348 concurrency_level: Optional[ConcurrencyLevel] = None 2349 api_budget: Optional[HTTPAPIBudget] = None 2350 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2351 None, 2352 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2353 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2354 title="Maximum Concurrent Asynchronous Jobs", 2355 ) 2356 metadata: Optional[Dict[str, Any]] = Field( 2357 None, 2358 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2359 ) 2360 description: Optional[str] = Field( 2361 None, 2362 description="A description of the connector. It will be presented on the Source documentation page.", 2363 ) 2364 2365 2366class DeclarativeSource2(BaseModel): 2367 class Config: 2368 extra = Extra.forbid 2369 2370 type: Literal["DeclarativeSource"] 2371 check: Union[CheckStream, CheckDynamicStream] 2372 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2373 None 2374 ) 2375 dynamic_streams: List[DynamicDeclarativeStream] 2376 version: str = Field( 2377 ..., 2378 description="The version of the Airbyte CDK used to build and test the source.", 2379 ) 2380 schemas: Optional[Schemas] = None 2381 definitions: Optional[Dict[str, Any]] = None 2382 spec: Optional[Spec] = None 2383 concurrency_level: Optional[ConcurrencyLevel] = None 2384 api_budget: Optional[HTTPAPIBudget] = None 2385 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2386 None, 2387 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2388 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2389 title="Maximum Concurrent Asynchronous Jobs", 2390 ) 2391 metadata: Optional[Dict[str, Any]] = Field( 2392 None, 2393 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2394 ) 2395 description: Optional[str] = Field( 2396 None, 2397 description="A description of the connector. It will be presented on the Source documentation page.", 2398 ) 2399 2400 2401class DeclarativeSource(BaseModel): 2402 class Config: 2403 extra = Extra.forbid 2404 2405 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2406 ..., 2407 description="An API source that extracts data according to its declarative components.", 2408 title="DeclarativeSource", 2409 ) 2410 2411 2412class SelectiveAuthenticator(BaseModel): 2413 class Config: 2414 extra = Extra.allow 2415 2416 type: Literal["SelectiveAuthenticator"] 2417 authenticator_selection_path: List[str] = Field( 2418 ..., 2419 description="Path of the field in config with selected authenticator name", 2420 examples=[["auth"], ["auth", "type"]], 2421 title="Authenticator Selection Path", 2422 ) 2423 authenticators: Dict[ 2424 str, 2425 Union[ 2426 ApiKeyAuthenticator, 2427 BasicHttpAuthenticator, 2428 BearerAuthenticator, 2429 OAuthAuthenticator, 2430 JwtAuthenticator, 2431 SessionTokenAuthenticator, 2432 LegacySessionTokenAuthenticator, 2433 CustomAuthenticator, 2434 NoAuth, 2435 ], 2436 ] = Field( 2437 ..., 2438 description="Authenticators to select from.", 2439 examples=[ 2440 { 2441 "authenticators": { 2442 "token": "#/definitions/ApiKeyAuthenticator", 2443 "oauth": "#/definitions/OAuthAuthenticator", 2444 "jwt": "#/definitions/JwtAuthenticator", 2445 } 2446 } 2447 ], 2448 title="Authenticators", 2449 ) 2450 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2451 2452 2453class ConditionalStreams(BaseModel): 2454 type: Literal["ConditionalStreams"] 2455 condition: str = Field( 2456 ..., 2457 description="Condition that will be evaluated to determine if a set of streams should be available.", 2458 examples=["{{ config['is_sandbox'] }}"], 2459 title="Condition", 2460 ) 2461 streams: List[DeclarativeStream] = Field( 2462 ..., 2463 description="Streams that will be used during an operation based on the condition.", 2464 title="Streams", 2465 ) 2466 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2467 2468 2469class FileUploader(BaseModel): 2470 type: Literal["FileUploader"] 2471 requester: Union[HttpRequester, CustomRequester] = Field( 2472 ..., 2473 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2474 ) 2475 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2476 ..., 2477 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2478 ) 2479 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2480 None, 2481 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2482 ) 2483 filename_extractor: Optional[str] = Field( 2484 None, 2485 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2486 examples=[ 2487 "{{ record.id }}/{{ record.file_name }}/", 2488 "{{ record.id }}_{{ record.file_name }}/", 2489 ], 2490 ) 2491 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2492 2493 2494class DeclarativeStream(BaseModel): 2495 class Config: 2496 extra = Extra.allow 2497 2498 type: Literal["DeclarativeStream"] 2499 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2500 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2501 ..., 2502 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2503 title="Retriever", 2504 ) 2505 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2506 None, 2507 description="Component used to fetch data incrementally based on a time field in the data.", 2508 title="Incremental Sync", 2509 ) 2510 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2511 schema_loader: Optional[ 2512 Union[ 2513 InlineSchemaLoader, 2514 DynamicSchemaLoader, 2515 JsonFileSchemaLoader, 2516 List[ 2517 Union[ 2518 InlineSchemaLoader, 2519 DynamicSchemaLoader, 2520 JsonFileSchemaLoader, 2521 CustomSchemaLoader, 2522 ] 2523 ], 2524 CustomSchemaLoader, 2525 ] 2526 ] = Field( 2527 None, 2528 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2529 title="Schema Loader", 2530 ) 2531 transformations: Optional[ 2532 List[ 2533 Union[ 2534 AddFields, 2535 RemoveFields, 2536 KeysToLower, 2537 KeysToSnakeCase, 2538 FlattenFields, 2539 DpathFlattenFields, 2540 KeysReplace, 2541 CustomTransformation, 2542 ] 2543 ] 2544 ] = Field( 2545 None, 2546 description="A list of transformations to be applied to each output record.", 2547 title="Transformations", 2548 ) 2549 state_migrations: Optional[ 2550 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2551 ] = Field( 2552 [], 2553 description="Array of state migrations to be applied on the input state", 2554 title="State Migrations", 2555 ) 2556 file_uploader: Optional[FileUploader] = Field( 2557 None, 2558 description="(experimental) Describes how to fetch a file", 2559 title="File Uploader", 2560 ) 2561 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2562 2563 2564class SessionTokenAuthenticator(BaseModel): 2565 type: Literal["SessionTokenAuthenticator"] 2566 login_requester: HttpRequester = Field( 2567 ..., 2568 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2569 examples=[ 2570 { 2571 "type": "HttpRequester", 2572 "url_base": "https://my_api.com", 2573 "path": "/login", 2574 "authenticator": { 2575 "type": "BasicHttpAuthenticator", 2576 "username": "{{ config.username }}", 2577 "password": "{{ config.password }}", 2578 }, 2579 } 2580 ], 2581 title="Login Requester", 2582 ) 2583 session_token_path: List[str] = Field( 2584 ..., 2585 description="The path in the response body returned from the login requester to the session token.", 2586 examples=[["access_token"], ["result", "token"]], 2587 title="Session Token Path", 2588 ) 2589 expiration_duration: Optional[str] = Field( 2590 None, 2591 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2592 examples=["PT1H", "P1D"], 2593 title="Expiration Duration", 2594 ) 2595 request_authentication: Union[ 2596 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2597 ] = Field( 2598 ..., 2599 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2600 title="Data Request Authentication", 2601 ) 2602 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2603 None, description="Component used to decode the response.", title="Decoder" 2604 ) 2605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2606 2607 2608class HttpRequester(BaseModelWithDeprecations): 2609 type: Literal["HttpRequester"] 2610 url_base: Optional[str] = Field( 2611 None, 2612 deprecated=True, 2613 deprecation_message="Use `url` field instead.", 2614 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2615 examples=[ 2616 "https://connect.squareup.com/v2", 2617 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2618 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2619 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2620 ], 2621 title="API Base URL", 2622 ) 2623 url: Optional[str] = Field( 2624 None, 2625 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2626 examples=[ 2627 "https://connect.squareup.com/v2", 2628 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2629 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2630 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2631 ], 2632 title="API Endpoint URL", 2633 ) 2634 path: Optional[str] = Field( 2635 None, 2636 deprecated=True, 2637 deprecation_message="Use `url` field instead.", 2638 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2639 examples=[ 2640 "/products", 2641 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2642 "/trades/{{ config['symbol_id'] }}/history", 2643 ], 2644 title="URL Path", 2645 ) 2646 http_method: Optional[HttpMethod] = Field( 2647 HttpMethod.GET, 2648 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2649 examples=["GET", "POST"], 2650 title="HTTP Method", 2651 ) 2652 authenticator: Optional[ 2653 Union[ 2654 ApiKeyAuthenticator, 2655 BasicHttpAuthenticator, 2656 BearerAuthenticator, 2657 OAuthAuthenticator, 2658 JwtAuthenticator, 2659 SessionTokenAuthenticator, 2660 SelectiveAuthenticator, 2661 CustomAuthenticator, 2662 NoAuth, 2663 LegacySessionTokenAuthenticator, 2664 ] 2665 ] = Field( 2666 None, 2667 description="Authentication method to use for requests sent to the API.", 2668 title="Authenticator", 2669 ) 2670 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2671 None, 2672 deprecated=True, 2673 deprecation_message="Use `query_properties` field instead.", 2674 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2675 title="Fetch Properties from Endpoint", 2676 ) 2677 query_properties: Optional[QueryProperties] = Field( 2678 None, 2679 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2680 title="Query Properties", 2681 ) 2682 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2683 None, 2684 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2685 examples=[ 2686 {"unit": "day"}, 2687 { 2688 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2689 }, 2690 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2691 {"sort_by[asc]": "updated_at"}, 2692 ], 2693 title="Query Parameters", 2694 ) 2695 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2696 None, 2697 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2698 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2699 title="Request Headers", 2700 ) 2701 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2702 None, 2703 deprecated=True, 2704 deprecation_message="Use `request_body` field instead.", 2705 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2706 examples=[ 2707 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2708 ], 2709 title="Request Body Payload (Non-JSON)", 2710 ) 2711 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2712 None, 2713 deprecated=True, 2714 deprecation_message="Use `request_body` field instead.", 2715 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2716 examples=[ 2717 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2718 {"key": "{{ config['value'] }}"}, 2719 {"sort": {"field": "updated_at", "order": "ascending"}}, 2720 ], 2721 title="Request Body JSON Payload", 2722 ) 2723 request_body: Optional[ 2724 Union[ 2725 RequestBodyPlainText, 2726 RequestBodyUrlEncodedForm, 2727 RequestBodyJsonObject, 2728 RequestBodyGraphQL, 2729 ] 2730 ] = Field( 2731 None, 2732 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2733 title="Request Body", 2734 ) 2735 error_handler: Optional[ 2736 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2737 ] = Field( 2738 None, 2739 description="Error handler component that defines how to handle errors.", 2740 title="Error Handler", 2741 ) 2742 use_cache: Optional[bool] = Field( 2743 False, 2744 description="Enables stream requests caching. This field is automatically set by the CDK.", 2745 title="Use Cache", 2746 ) 2747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2748 2749 2750class DynamicSchemaLoader(BaseModel): 2751 type: Literal["DynamicSchemaLoader"] 2752 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2753 ..., 2754 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2755 title="Retriever", 2756 ) 2757 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2758 None, 2759 description="Responsible for filtering fields to be added to json schema.", 2760 title="Schema Filter", 2761 ) 2762 schema_transformations: Optional[ 2763 List[ 2764 Union[ 2765 AddFields, 2766 RemoveFields, 2767 KeysToLower, 2768 KeysToSnakeCase, 2769 FlattenFields, 2770 DpathFlattenFields, 2771 KeysReplace, 2772 CustomTransformation, 2773 ] 2774 ] 2775 ] = Field( 2776 None, 2777 description="A list of transformations to be applied to the schema.", 2778 title="Schema Transformations", 2779 ) 2780 schema_type_identifier: SchemaTypeIdentifier 2781 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2782 2783 2784class ParentStreamConfig(BaseModel): 2785 type: Literal["ParentStreamConfig"] 2786 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2787 ..., description="Reference to the parent stream.", title="Parent Stream" 2788 ) 2789 parent_key: str = Field( 2790 ..., 2791 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2792 examples=["id", "{{ config['parent_record_id'] }}"], 2793 title="Parent Key", 2794 ) 2795 partition_field: str = Field( 2796 ..., 2797 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2798 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2799 title="Current Parent Key Value Identifier", 2800 ) 2801 request_option: Optional[RequestOption] = Field( 2802 None, 2803 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2804 title="Request Option", 2805 ) 2806 incremental_dependency: Optional[bool] = Field( 2807 False, 2808 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2809 title="Incremental Dependency", 2810 ) 2811 lazy_read_pointer: Optional[List[str]] = Field( 2812 [], 2813 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2814 title="Lazy Read Pointer", 2815 ) 2816 extra_fields: Optional[List[List[str]]] = Field( 2817 None, 2818 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2819 title="Extra Fields", 2820 ) 2821 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2822 2823 2824class PropertiesFromEndpoint(BaseModel): 2825 type: Literal["PropertiesFromEndpoint"] 2826 property_field_path: List[str] = Field( 2827 ..., 2828 description="Describes the path to the field that should be extracted", 2829 examples=[["name"]], 2830 ) 2831 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2832 ..., 2833 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2834 ) 2835 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2836 2837 2838class QueryProperties(BaseModel): 2839 type: Literal["QueryProperties"] 2840 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2841 ..., 2842 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2843 title="Property List", 2844 ) 2845 always_include_properties: Optional[List[str]] = Field( 2846 None, 2847 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2848 title="Always Include Properties", 2849 ) 2850 property_chunking: Optional[PropertyChunking] = Field( 2851 None, 2852 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2853 title="Property Chunking", 2854 ) 2855 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2856 None, 2857 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2858 title="Property Selector", 2859 ) 2860 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2861 2862 2863class StateDelegatingStream(BaseModel): 2864 type: Literal["StateDelegatingStream"] 2865 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2866 full_refresh_stream: DeclarativeStream = Field( 2867 ..., 2868 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2869 title="Full Refresh Stream", 2870 ) 2871 incremental_stream: DeclarativeStream = Field( 2872 ..., 2873 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2874 title="Incremental Stream", 2875 ) 2876 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2877 2878 2879class SimpleRetriever(BaseModel): 2880 type: Literal["SimpleRetriever"] 2881 requester: Union[HttpRequester, CustomRequester] = Field( 2882 ..., 2883 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2884 ) 2885 decoder: Optional[ 2886 Union[ 2887 JsonDecoder, 2888 XmlDecoder, 2889 CsvDecoder, 2890 JsonlDecoder, 2891 GzipDecoder, 2892 IterableDecoder, 2893 ZipfileDecoder, 2894 CustomDecoder, 2895 ] 2896 ] = Field( 2897 None, 2898 description="Component decoding the response so records can be extracted.", 2899 title="HTTP Response Format", 2900 ) 2901 record_selector: RecordSelector = Field( 2902 ..., 2903 description="Component that describes how to extract records from a HTTP response.", 2904 ) 2905 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2906 None, 2907 description="Paginator component that describes how to navigate through the API's pages.", 2908 ) 2909 pagination_reset: Optional[PaginationReset] = Field( 2910 None, 2911 description="Describes what triggers pagination reset and how to handle it.", 2912 ) 2913 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2914 False, 2915 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2916 ) 2917 partition_router: Optional[ 2918 Union[ 2919 SubstreamPartitionRouter, 2920 ListPartitionRouter, 2921 GroupingPartitionRouter, 2922 CustomPartitionRouter, 2923 List[ 2924 Union[ 2925 SubstreamPartitionRouter, 2926 ListPartitionRouter, 2927 GroupingPartitionRouter, 2928 CustomPartitionRouter, 2929 ] 2930 ], 2931 ] 2932 ] = Field( 2933 None, 2934 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2935 title="Partition Router", 2936 ) 2937 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2938 2939 2940class AsyncRetriever(BaseModel): 2941 type: Literal["AsyncRetriever"] 2942 record_selector: RecordSelector = Field( 2943 ..., 2944 description="Component that describes how to extract records from a HTTP response.", 2945 ) 2946 status_mapping: AsyncJobStatusMap = Field( 2947 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2948 ) 2949 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2950 ..., description="Responsible for fetching the actual status of the async job." 2951 ) 2952 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2953 None, 2954 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2955 ) 2956 download_extractor: Optional[ 2957 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2958 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2959 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2960 ..., 2961 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2962 ) 2963 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2964 ..., 2965 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2966 ) 2967 polling_job_timeout: Optional[Union[int, str]] = Field( 2968 None, 2969 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2970 ) 2971 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2972 None, 2973 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2974 ) 2975 download_requester: Union[HttpRequester, CustomRequester] = Field( 2976 ..., 2977 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2978 ) 2979 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2980 None, 2981 description="Paginator component that describes how to navigate through the API's pages during download.", 2982 ) 2983 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2984 None, 2985 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2986 ) 2987 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2988 None, 2989 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2990 ) 2991 partition_router: Optional[ 2992 Union[ 2993 ListPartitionRouter, 2994 SubstreamPartitionRouter, 2995 GroupingPartitionRouter, 2996 CustomPartitionRouter, 2997 List[ 2998 Union[ 2999 ListPartitionRouter, 3000 SubstreamPartitionRouter, 3001 GroupingPartitionRouter, 3002 CustomPartitionRouter, 3003 ] 3004 ], 3005 ] 3006 ] = Field( 3007 [], 3008 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3009 title="Partition Router", 3010 ) 3011 decoder: Optional[ 3012 Union[ 3013 CsvDecoder, 3014 GzipDecoder, 3015 JsonDecoder, 3016 JsonlDecoder, 3017 IterableDecoder, 3018 XmlDecoder, 3019 ZipfileDecoder, 3020 CustomDecoder, 3021 ] 3022 ] = Field( 3023 None, 3024 description="Component decoding the response so records can be extracted.", 3025 title="HTTP Response Format", 3026 ) 3027 download_decoder: Optional[ 3028 Union[ 3029 CsvDecoder, 3030 GzipDecoder, 3031 JsonDecoder, 3032 JsonlDecoder, 3033 IterableDecoder, 3034 XmlDecoder, 3035 ZipfileDecoder, 3036 CustomDecoder, 3037 ] 3038 ] = Field( 3039 None, 3040 description="Component decoding the download response so records can be extracted.", 3041 title="Download HTTP Response Format", 3042 ) 3043 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3044 3045 3046class SubstreamPartitionRouter(BaseModel): 3047 type: Literal["SubstreamPartitionRouter"] 3048 parent_stream_configs: List[ParentStreamConfig] = Field( 3049 ..., 3050 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3051 title="Parent Stream Configs", 3052 ) 3053 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3054 3055 3056class GroupingPartitionRouter(BaseModel): 3057 type: Literal["GroupingPartitionRouter"] 3058 group_size: int = Field( 3059 ..., 3060 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3061 examples=[10, 50], 3062 title="Group Size", 3063 ) 3064 underlying_partition_router: Union[ 3065 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3066 ] = Field( 3067 ..., 3068 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3069 title="Underlying Partition Router", 3070 ) 3071 deduplicate: Optional[bool] = Field( 3072 True, 3073 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3074 title="Deduplicate Partitions", 3075 ) 3076 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3077 3078 3079class HttpComponentsResolver(BaseModel): 3080 type: Literal["HttpComponentsResolver"] 3081 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3082 ..., 3083 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3084 title="Retriever", 3085 ) 3086 components_mapping: List[ComponentMappingDefinition] 3087 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3088 3089 3090class DynamicDeclarativeStream(BaseModel): 3091 type: Literal["DynamicDeclarativeStream"] 3092 name: Optional[str] = Field( 3093 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3094 ) 3095 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3096 ..., description="Reference to the stream template.", title="Stream Template" 3097 ) 3098 components_resolver: Union[ 3099 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3100 ] = Field( 3101 ..., 3102 description="Component resolve and populates stream templates with components values.", 3103 title="Components Resolver", 3104 ) 3105 use_parent_parameters: Optional[bool] = Field( 3106 True, 3107 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3108 title="Use Parent Parameters", 3109 ) 3110 3111 3112ComplexFieldType.update_forward_refs() 3113GzipDecoder.update_forward_refs() 3114CompositeErrorHandler.update_forward_refs() 3115DeclarativeSource1.update_forward_refs() 3116DeclarativeSource2.update_forward_refs() 3117SelectiveAuthenticator.update_forward_refs() 3118ConditionalStreams.update_forward_refs() 3119FileUploader.update_forward_refs() 3120DeclarativeStream.update_forward_refs() 3121SessionTokenAuthenticator.update_forward_refs() 3122HttpRequester.update_forward_refs() 3123DynamicSchemaLoader.update_forward_refs() 3124ParentStreamConfig.update_forward_refs() 3125PropertiesFromEndpoint.update_forward_refs() 3126SimpleRetriever.update_forward_refs() 3127AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
24class BasicHttpAuthenticator(BaseModel): 25 type: Literal["BasicHttpAuthenticator"] 26 username: str = Field( 27 ..., 28 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 29 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 30 title="Username", 31 ) 32 password: Optional[str] = Field( 33 "", 34 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 35 examples=["{{ config['password'] }}", ""], 36 title="Password", 37 ) 38 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
41class BearerAuthenticator(BaseModel): 42 type: Literal["BearerAuthenticator"] 43 api_token: str = Field( 44 ..., 45 description="Token to inject as request header for authenticating with the API.", 46 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 47 title="Bearer Token", 48 ) 49 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
52class DynamicStreamCheckConfig(BaseModel): 53 type: Literal["DynamicStreamCheckConfig"] 54 dynamic_stream_name: str = Field( 55 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 56 ) 57 stream_count: Optional[int] = Field( 58 0, 59 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 60 title="Stream Count", 61 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
64class CheckDynamicStream(BaseModel): 65 type: Literal["CheckDynamicStream"] 66 stream_count: int = Field( 67 ..., 68 description="Numbers of the streams to try reading from when running a check operation.", 69 title="Stream Count", 70 ) 71 use_check_availability: Optional[bool] = Field( 72 True, 73 description="Enables stream check availability. This field is automatically set by the CDK.", 74 title="Use Check Availability", 75 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
78class ConcurrencyLevel(BaseModel): 79 type: Optional[Literal["ConcurrencyLevel"]] = None 80 default_concurrency: Union[int, str] = Field( 81 ..., 82 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 83 examples=[10, "{{ config['num_workers'] or 10 }}"], 84 title="Default Concurrency", 85 ) 86 max_concurrency: Optional[int] = Field( 87 None, 88 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 89 examples=[20, 100], 90 title="Max Concurrency", 91 ) 92 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
95class ConstantBackoffStrategy(BaseModel): 96 type: Literal["ConstantBackoffStrategy"] 97 backoff_time_in_seconds: Union[float, str] = Field( 98 ..., 99 description="Backoff time in seconds.", 100 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 101 title="Backoff Time", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
106class CursorPagination(BaseModel): 107 type: Literal["CursorPagination"] 108 cursor_value: str = Field( 109 ..., 110 description="Value of the cursor defining the next page to fetch.", 111 examples=[ 112 "{{ headers.link.next.cursor }}", 113 "{{ last_record['key'] }}", 114 "{{ response['nextPage'] }}", 115 ], 116 title="Cursor Value", 117 ) 118 page_size: Optional[int] = Field( 119 None, 120 description="The number of records to include in each pages.", 121 examples=[100], 122 title="Page Size", 123 ) 124 stop_condition: Optional[str] = Field( 125 None, 126 description="Template string evaluating when to stop paginating.", 127 examples=[ 128 "{{ response.data.has_more is false }}", 129 "{{ 'next' not in headers['link'] }}", 130 ], 131 title="Stop Condition", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
136class CustomAuthenticator(BaseModel): 137 class Config: 138 extra = Extra.allow 139 140 type: Literal["CustomAuthenticator"] 141 class_name: str = Field( 142 ..., 143 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 144 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 145 title="Class Name", 146 ) 147 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
150class CustomBackoffStrategy(BaseModel): 151 class Config: 152 extra = Extra.allow 153 154 type: Literal["CustomBackoffStrategy"] 155 class_name: str = Field( 156 ..., 157 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 158 examples=["source_railz.components.MyCustomBackoffStrategy"], 159 title="Class Name", 160 ) 161 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
164class CustomErrorHandler(BaseModel): 165 class Config: 166 extra = Extra.allow 167 168 type: Literal["CustomErrorHandler"] 169 class_name: str = Field( 170 ..., 171 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 172 examples=["source_railz.components.MyCustomErrorHandler"], 173 title="Class Name", 174 ) 175 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
178class CustomPaginationStrategy(BaseModel): 179 class Config: 180 extra = Extra.allow 181 182 type: Literal["CustomPaginationStrategy"] 183 class_name: str = Field( 184 ..., 185 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 186 examples=["source_railz.components.MyCustomPaginationStrategy"], 187 title="Class Name", 188 ) 189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
192class CustomRecordExtractor(BaseModel): 193 class Config: 194 extra = Extra.allow 195 196 type: Literal["CustomRecordExtractor"] 197 class_name: str = Field( 198 ..., 199 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 200 examples=["source_railz.components.MyCustomRecordExtractor"], 201 title="Class Name", 202 ) 203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
206class CustomRecordFilter(BaseModel): 207 class Config: 208 extra = Extra.allow 209 210 type: Literal["CustomRecordFilter"] 211 class_name: str = Field( 212 ..., 213 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 214 examples=["source_railz.components.MyCustomCustomRecordFilter"], 215 title="Class Name", 216 ) 217 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
220class CustomRequester(BaseModel): 221 class Config: 222 extra = Extra.allow 223 224 type: Literal["CustomRequester"] 225 class_name: str = Field( 226 ..., 227 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 228 examples=["source_railz.components.MyCustomRecordExtractor"], 229 title="Class Name", 230 ) 231 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
234class CustomRetriever(BaseModel): 235 class Config: 236 extra = Extra.allow 237 238 type: Literal["CustomRetriever"] 239 class_name: str = Field( 240 ..., 241 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 242 examples=["source_railz.components.MyCustomRetriever"], 243 title="Class Name", 244 ) 245 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
248class CustomPartitionRouter(BaseModel): 249 class Config: 250 extra = Extra.allow 251 252 type: Literal["CustomPartitionRouter"] 253 class_name: str = Field( 254 ..., 255 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 256 examples=["source_railz.components.MyCustomPartitionRouter"], 257 title="Class Name", 258 ) 259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
262class CustomSchemaLoader(BaseModel): 263 class Config: 264 extra = Extra.allow 265 266 type: Literal["CustomSchemaLoader"] 267 class_name: str = Field( 268 ..., 269 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 270 examples=["source_railz.components.MyCustomSchemaLoader"], 271 title="Class Name", 272 ) 273 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
276class CustomSchemaNormalization(BaseModel): 277 class Config: 278 extra = Extra.allow 279 280 type: Literal["CustomSchemaNormalization"] 281 class_name: str = Field( 282 ..., 283 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 284 examples=[ 285 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 286 ], 287 title="Class Name", 288 ) 289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
292class CustomStateMigration(BaseModel): 293 class Config: 294 extra = Extra.allow 295 296 type: Literal["CustomStateMigration"] 297 class_name: str = Field( 298 ..., 299 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 300 examples=["source_railz.components.MyCustomStateMigration"], 301 title="Class Name", 302 ) 303 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
306class CustomTransformation(BaseModel): 307 class Config: 308 extra = Extra.allow 309 310 type: Literal["CustomTransformation"] 311 class_name: str = Field( 312 ..., 313 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 314 examples=["source_railz.components.MyCustomTransformation"], 315 title="Class Name", 316 ) 317 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
337class Algorithm(Enum): 338 HS256 = "HS256" 339 HS384 = "HS384" 340 HS512 = "HS512" 341 ES256 = "ES256" 342 ES256K = "ES256K" 343 ES384 = "ES384" 344 ES512 = "ES512" 345 RS256 = "RS256" 346 RS384 = "RS384" 347 RS512 = "RS512" 348 PS256 = "PS256" 349 PS384 = "PS384" 350 PS512 = "PS512" 351 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
354class JwtHeaders(BaseModel): 355 class Config: 356 extra = Extra.forbid 357 358 kid: Optional[str] = Field( 359 None, 360 description="Private key ID for user account.", 361 examples=["{{ config['kid'] }}"], 362 title="Key Identifier", 363 ) 364 typ: Optional[str] = Field( 365 "JWT", 366 description="The media type of the complete JWT.", 367 examples=["JWT"], 368 title="Type", 369 ) 370 cty: Optional[str] = Field( 371 None, 372 description="Content type of JWT header.", 373 examples=["JWT"], 374 title="Content Type", 375 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
378class JwtPayload(BaseModel): 379 class Config: 380 extra = Extra.forbid 381 382 iss: Optional[str] = Field( 383 None, 384 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 385 examples=["{{ config['iss'] }}"], 386 title="Issuer", 387 ) 388 sub: Optional[str] = Field( 389 None, 390 description="The subject of the JWT. Commonly defined by the API.", 391 title="Subject", 392 ) 393 aud: Optional[str] = Field( 394 None, 395 description="The recipient that the JWT is intended for. Commonly defined by the API.", 396 examples=["appstoreconnect-v1"], 397 title="Audience", 398 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
401class RefreshTokenUpdater(BaseModel): 402 refresh_token_name: Optional[str] = Field( 403 "refresh_token", 404 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 405 examples=["refresh_token"], 406 title="Refresh Token Property Name", 407 ) 408 access_token_config_path: Optional[List[str]] = Field( 409 ["credentials", "access_token"], 410 description="Config path to the access token. Make sure the field actually exists in the config.", 411 examples=[["credentials", "access_token"], ["access_token"]], 412 title="Config Path To Access Token", 413 ) 414 refresh_token_config_path: Optional[List[str]] = Field( 415 ["credentials", "refresh_token"], 416 description="Config path to the access token. Make sure the field actually exists in the config.", 417 examples=[["credentials", "refresh_token"], ["refresh_token"]], 418 title="Config Path To Refresh Token", 419 ) 420 token_expiry_date_config_path: Optional[List[str]] = Field( 421 ["credentials", "token_expiry_date"], 422 description="Config path to the expiry date. Make sure actually exists in the config.", 423 examples=[["credentials", "token_expiry_date"]], 424 title="Config Path To Expiry Date", 425 ) 426 refresh_token_error_status_codes: Optional[List[int]] = Field( 427 [], 428 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 429 examples=[[400, 500]], 430 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 431 ) 432 refresh_token_error_key: Optional[str] = Field( 433 "", 434 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 435 examples=["error"], 436 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 437 ) 438 refresh_token_error_values: Optional[List[str]] = Field( 439 [], 440 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 441 examples=[["invalid_grant", "invalid_permissions"]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 443 )
class
Rate(pydantic.v1.main.BaseModel):
446class Rate(BaseModel): 447 class Config: 448 extra = Extra.allow 449 450 limit: Union[int, str] = Field( 451 ..., 452 description="The maximum number of calls allowed within the interval.", 453 title="Limit", 454 ) 455 interval: str = Field( 456 ..., 457 description="The time interval for the rate limit.", 458 examples=["PT1H", "P1D"], 459 title="Interval", 460 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
463class HttpRequestRegexMatcher(BaseModel): 464 class Config: 465 extra = Extra.allow 466 467 method: Optional[str] = Field( 468 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 469 ) 470 url_base: Optional[str] = Field( 471 None, 472 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 473 title="URL Base", 474 ) 475 url_path_pattern: Optional[str] = Field( 476 None, 477 description="A regular expression pattern to match the URL path.", 478 title="URL Path Pattern", 479 ) 480 params: Optional[Dict[str, Any]] = Field( 481 None, description="The query parameters to match.", title="Parameters" 482 ) 483 headers: Optional[Dict[str, Any]] = Field( 484 None, description="The headers to match.", title="Headers" 485 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
488class DpathExtractor(BaseModel): 489 type: Literal["DpathExtractor"] 490 field_path: List[str] = Field( 491 ..., 492 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 493 examples=[ 494 ["data"], 495 ["data", "records"], 496 ["data", "{{ parameters.name }}"], 497 ["data", "*", "record"], 498 ], 499 title="Field Path", 500 ) 501 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
509class ExponentialBackoffStrategy(BaseModel): 510 type: Literal["ExponentialBackoffStrategy"] 511 factor: Optional[Union[float, str]] = Field( 512 5, 513 description="Multiplicative constant applied on each retry.", 514 examples=[5, 5.5, "10"], 515 title="Factor", 516 ) 517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
520class GroupByKeyMergeStrategy(BaseModel): 521 type: Literal["GroupByKeyMergeStrategy"] 522 key: Union[str, List[str]] = Field( 523 ..., 524 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 525 examples=["id", ["parent_id", "end_date"]], 526 title="Key", 527 ) 528 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
540class Action(Enum): 541 SUCCESS = "SUCCESS" 542 FAIL = "FAIL" 543 RETRY = "RETRY" 544 IGNORE = "IGNORE" 545 RESET_PAGINATION = "RESET_PAGINATION" 546 RATE_LIMITED = "RATE_LIMITED"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
549class FailureType(Enum): 550 system_error = "system_error" 551 config_error = "config_error" 552 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
555class HttpResponseFilter(BaseModel): 556 type: Literal["HttpResponseFilter"] 557 action: Optional[Action] = Field( 558 None, 559 description="Action to execute if a response matches the filter.", 560 examples=[ 561 "SUCCESS", 562 "FAIL", 563 "RETRY", 564 "IGNORE", 565 "RESET_PAGINATION", 566 "RATE_LIMITED", 567 ], 568 title="Action", 569 ) 570 failure_type: Optional[FailureType] = Field( 571 None, 572 description="Failure type of traced exception if a response matches the filter.", 573 examples=["system_error", "config_error", "transient_error"], 574 title="Failure Type", 575 ) 576 error_message: Optional[str] = Field( 577 None, 578 description="Error Message to display if the response matches the filter.", 579 title="Error Message", 580 ) 581 error_message_contains: Optional[str] = Field( 582 None, 583 description="Match the response if its error message contains the substring.", 584 example=["This API operation is not enabled for this site"], 585 title="Error Message Substring", 586 ) 587 http_codes: Optional[List[int]] = Field( 588 None, 589 description="Match the response if its HTTP code is included in this list.", 590 examples=[[420, 429], [500]], 591 title="HTTP Codes", 592 unique_items=True, 593 ) 594 predicate: Optional[str] = Field( 595 None, 596 description="Match the response if the predicate evaluates to true.", 597 examples=[ 598 "{{ 'Too much requests' in response }}", 599 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 600 ], 601 title="Predicate", 602 ) 603 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
606class ComplexFieldType(BaseModel): 607 field_type: str 608 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
611class TypesMap(BaseModel): 612 target_type: Union[str, List[str], ComplexFieldType] 613 current_type: Union[str, List[str]] 614 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
617class SchemaTypeIdentifier(BaseModel): 618 type: Optional[Literal["SchemaTypeIdentifier"]] = None 619 schema_pointer: Optional[List[str]] = Field( 620 [], 621 description="List of nested fields defining the schema field path to extract. Defaults to [].", 622 title="Schema Path", 623 ) 624 key_pointer: List[str] = Field( 625 ..., 626 description="List of potentially nested fields describing the full path of the field key to extract.", 627 title="Key Path", 628 ) 629 type_pointer: Optional[List[str]] = Field( 630 None, 631 description="List of potentially nested fields describing the full path of the field type to extract.", 632 title="Type Path", 633 ) 634 types_mapping: Optional[List[TypesMap]] = None 635 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
638class InlineSchemaLoader(BaseModel): 639 type: Literal["InlineSchemaLoader"] 640 schema_: Optional[Dict[str, Any]] = Field( 641 None, 642 alias="schema", 643 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 644 title="Schema", 645 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
648class JsonFileSchemaLoader(BaseModel): 649 type: Literal["JsonFileSchemaLoader"] 650 file_path: Optional[str] = Field( 651 None, 652 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 653 example=["./schemas/users.json"], 654 title="File Path", 655 ) 656 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
677class FlattenFields(BaseModel): 678 type: Literal["FlattenFields"] 679 flatten_lists: Optional[bool] = Field( 680 True, 681 description="Whether to flatten lists or leave it as is. Default is True.", 682 title="Flatten Lists", 683 ) 684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
687class KeyTransformation(BaseModel): 688 type: Literal["KeyTransformation"] 689 prefix: Optional[str] = Field( 690 None, 691 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 692 examples=["flattened_"], 693 title="Key Prefix", 694 ) 695 suffix: Optional[str] = Field( 696 None, 697 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 698 examples=["_flattened"], 699 title="Key Suffix", 700 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
703class DpathFlattenFields(BaseModel): 704 type: Literal["DpathFlattenFields"] 705 field_path: List[str] = Field( 706 ..., 707 description="A path to field that needs to be flattened.", 708 examples=[["data"], ["data", "*", "field"]], 709 title="Field Path", 710 ) 711 delete_origin_value: Optional[bool] = Field( 712 None, 713 description="Whether to delete the origin value or keep it. Default is False.", 714 title="Delete Origin Value", 715 ) 716 replace_record: Optional[bool] = Field( 717 None, 718 description="Whether to replace the origin record or not. Default is False.", 719 title="Replace Origin Record", 720 ) 721 key_transformation: Optional[KeyTransformation] = Field( 722 None, 723 description="Transformation for object keys. If not provided, original key will be used.", 724 title="Key transformation", 725 ) 726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
729class KeysReplace(BaseModel): 730 type: Literal["KeysReplace"] 731 old: str = Field( 732 ..., 733 description="Old value to replace.", 734 examples=[ 735 " ", 736 "{{ record.id }}", 737 "{{ config['id'] }}", 738 "{{ stream_slice['id'] }}", 739 ], 740 title="Old value", 741 ) 742 new: str = Field( 743 ..., 744 description="New value to set.", 745 examples=[ 746 "_", 747 "{{ record.id }}", 748 "{{ config['id'] }}", 749 "{{ stream_slice['id'] }}", 750 ], 751 title="New value", 752 ) 753 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
764class CustomDecoder(BaseModel): 765 class Config: 766 extra = Extra.allow 767 768 type: Literal["CustomDecoder"] 769 class_name: str = Field( 770 ..., 771 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 772 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 773 title="Class Name", 774 ) 775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
778class MinMaxDatetime(BaseModel): 779 type: Literal["MinMaxDatetime"] 780 datetime: str = Field( 781 ..., 782 description="Datetime value.", 783 examples=[ 784 "2021-01-01", 785 "2021-01-01T00:00:00Z", 786 "{{ config['start_time'] }}", 787 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 788 ], 789 title="Datetime", 790 ) 791 datetime_format: Optional[str] = Field( 792 "", 793 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 794 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 795 title="Datetime Format", 796 ) 797 max_datetime: Optional[str] = Field( 798 None, 799 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 800 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 801 title="Max Datetime", 802 ) 803 min_datetime: Optional[str] = Field( 804 None, 805 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 806 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 807 title="Min Datetime", 808 ) 809 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
829class OauthConnectorInputSpecification(BaseModel): 830 class Config: 831 extra = Extra.allow 832 833 consent_url: str = Field( 834 ..., 835 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 836 examples=[ 837 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 838 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 839 ], 840 title="Consent URL", 841 ) 842 scope: Optional[str] = Field( 843 None, 844 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 845 examples=["user:read user:read_orders workspaces:read"], 846 title="Scopes", 847 ) 848 access_token_url: str = Field( 849 ..., 850 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 851 examples=[ 852 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 853 ], 854 title="Access Token URL", 855 ) 856 access_token_headers: Optional[Dict[str, Any]] = Field( 857 None, 858 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 859 examples=[ 860 { 861 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 862 } 863 ], 864 title="Access Token Headers", 865 ) 866 access_token_params: Optional[Dict[str, Any]] = Field( 867 None, 868 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 869 examples=[ 870 { 871 "{{ auth_code_key }}": "{{ auth_code_value }}", 872 "{{ client_id_key }}": "{{ client_id_value }}", 873 "{{ client_secret_key }}": "{{ client_secret_value }}", 874 } 875 ], 876 title="Access Token Query Params (Json Encoded)", 877 ) 878 extract_output: Optional[List[str]] = Field( 879 None, 880 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 881 examples=[["access_token", "refresh_token", "other_field"]], 882 title="Extract Output", 883 ) 884 state: Optional[State] = Field( 885 None, 886 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 887 examples=[{"min": 7, "max": 128}], 888 title="Configurable State Query Param", 889 ) 890 client_id_key: Optional[str] = Field( 891 None, 892 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 893 examples=["my_custom_client_id_key_name"], 894 title="Client ID Key Override", 895 ) 896 client_secret_key: Optional[str] = Field( 897 None, 898 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 899 examples=["my_custom_client_secret_key_name"], 900 title="Client Secret Key Override", 901 ) 902 scope_key: Optional[str] = Field( 903 None, 904 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 905 examples=["my_custom_scope_key_key_name"], 906 title="Scopes Key Override", 907 ) 908 state_key: Optional[str] = Field( 909 None, 910 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 911 examples=["my_custom_state_key_key_name"], 912 title="State Key Override", 913 ) 914 auth_code_key: Optional[str] = Field( 915 None, 916 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 917 examples=["my_custom_auth_code_key_name"], 918 title="Auth Code Key Override", 919 ) 920 redirect_uri_key: Optional[str] = Field( 921 None, 922 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 923 examples=["my_custom_redirect_uri_key_name"], 924 title="Redirect URI Key Override", 925 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
928class OAuthConfigSpecification(BaseModel): 929 class Config: 930 extra = Extra.allow 931 932 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 933 None, 934 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 935 examples=[ 936 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 937 { 938 "app_id": { 939 "type": "string", 940 "path_in_connector_config": ["info", "app_id"], 941 } 942 }, 943 ], 944 title="OAuth user input", 945 ) 946 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 947 None, 948 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 949 title="DeclarativeOAuth Connector Specification", 950 ) 951 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 952 None, 953 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 954 examples=[ 955 { 956 "refresh_token": { 957 "type": "string,", 958 "path_in_connector_config": ["credentials", "refresh_token"], 959 } 960 } 961 ], 962 title="OAuth output specification", 963 ) 964 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 965 None, 966 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 967 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 968 title="OAuth input specification", 969 ) 970 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 971 None, 972 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 973 examples=[ 974 { 975 "client_id": { 976 "type": "string,", 977 "path_in_connector_config": ["credentials", "client_id"], 978 }, 979 "client_secret": { 980 "type": "string,", 981 "path_in_connector_config": ["credentials", "client_secret"], 982 }, 983 } 984 ], 985 title="OAuth server output specification", 986 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
989class OffsetIncrement(BaseModel): 990 type: Literal["OffsetIncrement"] 991 page_size: Optional[Union[int, str]] = Field( 992 None, 993 description="The number of records to include in each pages.", 994 examples=[100, "{{ config['page_size'] }}"], 995 title="Limit", 996 ) 997 inject_on_first_request: Optional[bool] = Field( 998 False, 999 description="Using the `offset` with value `0` during the first request", 1000 title="Inject Offset on First Request", 1001 ) 1002 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1005class PageIncrement(BaseModel): 1006 type: Literal["PageIncrement"] 1007 page_size: Optional[Union[int, str]] = Field( 1008 None, 1009 description="The number of records to include in each pages.", 1010 examples=[100, "100", "{{ config['page_size'] }}"], 1011 title="Page Size", 1012 ) 1013 start_from_page: Optional[int] = Field( 1014 0, 1015 description="Index of the first page to request.", 1016 examples=[0, 1], 1017 title="Start From Page", 1018 ) 1019 inject_on_first_request: Optional[bool] = Field( 1020 False, 1021 description="Using the `page number` with value defined by `start_from_page` during the first request", 1022 title="Inject Page Number on First Request", 1023 ) 1024 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1027class PrimaryKey(BaseModel): 1028 __root__: Union[str, List[str], List[List[str]]] = Field( 1029 ..., 1030 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1031 examples=["id", ["code", "type"]], 1032 title="Primary Key", 1033 )
class
PropertyLimitType(enum.Enum):
1036class PropertyLimitType(Enum): 1037 characters = "characters" 1038 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1041class PropertyChunking(BaseModel): 1042 type: Literal["PropertyChunking"] 1043 property_limit_type: PropertyLimitType = Field( 1044 ..., 1045 description="The type used to determine the maximum number of properties per chunk", 1046 title="Property Limit Type", 1047 ) 1048 property_limit: Optional[int] = Field( 1049 None, 1050 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1051 title="Property Limit", 1052 ) 1053 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1054 None, 1055 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1056 title="Record Merge Strategy", 1057 ) 1058 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1061class RecordFilter(BaseModel): 1062 type: Literal["RecordFilter"] 1063 condition: Optional[str] = Field( 1064 "", 1065 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1066 examples=[ 1067 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1068 "{{ record.status in ['active', 'expired'] }}", 1069 ], 1070 title="Condition", 1071 ) 1072 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1080class RemoveFields(BaseModel): 1081 type: Literal["RemoveFields"] 1082 condition: Optional[str] = Field( 1083 "", 1084 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1085 examples=[ 1086 "{{ property|string == '' }}", 1087 "{{ property is integer }}", 1088 "{{ property|length > 5 }}", 1089 "{{ property == 'some_string_to_match' }}", 1090 ], 1091 ) 1092 field_pointers: List[List[str]] = Field( 1093 ..., 1094 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1095 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1096 title="Field Paths", 1097 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1104class InjectInto(Enum): 1105 request_parameter = "request_parameter" 1106 header = "header" 1107 body_data = "body_data" 1108 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1111class RequestOption(BaseModel): 1112 type: Literal["RequestOption"] 1113 inject_into: InjectInto = Field( 1114 ..., 1115 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1116 examples=["request_parameter", "header", "body_data", "body_json"], 1117 title="Inject Into", 1118 ) 1119 field_name: Optional[str] = Field( 1120 None, 1121 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1122 examples=["segment_id"], 1123 title="Field Name", 1124 ) 1125 field_path: Optional[List[str]] = Field( 1126 None, 1127 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1128 examples=[["data", "viewer", "id"]], 1129 title="Field Path", 1130 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1140class LegacySessionTokenAuthenticator(BaseModel): 1141 type: Literal["LegacySessionTokenAuthenticator"] 1142 header: str = Field( 1143 ..., 1144 description="The name of the session token header that will be injected in the request", 1145 examples=["X-Session"], 1146 title="Session Request Header", 1147 ) 1148 login_url: str = Field( 1149 ..., 1150 description="Path of the login URL (do not include the base URL)", 1151 examples=["session"], 1152 title="Login Path", 1153 ) 1154 session_token: Optional[str] = Field( 1155 None, 1156 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1157 example=["{{ config['session_token'] }}"], 1158 title="Session Token", 1159 ) 1160 session_token_response_key: str = Field( 1161 ..., 1162 description="Name of the key of the session token to be extracted from the response", 1163 examples=["id"], 1164 title="Response Token Response Key", 1165 ) 1166 username: Optional[str] = Field( 1167 None, 1168 description="Username used to authenticate and obtain a session token", 1169 examples=[" {{ config['username'] }}"], 1170 title="Username", 1171 ) 1172 password: Optional[str] = Field( 1173 "", 1174 description="Password used to authenticate and obtain a session token", 1175 examples=["{{ config['password'] }}", ""], 1176 title="Password", 1177 ) 1178 validate_session_url: str = Field( 1179 ..., 1180 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1181 examples=["user/current"], 1182 title="Validate Session Path", 1183 ) 1184 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1212class ValueType(Enum): 1213 string = "string" 1214 number = "number" 1215 integer = "integer" 1216 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1219class WaitTimeFromHeader(BaseModel): 1220 type: Literal["WaitTimeFromHeader"] 1221 header: str = Field( 1222 ..., 1223 description="The name of the response header defining how long to wait before retrying.", 1224 examples=["Retry-After"], 1225 title="Response Header Name", 1226 ) 1227 regex: Optional[str] = Field( 1228 None, 1229 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1230 examples=["([-+]?\\d+)"], 1231 title="Extraction Regex", 1232 ) 1233 max_waiting_time_in_seconds: Optional[float] = Field( 1234 None, 1235 description="Given the value extracted from the header is greater than this value, stop the stream.", 1236 examples=[3600], 1237 title="Max Waiting Time in Seconds", 1238 ) 1239 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1242class WaitUntilTimeFromHeader(BaseModel): 1243 type: Literal["WaitUntilTimeFromHeader"] 1244 header: str = Field( 1245 ..., 1246 description="The name of the response header defining how long to wait before retrying.", 1247 examples=["wait_time"], 1248 title="Response Header", 1249 ) 1250 min_wait: Optional[Union[float, str]] = Field( 1251 None, 1252 description="Minimum time to wait before retrying.", 1253 examples=[10, "60"], 1254 title="Minimum Wait Time", 1255 ) 1256 regex: Optional[str] = Field( 1257 None, 1258 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1259 examples=["([-+]?\\d+)"], 1260 title="Extraction Regex", 1261 ) 1262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1265class ComponentMappingDefinition(BaseModel): 1266 type: Literal["ComponentMappingDefinition"] 1267 field_path: List[str] = Field( 1268 ..., 1269 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1270 examples=[ 1271 ["name"], 1272 ["retriever", "requester", "url"], 1273 ["retriever", "requester", "{{ components_values.field }}"], 1274 ["*", "**", "name"], 1275 ], 1276 title="Field Path", 1277 ) 1278 value: str = Field( 1279 ..., 1280 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1281 examples=[ 1282 "{{ components_values['updates'] }}", 1283 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1284 "{{ config['segment_id'] }}", 1285 "{{ stream_slice['parent_id'] }}", 1286 "{{ stream_slice['extra_fields']['name'] }}", 1287 ], 1288 title="Value", 1289 ) 1290 value_type: Optional[ValueType] = Field( 1291 None, 1292 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1293 title="Value Type", 1294 ) 1295 create_or_update: Optional[bool] = Field( 1296 False, 1297 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1298 title="Create or Update", 1299 ) 1300 condition: Optional[str] = Field( 1301 None, 1302 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1303 examples=[ 1304 "{{ components_values.get('cursor_field', None) }}", 1305 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1306 ], 1307 title="Condition", 1308 ) 1309 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1312class StreamConfig(BaseModel): 1313 type: Literal["StreamConfig"] 1314 configs_pointer: List[str] = Field( 1315 ..., 1316 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1317 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1318 title="Configs Pointer", 1319 ) 1320 default_values: Optional[List[Dict[str, Any]]] = Field( 1321 None, 1322 description="A list of default values, each matching the structure expected from the parsed component value.", 1323 title="Default Values", 1324 ) 1325 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1328class ConfigComponentsResolver(BaseModel): 1329 type: Literal["ConfigComponentsResolver"] 1330 stream_config: Union[List[StreamConfig], StreamConfig] 1331 components_mapping: List[ComponentMappingDefinition] 1332 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1335class StreamParametersDefinition(BaseModel): 1336 type: Literal["StreamParametersDefinition"] 1337 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1338 ..., 1339 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1340 examples=[ 1341 [ 1342 { 1343 "name": "test stream", 1344 "$parameters": {"entity": "test entity"}, 1345 "primary_key": "test key", 1346 } 1347 ] 1348 ], 1349 title="Stream Parameters", 1350 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1353class ParametrizedComponentsResolver(BaseModel): 1354 type: Literal["ParametrizedComponentsResolver"] 1355 stream_parameters: StreamParametersDefinition 1356 components_mapping: List[ComponentMappingDefinition] 1357 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1382class ValidateAdheresToSchema(BaseModel): 1383 type: Literal["ValidateAdheresToSchema"] 1384 base_schema: Union[str, Dict[str, Any]] = Field( 1385 ..., 1386 description="The base JSON schema against which the user-provided schema will be validated.", 1387 examples=[ 1388 "{{ config['report_validation_schema'] }}", 1389 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1390 { 1391 "$schema": "http://json-schema.org/draft-07/schema#", 1392 "title": "Person", 1393 "type": "object", 1394 "properties": { 1395 "name": {"type": "string", "description": "The person's name"}, 1396 "age": { 1397 "type": "integer", 1398 "minimum": 0, 1399 "description": "The person's age", 1400 }, 1401 }, 1402 "required": ["name", "age"], 1403 }, 1404 ], 1405 title="Base JSON Schema", 1406 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1409class CustomValidationStrategy(BaseModel): 1410 class Config: 1411 extra = Extra.allow 1412 1413 type: Literal["CustomValidationStrategy"] 1414 class_name: str = Field( 1415 ..., 1416 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1417 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1418 title="Class Name", 1419 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1422class ConfigRemapField(BaseModel): 1423 type: Literal["ConfigRemapField"] 1424 map: Union[Dict[str, Any], str] = Field( 1425 ..., 1426 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1427 examples=[ 1428 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1429 "{{ config['status_mapping'] }}", 1430 ], 1431 title="Value Mapping", 1432 ) 1433 field_path: List[str] = Field( 1434 ..., 1435 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1436 examples=[ 1437 ["status"], 1438 ["data", "status"], 1439 ["data", "{{ config.name }}", "status"], 1440 ["data", "*", "status"], 1441 ], 1442 title="Field Path", 1443 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1446class ConfigRemoveFields(BaseModel): 1447 type: Literal["ConfigRemoveFields"] 1448 field_pointers: List[List[str]] = Field( 1449 ..., 1450 description="A list of field pointers to be removed from the config.", 1451 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1452 title="Field Pointers", 1453 ) 1454 condition: Optional[str] = Field( 1455 "", 1456 description="Fields will be removed if expression is evaluated to True.", 1457 examples=[ 1458 "{{ config['environemnt'] == 'sandbox' }}", 1459 "{{ property is integer }}", 1460 "{{ property|length > 5 }}", 1461 "{{ property == 'some_string_to_match' }}", 1462 ], 1463 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1466class CustomConfigTransformation(BaseModel): 1467 type: Literal["CustomConfigTransformation"] 1468 class_name: str = Field( 1469 ..., 1470 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1471 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1472 ) 1473 parameters: Optional[Dict[str, Any]] = Field( 1474 None, 1475 alias="$parameters", 1476 description="Additional parameters to be passed to the custom config transformation.", 1477 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1480class AddedFieldDefinition(BaseModel): 1481 type: Literal["AddedFieldDefinition"] 1482 path: List[str] = Field( 1483 ..., 1484 description="List of strings defining the path where to add the value on the record.", 1485 examples=[["segment_id"], ["metadata", "segment_id"]], 1486 title="Path", 1487 ) 1488 value: str = Field( 1489 ..., 1490 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1491 examples=[ 1492 "{{ record['updates'] }}", 1493 "{{ record['MetaData']['LastUpdatedTime'] }}", 1494 "{{ stream_partition['segment_id'] }}", 1495 ], 1496 title="Value", 1497 ) 1498 value_type: Optional[ValueType] = Field( 1499 None, 1500 description="Type of the value. If not specified, the type will be inferred from the value.", 1501 title="Value Type", 1502 ) 1503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1506class AddFields(BaseModel): 1507 type: Literal["AddFields"] 1508 fields: List[AddedFieldDefinition] = Field( 1509 ..., 1510 description="List of transformations (path and corresponding value) that will be added to the record.", 1511 title="Fields", 1512 ) 1513 condition: Optional[str] = Field( 1514 "", 1515 description="Fields will be added if expression is evaluated to True.", 1516 examples=[ 1517 "{{ property|string == '' }}", 1518 "{{ property is integer }}", 1519 "{{ property|length > 5 }}", 1520 "{{ property == 'some_string_to_match' }}", 1521 ], 1522 ) 1523 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1526class ApiKeyAuthenticator(BaseModel): 1527 type: Literal["ApiKeyAuthenticator"] 1528 api_token: Optional[str] = Field( 1529 None, 1530 description="The API key to inject in the request. Fill it in the user inputs.", 1531 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1532 title="API Key", 1533 ) 1534 header: Optional[str] = Field( 1535 None, 1536 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1537 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1538 title="Header Name", 1539 ) 1540 inject_into: Optional[RequestOption] = Field( 1541 None, 1542 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1543 examples=[ 1544 {"inject_into": "header", "field_name": "Authorization"}, 1545 {"inject_into": "request_parameter", "field_name": "authKey"}, 1546 ], 1547 title="Inject API Key Into Outgoing HTTP Request", 1548 ) 1549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1552class AuthFlow(BaseModel): 1553 auth_flow_type: Optional[AuthFlowType] = Field( 1554 None, description="The type of auth to use", title="Auth flow type" 1555 ) 1556 predicate_key: Optional[List[str]] = Field( 1557 None, 1558 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1559 examples=[["credentials", "auth_type"]], 1560 title="Predicate key", 1561 ) 1562 predicate_value: Optional[str] = Field( 1563 None, 1564 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1565 examples=["Oauth"], 1566 title="Predicate value", 1567 ) 1568 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1571class CheckStream(BaseModel): 1572 type: Literal["CheckStream"] 1573 stream_names: Optional[List[str]] = Field( 1574 None, 1575 description="Names of the streams to try reading from when running a check operation.", 1576 examples=[["users"], ["users", "contacts"]], 1577 title="Stream Names", 1578 ) 1579 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1582class IncrementingCountCursor(BaseModel): 1583 type: Literal["IncrementingCountCursor"] 1584 cursor_field: str = Field( 1585 ..., 1586 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1587 examples=["created_at", "{{ config['record_cursor'] }}"], 1588 title="Cursor Field", 1589 ) 1590 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1591 None, 1592 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1593 title="Allow Catalog Defined Cursor Field", 1594 ) 1595 start_value: Optional[Union[str, int]] = Field( 1596 None, 1597 description="The value that determines the earliest record that should be synced.", 1598 examples=[0, "{{ config['start_value'] }}"], 1599 title="Start Value", 1600 ) 1601 start_value_option: Optional[RequestOption] = Field( 1602 None, 1603 description="Optionally configures how the start value will be sent in requests to the source API.", 1604 title="Inject Start Value Into Outgoing HTTP Request", 1605 ) 1606 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1609class DatetimeBasedCursor(BaseModel): 1610 type: Literal["DatetimeBasedCursor"] 1611 clamping: Optional[Clamping] = Field( 1612 None, 1613 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1614 title="Date Range Clamping", 1615 ) 1616 cursor_field: str = Field( 1617 ..., 1618 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1619 examples=["created_at", "{{ config['record_cursor'] }}"], 1620 title="Cursor Field", 1621 ) 1622 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1623 None, 1624 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1625 title="Allow Catalog Defined Cursor Field", 1626 ) 1627 cursor_datetime_formats: Optional[List[str]] = Field( 1628 None, 1629 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1630 examples=[ 1631 "%Y-%m-%d", 1632 "%Y-%m-%d %H:%M:%S", 1633 "%Y-%m-%dT%H:%M:%S", 1634 "%Y-%m-%dT%H:%M:%SZ", 1635 "%Y-%m-%dT%H:%M:%S%z", 1636 "%Y-%m-%dT%H:%M:%S.%fZ", 1637 "%Y-%m-%dT%H:%M:%S.%f%z", 1638 "%Y-%m-%d %H:%M:%S.%f+00:00", 1639 "%s", 1640 "%ms", 1641 ], 1642 title="Cursor Datetime Formats", 1643 ) 1644 start_datetime: Union[MinMaxDatetime, str] = Field( 1645 ..., 1646 description="The datetime that determines the earliest record that should be synced.", 1647 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1648 title="Start Datetime", 1649 ) 1650 start_time_option: Optional[RequestOption] = Field( 1651 None, 1652 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1653 title="Inject Start Time Into Outgoing HTTP Request", 1654 ) 1655 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1656 None, 1657 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1658 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1659 title="End Datetime", 1660 ) 1661 end_time_option: Optional[RequestOption] = Field( 1662 None, 1663 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1664 title="Inject End Time Into Outgoing HTTP Request", 1665 ) 1666 datetime_format: str = Field( 1667 ..., 1668 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1669 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1670 title="Outgoing Datetime Format", 1671 ) 1672 cursor_granularity: Optional[str] = Field( 1673 None, 1674 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1675 examples=["PT1S"], 1676 title="Cursor Granularity", 1677 ) 1678 is_data_feed: Optional[bool] = Field( 1679 None, 1680 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1681 title="Data Feed API", 1682 ) 1683 is_client_side_incremental: Optional[bool] = Field( 1684 None, 1685 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1686 title="Client-side Incremental Filtering", 1687 ) 1688 is_compare_strictly: Optional[bool] = Field( 1689 False, 1690 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1691 title="Strict Start-End Time Comparison", 1692 ) 1693 global_substream_cursor: Optional[bool] = Field( 1694 False, 1695 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1696 title="Global Substream Cursor", 1697 ) 1698 lookback_window: Optional[str] = Field( 1699 None, 1700 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1701 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1702 title="Lookback Window", 1703 ) 1704 partition_field_end: Optional[str] = Field( 1705 None, 1706 description="Name of the partition start time field.", 1707 examples=["ending_time"], 1708 title="Partition Field End", 1709 ) 1710 partition_field_start: Optional[str] = Field( 1711 None, 1712 description="Name of the partition end time field.", 1713 examples=["starting_time"], 1714 title="Partition Field Start", 1715 ) 1716 step: Optional[str] = Field( 1717 None, 1718 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1719 examples=["P1W", "{{ config['step_increment'] }}"], 1720 title="Step", 1721 ) 1722 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1725class JwtAuthenticator(BaseModel): 1726 type: Literal["JwtAuthenticator"] 1727 secret_key: str = Field( 1728 ..., 1729 description="Secret used to sign the JSON web token.", 1730 examples=["{{ config['secret_key'] }}"], 1731 title="Secret Key", 1732 ) 1733 base64_encode_secret_key: Optional[bool] = Field( 1734 False, 1735 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1736 title="Base64-encode Secret Key", 1737 ) 1738 algorithm: Algorithm = Field( 1739 ..., 1740 description="Algorithm used to sign the JSON web token.", 1741 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1742 title="Algorithm", 1743 ) 1744 token_duration: Optional[int] = Field( 1745 1200, 1746 description="The amount of time in seconds a JWT token can be valid after being issued.", 1747 examples=[1200, 3600], 1748 title="Token Duration", 1749 ) 1750 header_prefix: Optional[str] = Field( 1751 None, 1752 description="The prefix to be used within the Authentication header.", 1753 examples=["Bearer", "Basic"], 1754 title="Header Prefix", 1755 ) 1756 jwt_headers: Optional[JwtHeaders] = Field( 1757 None, 1758 description="JWT headers used when signing JSON web token.", 1759 title="JWT Headers", 1760 ) 1761 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1762 None, 1763 description="Additional headers to be included with the JWT headers object.", 1764 title="Additional JWT Headers", 1765 ) 1766 jwt_payload: Optional[JwtPayload] = Field( 1767 None, 1768 description="JWT Payload used when signing JSON web token.", 1769 title="JWT Payload", 1770 ) 1771 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1772 None, 1773 description="Additional properties to be added to the JWT payload.", 1774 title="Additional JWT Payload Properties", 1775 ) 1776 passphrase: Optional[str] = Field( 1777 None, 1778 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1779 examples=["{{ config['passphrase'] }}"], 1780 title="Passphrase", 1781 ) 1782 request_option: Optional[RequestOption] = Field( 1783 None, 1784 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1785 title="Request Option", 1786 ) 1787 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1790class OAuthAuthenticator(BaseModel): 1791 type: Literal["OAuthAuthenticator"] 1792 client_id_name: Optional[str] = Field( 1793 "client_id", 1794 description="The name of the property to use to refresh the `access_token`.", 1795 examples=["custom_app_id"], 1796 title="Client ID Property Name", 1797 ) 1798 client_id: Optional[str] = Field( 1799 None, 1800 description="The OAuth client ID. Fill it in the user inputs.", 1801 examples=[ 1802 "{{ config['client_id'] }}", 1803 "{{ config['credentials']['client_id }}", 1804 ], 1805 title="Client ID", 1806 ) 1807 client_secret_name: Optional[str] = Field( 1808 "client_secret", 1809 description="The name of the property to use to refresh the `access_token`.", 1810 examples=["custom_app_secret"], 1811 title="Client Secret Property Name", 1812 ) 1813 client_secret: Optional[str] = Field( 1814 None, 1815 description="The OAuth client secret. Fill it in the user inputs.", 1816 examples=[ 1817 "{{ config['client_secret'] }}", 1818 "{{ config['credentials']['client_secret }}", 1819 ], 1820 title="Client Secret", 1821 ) 1822 refresh_token_name: Optional[str] = Field( 1823 "refresh_token", 1824 description="The name of the property to use to refresh the `access_token`.", 1825 examples=["custom_app_refresh_value"], 1826 title="Refresh Token Property Name", 1827 ) 1828 refresh_token: Optional[str] = Field( 1829 None, 1830 description="Credential artifact used to get a new access token.", 1831 examples=[ 1832 "{{ config['refresh_token'] }}", 1833 "{{ config['credentials]['refresh_token'] }}", 1834 ], 1835 title="Refresh Token", 1836 ) 1837 token_refresh_endpoint: Optional[str] = Field( 1838 None, 1839 description="The full URL to call to obtain a new access token.", 1840 examples=["https://connect.squareup.com/oauth2/token"], 1841 title="Token Refresh Endpoint", 1842 ) 1843 access_token_name: Optional[str] = Field( 1844 "access_token", 1845 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1846 examples=["access_token"], 1847 title="Access Token Property Name", 1848 ) 1849 access_token_value: Optional[str] = Field( 1850 None, 1851 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1852 examples=["secret_access_token_value"], 1853 title="Access Token Value", 1854 ) 1855 expires_in_name: Optional[str] = Field( 1856 "expires_in", 1857 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1858 examples=["expires_in"], 1859 title="Token Expiry Property Name", 1860 ) 1861 grant_type_name: Optional[str] = Field( 1862 "grant_type", 1863 description="The name of the property to use to refresh the `access_token`.", 1864 examples=["custom_grant_type"], 1865 title="Grant Type Property Name", 1866 ) 1867 grant_type: Optional[str] = Field( 1868 "refresh_token", 1869 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1870 examples=["refresh_token", "client_credentials"], 1871 title="Grant Type", 1872 ) 1873 refresh_request_body: Optional[Dict[str, Any]] = Field( 1874 None, 1875 description="Body of the request sent to get a new access token.", 1876 examples=[ 1877 { 1878 "applicationId": "{{ config['application_id'] }}", 1879 "applicationSecret": "{{ config['application_secret'] }}", 1880 "token": "{{ config['token'] }}", 1881 } 1882 ], 1883 title="Refresh Request Body", 1884 ) 1885 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1886 None, 1887 description="Headers of the request sent to get a new access token.", 1888 examples=[ 1889 { 1890 "Authorization": "<AUTH_TOKEN>", 1891 "Content-Type": "application/x-www-form-urlencoded", 1892 } 1893 ], 1894 title="Refresh Request Headers", 1895 ) 1896 scopes: Optional[List[str]] = Field( 1897 None, 1898 description="List of scopes that should be granted to the access token.", 1899 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1900 title="Scopes", 1901 ) 1902 token_expiry_date: Optional[str] = Field( 1903 None, 1904 description="The access token expiry date.", 1905 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1906 title="Token Expiry Date", 1907 ) 1908 token_expiry_date_format: Optional[str] = Field( 1909 None, 1910 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1911 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1912 title="Token Expiry Date Format", 1913 ) 1914 refresh_token_error_status_codes: Optional[List[int]] = Field( 1915 None, 1916 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 1917 examples=[[400, 500]], 1918 title="Refresh Token Error Status Codes", 1919 ) 1920 refresh_token_error_key: Optional[str] = Field( 1921 None, 1922 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 1923 examples=["error"], 1924 title="Refresh Token Error Key", 1925 ) 1926 refresh_token_error_values: Optional[List[str]] = Field( 1927 None, 1928 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 1929 examples=[["invalid_grant", "invalid_permissions"]], 1930 title="Refresh Token Error Values", 1931 ) 1932 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1933 None, 1934 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1935 title="Refresh Token Updater", 1936 ) 1937 profile_assertion: Optional[JwtAuthenticator] = Field( 1938 None, 1939 description="The authenticator being used to authenticate the client authenticator.", 1940 title="Profile Assertion", 1941 ) 1942 use_profile_assertion: Optional[bool] = Field( 1943 False, 1944 description="Enable using profile assertion as a flow for OAuth authorization.", 1945 title="Use Profile Assertion", 1946 ) 1947 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1950class FixedWindowCallRatePolicy(BaseModel): 1951 class Config: 1952 extra = Extra.allow 1953 1954 type: Literal["FixedWindowCallRatePolicy"] 1955 period: str = Field( 1956 ..., description="The time interval for the rate limit window.", title="Period" 1957 ) 1958 call_limit: int = Field( 1959 ..., 1960 description="The maximum number of calls allowed within the period.", 1961 title="Call Limit", 1962 ) 1963 matchers: List[HttpRequestRegexMatcher] = Field( 1964 ..., 1965 description="List of matchers that define which requests this policy applies to.", 1966 title="Matchers", 1967 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1970class MovingWindowCallRatePolicy(BaseModel): 1971 class Config: 1972 extra = Extra.allow 1973 1974 type: Literal["MovingWindowCallRatePolicy"] 1975 rates: List[Rate] = Field( 1976 ..., 1977 description="List of rates that define the call limits for different time intervals.", 1978 title="Rates", 1979 ) 1980 matchers: List[HttpRequestRegexMatcher] = Field( 1981 ..., 1982 description="List of matchers that define which requests this policy applies to.", 1983 title="Matchers", 1984 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1987class UnlimitedCallRatePolicy(BaseModel): 1988 class Config: 1989 extra = Extra.allow 1990 1991 type: Literal["UnlimitedCallRatePolicy"] 1992 matchers: List[HttpRequestRegexMatcher] = Field( 1993 ..., 1994 description="List of matchers that define which requests this policy applies to.", 1995 title="Matchers", 1996 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1999class DefaultErrorHandler(BaseModel): 2000 type: Literal["DefaultErrorHandler"] 2001 backoff_strategies: Optional[ 2002 List[ 2003 Union[ 2004 ConstantBackoffStrategy, 2005 ExponentialBackoffStrategy, 2006 WaitTimeFromHeader, 2007 WaitUntilTimeFromHeader, 2008 CustomBackoffStrategy, 2009 ] 2010 ] 2011 ] = Field( 2012 None, 2013 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2014 title="Backoff Strategies", 2015 ) 2016 max_retries: Optional[int] = Field( 2017 5, 2018 description="The maximum number of time to retry a retryable request before giving up and failing.", 2019 examples=[5, 0, 10], 2020 title="Max Retry Count", 2021 ) 2022 response_filters: Optional[List[HttpResponseFilter]] = Field( 2023 None, 2024 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2025 title="Response Filters", 2026 ) 2027 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2030class DefaultPaginator(BaseModel): 2031 type: Literal["DefaultPaginator"] 2032 pagination_strategy: Union[ 2033 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2034 ] = Field( 2035 ..., 2036 description="Strategy defining how records are paginated.", 2037 title="Pagination Strategy", 2038 ) 2039 page_size_option: Optional[RequestOption] = Field( 2040 None, title="Inject Page Size Into Outgoing HTTP Request" 2041 ) 2042 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2043 None, 2044 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2045 title="Inject Page Token Into Outgoing HTTP Request", 2046 ) 2047 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2050class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2051 type: Literal["ApiKey"] 2052 inject_into: RequestOption = Field( 2053 ..., 2054 description="Configure how the API Key will be sent in requests to the source API.", 2055 examples=[ 2056 {"inject_into": "header", "field_name": "Authorization"}, 2057 {"inject_into": "request_parameter", "field_name": "authKey"}, 2058 ], 2059 title="Inject API Key Into Outgoing HTTP Request", 2060 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2063class JsonSchemaPropertySelector(BaseModel): 2064 type: Literal["JsonSchemaPropertySelector"] 2065 transformations: Optional[ 2066 List[ 2067 Union[ 2068 AddFields, 2069 RemoveFields, 2070 KeysToLower, 2071 KeysToSnakeCase, 2072 FlattenFields, 2073 DpathFlattenFields, 2074 KeysReplace, 2075 CustomTransformation, 2076 ] 2077 ] 2078 ] = Field( 2079 None, 2080 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2081 title="Transformations", 2082 ) 2083 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2086class ListPartitionRouter(BaseModel): 2087 type: Literal["ListPartitionRouter"] 2088 cursor_field: str = Field( 2089 ..., 2090 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2091 examples=["section", "{{ config['section_key'] }}"], 2092 title="Current Partition Value Identifier", 2093 ) 2094 values: Union[str, List[str]] = Field( 2095 ..., 2096 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2097 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2098 title="Partition Values", 2099 ) 2100 request_option: Optional[RequestOption] = Field( 2101 None, 2102 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2103 title="Inject Partition Value Into Outgoing HTTP Request", 2104 ) 2105 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2108class RecordSelector(BaseModel): 2109 type: Literal["RecordSelector"] 2110 extractor: Union[DpathExtractor, CustomRecordExtractor] 2111 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2112 None, 2113 description="Responsible for filtering records to be emitted by the Source.", 2114 title="Record Filter", 2115 ) 2116 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2117 None, 2118 description="Responsible for normalization according to the schema.", 2119 title="Schema Normalization", 2120 ) 2121 transform_before_filtering: Optional[bool] = Field( 2122 None, 2123 description="If true, transformation will be applied before record filtering.", 2124 title="Transform Before Filtering", 2125 ) 2126 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
PaginationReset(pydantic.v1.main.BaseModel):
2129class PaginationReset(BaseModel): 2130 type: Literal["PaginationReset"] 2131 action: Action1 2132 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2135class GzipDecoder(BaseModel): 2136 type: Literal["GzipDecoder"] 2137 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2140class RequestBodyGraphQL(BaseModel): 2141 type: Literal["RequestBodyGraphQL"] 2142 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2145class DpathValidator(BaseModel): 2146 type: Literal["DpathValidator"] 2147 field_path: List[str] = Field( 2148 ..., 2149 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2150 examples=[ 2151 ["data"], 2152 ["data", "records"], 2153 ["data", "{{ parameters.name }}"], 2154 ["data", "*", "record"], 2155 ], 2156 title="Field Path", 2157 ) 2158 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2159 ..., 2160 description="The condition that the specified config value will be evaluated against", 2161 title="Validation Strategy", 2162 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2165class PredicateValidator(BaseModel): 2166 type: Literal["PredicateValidator"] 2167 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2168 ..., 2169 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2170 examples=[ 2171 "test-value", 2172 "{{ config['api_version'] }}", 2173 "{{ config['tenant_id'] }}", 2174 123, 2175 ], 2176 title="Value", 2177 ) 2178 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2179 ..., 2180 description="The validation strategy to apply to the value.", 2181 title="Validation Strategy", 2182 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2185class ConfigAddFields(BaseModel): 2186 type: Literal["ConfigAddFields"] 2187 fields: List[AddedFieldDefinition] = Field( 2188 ..., 2189 description="A list of transformations (path and corresponding value) that will be added to the config.", 2190 title="Fields", 2191 ) 2192 condition: Optional[str] = Field( 2193 "", 2194 description="Fields will be added if expression is evaluated to True.", 2195 examples=[ 2196 "{{ config['environemnt'] == 'sandbox' }}", 2197 "{{ property is integer }}", 2198 "{{ property|length > 5 }}", 2199 "{{ property == 'some_string_to_match' }}", 2200 ], 2201 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2204class CompositeErrorHandler(BaseModel): 2205 type: Literal["CompositeErrorHandler"] 2206 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2207 Field( 2208 ..., 2209 description="List of error handlers to iterate on to determine how to handle a failed response.", 2210 title="Error Handlers", 2211 ) 2212 ) 2213 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2216class HTTPAPIBudget(BaseModel): 2217 class Config: 2218 extra = Extra.allow 2219 2220 type: Literal["HTTPAPIBudget"] 2221 policies: List[ 2222 Union[ 2223 FixedWindowCallRatePolicy, 2224 MovingWindowCallRatePolicy, 2225 UnlimitedCallRatePolicy, 2226 ] 2227 ] = Field( 2228 ..., 2229 description="List of call rate policies that define how many calls are allowed.", 2230 title="Policies", 2231 ) 2232 ratelimit_reset_header: Optional[str] = Field( 2233 "ratelimit-reset", 2234 description="The HTTP response header name that indicates when the rate limit resets.", 2235 title="Rate Limit Reset Header", 2236 ) 2237 ratelimit_remaining_header: Optional[str] = Field( 2238 "ratelimit-remaining", 2239 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2240 title="Rate Limit Remaining Header", 2241 ) 2242 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2243 [429], 2244 description="List of HTTP status codes that indicate a rate limit has been hit.", 2245 title="Status Codes for Rate Limit Hit", 2246 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2249class ZipfileDecoder(BaseModel): 2250 class Config: 2251 extra = Extra.allow 2252 2253 type: Literal["ZipfileDecoder"] 2254 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2255 ..., 2256 description="Parser to parse the decompressed data from the zipfile(s).", 2257 title="Parser", 2258 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2261class ConfigMigration(BaseModel): 2262 type: Literal["ConfigMigration"] 2263 description: Optional[str] = Field( 2264 None, description="The description/purpose of the config migration." 2265 ) 2266 transformations: List[ 2267 Union[ 2268 ConfigRemapField, 2269 ConfigAddFields, 2270 ConfigRemoveFields, 2271 CustomConfigTransformation, 2272 ] 2273 ] = Field( 2274 ..., 2275 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2276 title="Transformations", 2277 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2280class ConfigNormalizationRules(BaseModel): 2281 class Config: 2282 extra = Extra.forbid 2283 2284 type: Literal["ConfigNormalizationRules"] 2285 config_migrations: Optional[List[ConfigMigration]] = Field( 2286 [], 2287 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2288 title="Config Migrations", 2289 ) 2290 transformations: Optional[ 2291 List[ 2292 Union[ 2293 ConfigRemapField, 2294 ConfigAddFields, 2295 ConfigRemoveFields, 2296 CustomConfigTransformation, 2297 ] 2298 ] 2299 ] = Field( 2300 [], 2301 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2302 title="Transformations", 2303 ) 2304 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2305 [], 2306 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2307 title="Validations", 2308 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2311class Spec(BaseModel): 2312 type: Literal["Spec"] 2313 connection_specification: Dict[str, Any] = Field( 2314 ..., 2315 description="A connection specification describing how a the connector can be configured.", 2316 title="Connection Specification", 2317 ) 2318 documentation_url: Optional[str] = Field( 2319 None, 2320 description="URL of the connector's documentation page.", 2321 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2322 title="Documentation URL", 2323 ) 2324 advanced_auth: Optional[AuthFlow] = Field( 2325 None, 2326 description="Advanced specification for configuring the authentication flow.", 2327 title="Advanced Auth", 2328 ) 2329 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2330 None, title="Config Normalization Rules" 2331 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2334class DeclarativeSource1(BaseModel): 2335 class Config: 2336 extra = Extra.forbid 2337 2338 type: Literal["DeclarativeSource"] 2339 check: Union[CheckStream, CheckDynamicStream] 2340 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2341 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2342 version: str = Field( 2343 ..., 2344 description="The version of the Airbyte CDK used to build and test the source.", 2345 ) 2346 schemas: Optional[Schemas] = None 2347 definitions: Optional[Dict[str, Any]] = None 2348 spec: Optional[Spec] = None 2349 concurrency_level: Optional[ConcurrencyLevel] = None 2350 api_budget: Optional[HTTPAPIBudget] = None 2351 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2352 None, 2353 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2354 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2355 title="Maximum Concurrent Asynchronous Jobs", 2356 ) 2357 metadata: Optional[Dict[str, Any]] = Field( 2358 None, 2359 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2360 ) 2361 description: Optional[str] = Field( 2362 None, 2363 description="A description of the connector. It will be presented on the Source documentation page.", 2364 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2367class DeclarativeSource2(BaseModel): 2368 class Config: 2369 extra = Extra.forbid 2370 2371 type: Literal["DeclarativeSource"] 2372 check: Union[CheckStream, CheckDynamicStream] 2373 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2374 None 2375 ) 2376 dynamic_streams: List[DynamicDeclarativeStream] 2377 version: str = Field( 2378 ..., 2379 description="The version of the Airbyte CDK used to build and test the source.", 2380 ) 2381 schemas: Optional[Schemas] = None 2382 definitions: Optional[Dict[str, Any]] = None 2383 spec: Optional[Spec] = None 2384 concurrency_level: Optional[ConcurrencyLevel] = None 2385 api_budget: Optional[HTTPAPIBudget] = None 2386 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2387 None, 2388 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2389 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2390 title="Maximum Concurrent Asynchronous Jobs", 2391 ) 2392 metadata: Optional[Dict[str, Any]] = Field( 2393 None, 2394 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2395 ) 2396 description: Optional[str] = Field( 2397 None, 2398 description="A description of the connector. It will be presented on the Source documentation page.", 2399 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2402class DeclarativeSource(BaseModel): 2403 class Config: 2404 extra = Extra.forbid 2405 2406 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2407 ..., 2408 description="An API source that extracts data according to its declarative components.", 2409 title="DeclarativeSource", 2410 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2413class SelectiveAuthenticator(BaseModel): 2414 class Config: 2415 extra = Extra.allow 2416 2417 type: Literal["SelectiveAuthenticator"] 2418 authenticator_selection_path: List[str] = Field( 2419 ..., 2420 description="Path of the field in config with selected authenticator name", 2421 examples=[["auth"], ["auth", "type"]], 2422 title="Authenticator Selection Path", 2423 ) 2424 authenticators: Dict[ 2425 str, 2426 Union[ 2427 ApiKeyAuthenticator, 2428 BasicHttpAuthenticator, 2429 BearerAuthenticator, 2430 OAuthAuthenticator, 2431 JwtAuthenticator, 2432 SessionTokenAuthenticator, 2433 LegacySessionTokenAuthenticator, 2434 CustomAuthenticator, 2435 NoAuth, 2436 ], 2437 ] = Field( 2438 ..., 2439 description="Authenticators to select from.", 2440 examples=[ 2441 { 2442 "authenticators": { 2443 "token": "#/definitions/ApiKeyAuthenticator", 2444 "oauth": "#/definitions/OAuthAuthenticator", 2445 "jwt": "#/definitions/JwtAuthenticator", 2446 } 2447 } 2448 ], 2449 title="Authenticators", 2450 ) 2451 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2454class ConditionalStreams(BaseModel): 2455 type: Literal["ConditionalStreams"] 2456 condition: str = Field( 2457 ..., 2458 description="Condition that will be evaluated to determine if a set of streams should be available.", 2459 examples=["{{ config['is_sandbox'] }}"], 2460 title="Condition", 2461 ) 2462 streams: List[DeclarativeStream] = Field( 2463 ..., 2464 description="Streams that will be used during an operation based on the condition.", 2465 title="Streams", 2466 ) 2467 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2470class FileUploader(BaseModel): 2471 type: Literal["FileUploader"] 2472 requester: Union[HttpRequester, CustomRequester] = Field( 2473 ..., 2474 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2475 ) 2476 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2477 ..., 2478 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2479 ) 2480 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2481 None, 2482 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2483 ) 2484 filename_extractor: Optional[str] = Field( 2485 None, 2486 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2487 examples=[ 2488 "{{ record.id }}/{{ record.file_name }}/", 2489 "{{ record.id }}_{{ record.file_name }}/", 2490 ], 2491 ) 2492 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2495class DeclarativeStream(BaseModel): 2496 class Config: 2497 extra = Extra.allow 2498 2499 type: Literal["DeclarativeStream"] 2500 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2501 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2502 ..., 2503 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2504 title="Retriever", 2505 ) 2506 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2507 None, 2508 description="Component used to fetch data incrementally based on a time field in the data.", 2509 title="Incremental Sync", 2510 ) 2511 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2512 schema_loader: Optional[ 2513 Union[ 2514 InlineSchemaLoader, 2515 DynamicSchemaLoader, 2516 JsonFileSchemaLoader, 2517 List[ 2518 Union[ 2519 InlineSchemaLoader, 2520 DynamicSchemaLoader, 2521 JsonFileSchemaLoader, 2522 CustomSchemaLoader, 2523 ] 2524 ], 2525 CustomSchemaLoader, 2526 ] 2527 ] = Field( 2528 None, 2529 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2530 title="Schema Loader", 2531 ) 2532 transformations: Optional[ 2533 List[ 2534 Union[ 2535 AddFields, 2536 RemoveFields, 2537 KeysToLower, 2538 KeysToSnakeCase, 2539 FlattenFields, 2540 DpathFlattenFields, 2541 KeysReplace, 2542 CustomTransformation, 2543 ] 2544 ] 2545 ] = Field( 2546 None, 2547 description="A list of transformations to be applied to each output record.", 2548 title="Transformations", 2549 ) 2550 state_migrations: Optional[ 2551 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2552 ] = Field( 2553 [], 2554 description="Array of state migrations to be applied on the input state", 2555 title="State Migrations", 2556 ) 2557 file_uploader: Optional[FileUploader] = Field( 2558 None, 2559 description="(experimental) Describes how to fetch a file", 2560 title="File Uploader", 2561 ) 2562 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2565class SessionTokenAuthenticator(BaseModel): 2566 type: Literal["SessionTokenAuthenticator"] 2567 login_requester: HttpRequester = Field( 2568 ..., 2569 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2570 examples=[ 2571 { 2572 "type": "HttpRequester", 2573 "url_base": "https://my_api.com", 2574 "path": "/login", 2575 "authenticator": { 2576 "type": "BasicHttpAuthenticator", 2577 "username": "{{ config.username }}", 2578 "password": "{{ config.password }}", 2579 }, 2580 } 2581 ], 2582 title="Login Requester", 2583 ) 2584 session_token_path: List[str] = Field( 2585 ..., 2586 description="The path in the response body returned from the login requester to the session token.", 2587 examples=[["access_token"], ["result", "token"]], 2588 title="Session Token Path", 2589 ) 2590 expiration_duration: Optional[str] = Field( 2591 None, 2592 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2593 examples=["PT1H", "P1D"], 2594 title="Expiration Duration", 2595 ) 2596 request_authentication: Union[ 2597 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2598 ] = Field( 2599 ..., 2600 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2601 title="Data Request Authentication", 2602 ) 2603 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2604 None, description="Component used to decode the response.", title="Decoder" 2605 ) 2606 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2609class HttpRequester(BaseModelWithDeprecations): 2610 type: Literal["HttpRequester"] 2611 url_base: Optional[str] = Field( 2612 None, 2613 deprecated=True, 2614 deprecation_message="Use `url` field instead.", 2615 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2616 examples=[ 2617 "https://connect.squareup.com/v2", 2618 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2619 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2620 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2621 ], 2622 title="API Base URL", 2623 ) 2624 url: Optional[str] = Field( 2625 None, 2626 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2627 examples=[ 2628 "https://connect.squareup.com/v2", 2629 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2630 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2631 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2632 ], 2633 title="API Endpoint URL", 2634 ) 2635 path: Optional[str] = Field( 2636 None, 2637 deprecated=True, 2638 deprecation_message="Use `url` field instead.", 2639 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2640 examples=[ 2641 "/products", 2642 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2643 "/trades/{{ config['symbol_id'] }}/history", 2644 ], 2645 title="URL Path", 2646 ) 2647 http_method: Optional[HttpMethod] = Field( 2648 HttpMethod.GET, 2649 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2650 examples=["GET", "POST"], 2651 title="HTTP Method", 2652 ) 2653 authenticator: Optional[ 2654 Union[ 2655 ApiKeyAuthenticator, 2656 BasicHttpAuthenticator, 2657 BearerAuthenticator, 2658 OAuthAuthenticator, 2659 JwtAuthenticator, 2660 SessionTokenAuthenticator, 2661 SelectiveAuthenticator, 2662 CustomAuthenticator, 2663 NoAuth, 2664 LegacySessionTokenAuthenticator, 2665 ] 2666 ] = Field( 2667 None, 2668 description="Authentication method to use for requests sent to the API.", 2669 title="Authenticator", 2670 ) 2671 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2672 None, 2673 deprecated=True, 2674 deprecation_message="Use `query_properties` field instead.", 2675 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2676 title="Fetch Properties from Endpoint", 2677 ) 2678 query_properties: Optional[QueryProperties] = Field( 2679 None, 2680 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2681 title="Query Properties", 2682 ) 2683 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2684 None, 2685 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2686 examples=[ 2687 {"unit": "day"}, 2688 { 2689 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2690 }, 2691 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2692 {"sort_by[asc]": "updated_at"}, 2693 ], 2694 title="Query Parameters", 2695 ) 2696 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2697 None, 2698 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2699 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2700 title="Request Headers", 2701 ) 2702 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2703 None, 2704 deprecated=True, 2705 deprecation_message="Use `request_body` field instead.", 2706 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2707 examples=[ 2708 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2709 ], 2710 title="Request Body Payload (Non-JSON)", 2711 ) 2712 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2713 None, 2714 deprecated=True, 2715 deprecation_message="Use `request_body` field instead.", 2716 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2717 examples=[ 2718 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2719 {"key": "{{ config['value'] }}"}, 2720 {"sort": {"field": "updated_at", "order": "ascending"}}, 2721 ], 2722 title="Request Body JSON Payload", 2723 ) 2724 request_body: Optional[ 2725 Union[ 2726 RequestBodyPlainText, 2727 RequestBodyUrlEncodedForm, 2728 RequestBodyJsonObject, 2729 RequestBodyGraphQL, 2730 ] 2731 ] = Field( 2732 None, 2733 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2734 title="Request Body", 2735 ) 2736 error_handler: Optional[ 2737 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2738 ] = Field( 2739 None, 2740 description="Error handler component that defines how to handle errors.", 2741 title="Error Handler", 2742 ) 2743 use_cache: Optional[bool] = Field( 2744 False, 2745 description="Enables stream requests caching. This field is automatically set by the CDK.", 2746 title="Use Cache", 2747 ) 2748 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2751class DynamicSchemaLoader(BaseModel): 2752 type: Literal["DynamicSchemaLoader"] 2753 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2754 ..., 2755 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2756 title="Retriever", 2757 ) 2758 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2759 None, 2760 description="Responsible for filtering fields to be added to json schema.", 2761 title="Schema Filter", 2762 ) 2763 schema_transformations: Optional[ 2764 List[ 2765 Union[ 2766 AddFields, 2767 RemoveFields, 2768 KeysToLower, 2769 KeysToSnakeCase, 2770 FlattenFields, 2771 DpathFlattenFields, 2772 KeysReplace, 2773 CustomTransformation, 2774 ] 2775 ] 2776 ] = Field( 2777 None, 2778 description="A list of transformations to be applied to the schema.", 2779 title="Schema Transformations", 2780 ) 2781 schema_type_identifier: SchemaTypeIdentifier 2782 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2785class ParentStreamConfig(BaseModel): 2786 type: Literal["ParentStreamConfig"] 2787 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2788 ..., description="Reference to the parent stream.", title="Parent Stream" 2789 ) 2790 parent_key: str = Field( 2791 ..., 2792 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2793 examples=["id", "{{ config['parent_record_id'] }}"], 2794 title="Parent Key", 2795 ) 2796 partition_field: str = Field( 2797 ..., 2798 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2799 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2800 title="Current Parent Key Value Identifier", 2801 ) 2802 request_option: Optional[RequestOption] = Field( 2803 None, 2804 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2805 title="Request Option", 2806 ) 2807 incremental_dependency: Optional[bool] = Field( 2808 False, 2809 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2810 title="Incremental Dependency", 2811 ) 2812 lazy_read_pointer: Optional[List[str]] = Field( 2813 [], 2814 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2815 title="Lazy Read Pointer", 2816 ) 2817 extra_fields: Optional[List[List[str]]] = Field( 2818 None, 2819 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2820 title="Extra Fields", 2821 ) 2822 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2825class PropertiesFromEndpoint(BaseModel): 2826 type: Literal["PropertiesFromEndpoint"] 2827 property_field_path: List[str] = Field( 2828 ..., 2829 description="Describes the path to the field that should be extracted", 2830 examples=[["name"]], 2831 ) 2832 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2833 ..., 2834 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2835 ) 2836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2839class QueryProperties(BaseModel): 2840 type: Literal["QueryProperties"] 2841 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2842 ..., 2843 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2844 title="Property List", 2845 ) 2846 always_include_properties: Optional[List[str]] = Field( 2847 None, 2848 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2849 title="Always Include Properties", 2850 ) 2851 property_chunking: Optional[PropertyChunking] = Field( 2852 None, 2853 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2854 title="Property Chunking", 2855 ) 2856 property_selector: Optional[JsonSchemaPropertySelector] = Field( 2857 None, 2858 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 2859 title="Property Selector", 2860 ) 2861 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2864class StateDelegatingStream(BaseModel): 2865 type: Literal["StateDelegatingStream"] 2866 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2867 full_refresh_stream: DeclarativeStream = Field( 2868 ..., 2869 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2870 title="Full Refresh Stream", 2871 ) 2872 incremental_stream: DeclarativeStream = Field( 2873 ..., 2874 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2875 title="Incremental Stream", 2876 ) 2877 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2880class SimpleRetriever(BaseModel): 2881 type: Literal["SimpleRetriever"] 2882 requester: Union[HttpRequester, CustomRequester] = Field( 2883 ..., 2884 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2885 ) 2886 decoder: Optional[ 2887 Union[ 2888 JsonDecoder, 2889 XmlDecoder, 2890 CsvDecoder, 2891 JsonlDecoder, 2892 GzipDecoder, 2893 IterableDecoder, 2894 ZipfileDecoder, 2895 CustomDecoder, 2896 ] 2897 ] = Field( 2898 None, 2899 description="Component decoding the response so records can be extracted.", 2900 title="HTTP Response Format", 2901 ) 2902 record_selector: RecordSelector = Field( 2903 ..., 2904 description="Component that describes how to extract records from a HTTP response.", 2905 ) 2906 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2907 None, 2908 description="Paginator component that describes how to navigate through the API's pages.", 2909 ) 2910 pagination_reset: Optional[PaginationReset] = Field( 2911 None, 2912 description="Describes what triggers pagination reset and how to handle it.", 2913 ) 2914 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2915 False, 2916 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2917 ) 2918 partition_router: Optional[ 2919 Union[ 2920 SubstreamPartitionRouter, 2921 ListPartitionRouter, 2922 GroupingPartitionRouter, 2923 CustomPartitionRouter, 2924 List[ 2925 Union[ 2926 SubstreamPartitionRouter, 2927 ListPartitionRouter, 2928 GroupingPartitionRouter, 2929 CustomPartitionRouter, 2930 ] 2931 ], 2932 ] 2933 ] = Field( 2934 None, 2935 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2936 title="Partition Router", 2937 ) 2938 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2941class AsyncRetriever(BaseModel): 2942 type: Literal["AsyncRetriever"] 2943 record_selector: RecordSelector = Field( 2944 ..., 2945 description="Component that describes how to extract records from a HTTP response.", 2946 ) 2947 status_mapping: AsyncJobStatusMap = Field( 2948 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2949 ) 2950 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2951 ..., description="Responsible for fetching the actual status of the async job." 2952 ) 2953 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2954 None, 2955 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2956 ) 2957 download_extractor: Optional[ 2958 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2959 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2960 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2961 ..., 2962 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2963 ) 2964 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2965 ..., 2966 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2967 ) 2968 polling_job_timeout: Optional[Union[int, str]] = Field( 2969 None, 2970 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2971 ) 2972 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2973 None, 2974 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2975 ) 2976 download_requester: Union[HttpRequester, CustomRequester] = Field( 2977 ..., 2978 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2979 ) 2980 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2981 None, 2982 description="Paginator component that describes how to navigate through the API's pages during download.", 2983 ) 2984 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2985 None, 2986 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2987 ) 2988 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2989 None, 2990 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2991 ) 2992 partition_router: Optional[ 2993 Union[ 2994 ListPartitionRouter, 2995 SubstreamPartitionRouter, 2996 GroupingPartitionRouter, 2997 CustomPartitionRouter, 2998 List[ 2999 Union[ 3000 ListPartitionRouter, 3001 SubstreamPartitionRouter, 3002 GroupingPartitionRouter, 3003 CustomPartitionRouter, 3004 ] 3005 ], 3006 ] 3007 ] = Field( 3008 [], 3009 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3010 title="Partition Router", 3011 ) 3012 decoder: Optional[ 3013 Union[ 3014 CsvDecoder, 3015 GzipDecoder, 3016 JsonDecoder, 3017 JsonlDecoder, 3018 IterableDecoder, 3019 XmlDecoder, 3020 ZipfileDecoder, 3021 CustomDecoder, 3022 ] 3023 ] = Field( 3024 None, 3025 description="Component decoding the response so records can be extracted.", 3026 title="HTTP Response Format", 3027 ) 3028 download_decoder: Optional[ 3029 Union[ 3030 CsvDecoder, 3031 GzipDecoder, 3032 JsonDecoder, 3033 JsonlDecoder, 3034 IterableDecoder, 3035 XmlDecoder, 3036 ZipfileDecoder, 3037 CustomDecoder, 3038 ] 3039 ] = Field( 3040 None, 3041 description="Component decoding the download response so records can be extracted.", 3042 title="Download HTTP Response Format", 3043 ) 3044 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3047class SubstreamPartitionRouter(BaseModel): 3048 type: Literal["SubstreamPartitionRouter"] 3049 parent_stream_configs: List[ParentStreamConfig] = Field( 3050 ..., 3051 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3052 title="Parent Stream Configs", 3053 ) 3054 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3057class GroupingPartitionRouter(BaseModel): 3058 type: Literal["GroupingPartitionRouter"] 3059 group_size: int = Field( 3060 ..., 3061 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3062 examples=[10, 50], 3063 title="Group Size", 3064 ) 3065 underlying_partition_router: Union[ 3066 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3067 ] = Field( 3068 ..., 3069 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3070 title="Underlying Partition Router", 3071 ) 3072 deduplicate: Optional[bool] = Field( 3073 True, 3074 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3075 title="Deduplicate Partitions", 3076 ) 3077 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3080class HttpComponentsResolver(BaseModel): 3081 type: Literal["HttpComponentsResolver"] 3082 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3083 ..., 3084 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3085 title="Retriever", 3086 ) 3087 components_mapping: List[ComponentMappingDefinition] 3088 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3091class DynamicDeclarativeStream(BaseModel): 3092 type: Literal["DynamicDeclarativeStream"] 3093 name: Optional[str] = Field( 3094 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3095 ) 3096 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3097 ..., description="Reference to the stream template.", title="Stream Template" 3098 ) 3099 components_resolver: Union[ 3100 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3101 ] = Field( 3102 ..., 3103 description="Component resolve and populates stream templates with components values.", 3104 title="Components Resolver", 3105 ) 3106 use_parent_parameters: Optional[bool] = Field( 3107 True, 3108 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3109 title="Use Parent Parameters", 3110 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]