airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class DynamicStreamCheckConfig(BaseModel): 50 type: Literal["DynamicStreamCheckConfig"] 51 dynamic_stream_name: str = Field( 52 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 53 ) 54 stream_count: Optional[int] = Field( 55 0, 56 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 57 title="Stream Count", 58 ) 59 60 61class CheckDynamicStream(BaseModel): 62 type: Literal["CheckDynamicStream"] 63 stream_count: int = Field( 64 ..., 65 description="Numbers of the streams to try reading from when running a check operation.", 66 title="Stream Count", 67 ) 68 use_check_availability: Optional[bool] = Field( 69 True, 70 description="Enables stream check availability. This field is automatically set by the CDK.", 71 title="Use Check Availability", 72 ) 73 74 75class ConcurrencyLevel(BaseModel): 76 type: Optional[Literal["ConcurrencyLevel"]] = None 77 default_concurrency: Union[int, str] = Field( 78 ..., 79 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 80 examples=[10, "{{ config['num_workers'] or 10 }}"], 81 title="Default Concurrency", 82 ) 83 max_concurrency: Optional[int] = Field( 84 None, 85 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 86 examples=[20, 100], 87 title="Max Concurrency", 88 ) 89 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 90 91 92class ConstantBackoffStrategy(BaseModel): 93 type: Literal["ConstantBackoffStrategy"] 94 backoff_time_in_seconds: Union[float, str] = Field( 95 ..., 96 description="Backoff time in seconds.", 97 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 98 title="Backoff Time", 99 ) 100 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 101 102 103class CursorPagination(BaseModel): 104 type: Literal["CursorPagination"] 105 cursor_value: str = Field( 106 ..., 107 description="Value of the cursor defining the next page to fetch.", 108 examples=[ 109 "{{ headers.link.next.cursor }}", 110 "{{ last_record['key'] }}", 111 "{{ response['nextPage'] }}", 112 ], 113 title="Cursor Value", 114 ) 115 page_size: Optional[int] = Field( 116 None, 117 description="The number of records to include in each pages.", 118 examples=[100], 119 title="Page Size", 120 ) 121 stop_condition: Optional[str] = Field( 122 None, 123 description="Template string evaluating when to stop paginating.", 124 examples=[ 125 "{{ response.data.has_more is false }}", 126 "{{ 'next' not in headers['link'] }}", 127 ], 128 title="Stop Condition", 129 ) 130 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 131 132 133class CustomAuthenticator(BaseModel): 134 class Config: 135 extra = Extra.allow 136 137 type: Literal["CustomAuthenticator"] 138 class_name: str = Field( 139 ..., 140 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 141 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 142 title="Class Name", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomBackoffStrategy(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomBackoffStrategy"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.MyCustomBackoffStrategy"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomErrorHandler(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomErrorHandler"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomErrorHandler"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomPaginationStrategy(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomPaginationStrategy"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomPaginationStrategy"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomRecordExtractor(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomRecordExtractor"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomRecordExtractor"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordFilter(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordFilter"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomCustomRecordFilter"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRequester(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRequester"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomRecordExtractor"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRetriever(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRetriever"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRetriever"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomPartitionRouter(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomPartitionRouter"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomPartitionRouter"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomSchemaLoader(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomSchemaLoader"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomSchemaLoader"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaNormalization(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaNormalization"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 281 examples=[ 282 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 283 ], 284 title="Class Name", 285 ) 286 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 287 288 289class CustomStateMigration(BaseModel): 290 class Config: 291 extra = Extra.allow 292 293 type: Literal["CustomStateMigration"] 294 class_name: str = Field( 295 ..., 296 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 297 examples=["source_railz.components.MyCustomStateMigration"], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomTransformation(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomTransformation"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomTransformation"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class LegacyToPerPartitionStateMigration(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 322 323 324class Clamping(BaseModel): 325 target: str = Field( 326 ..., 327 description="The period of time that datetime windows will be clamped by", 328 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 329 title="Target", 330 ) 331 target_details: Optional[Dict[str, Any]] = None 332 333 334class Algorithm(Enum): 335 HS256 = "HS256" 336 HS384 = "HS384" 337 HS512 = "HS512" 338 ES256 = "ES256" 339 ES256K = "ES256K" 340 ES384 = "ES384" 341 ES512 = "ES512" 342 RS256 = "RS256" 343 RS384 = "RS384" 344 RS512 = "RS512" 345 PS256 = "PS256" 346 PS384 = "PS384" 347 PS512 = "PS512" 348 EdDSA = "EdDSA" 349 350 351class JwtHeaders(BaseModel): 352 class Config: 353 extra = Extra.forbid 354 355 kid: Optional[str] = Field( 356 None, 357 description="Private key ID for user account.", 358 examples=["{{ config['kid'] }}"], 359 title="Key Identifier", 360 ) 361 typ: Optional[str] = Field( 362 "JWT", 363 description="The media type of the complete JWT.", 364 examples=["JWT"], 365 title="Type", 366 ) 367 cty: Optional[str] = Field( 368 None, 369 description="Content type of JWT header.", 370 examples=["JWT"], 371 title="Content Type", 372 ) 373 374 375class JwtPayload(BaseModel): 376 class Config: 377 extra = Extra.forbid 378 379 iss: Optional[str] = Field( 380 None, 381 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 382 examples=["{{ config['iss'] }}"], 383 title="Issuer", 384 ) 385 sub: Optional[str] = Field( 386 None, 387 description="The subject of the JWT. Commonly defined by the API.", 388 title="Subject", 389 ) 390 aud: Optional[str] = Field( 391 None, 392 description="The recipient that the JWT is intended for. Commonly defined by the API.", 393 examples=["appstoreconnect-v1"], 394 title="Audience", 395 ) 396 397 398class RefreshTokenUpdater(BaseModel): 399 refresh_token_name: Optional[str] = Field( 400 "refresh_token", 401 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 402 examples=["refresh_token"], 403 title="Refresh Token Property Name", 404 ) 405 access_token_config_path: Optional[List[str]] = Field( 406 ["credentials", "access_token"], 407 description="Config path to the access token. Make sure the field actually exists in the config.", 408 examples=[["credentials", "access_token"], ["access_token"]], 409 title="Config Path To Access Token", 410 ) 411 refresh_token_config_path: Optional[List[str]] = Field( 412 ["credentials", "refresh_token"], 413 description="Config path to the access token. Make sure the field actually exists in the config.", 414 examples=[["credentials", "refresh_token"], ["refresh_token"]], 415 title="Config Path To Refresh Token", 416 ) 417 token_expiry_date_config_path: Optional[List[str]] = Field( 418 ["credentials", "token_expiry_date"], 419 description="Config path to the expiry date. Make sure actually exists in the config.", 420 examples=[["credentials", "token_expiry_date"]], 421 title="Config Path To Expiry Date", 422 ) 423 refresh_token_error_status_codes: Optional[List[int]] = Field( 424 [], 425 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 426 examples=[[400, 500]], 427 title="Refresh Token Error Status Codes", 428 ) 429 refresh_token_error_key: Optional[str] = Field( 430 "", 431 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 432 examples=["error"], 433 title="Refresh Token Error Key", 434 ) 435 refresh_token_error_values: Optional[List[str]] = Field( 436 [], 437 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 438 examples=[["invalid_grant", "invalid_permissions"]], 439 title="Refresh Token Error Values", 440 ) 441 442 443class Rate(BaseModel): 444 class Config: 445 extra = Extra.allow 446 447 limit: Union[int, str] = Field( 448 ..., 449 description="The maximum number of calls allowed within the interval.", 450 title="Limit", 451 ) 452 interval: str = Field( 453 ..., 454 description="The time interval for the rate limit.", 455 examples=["PT1H", "P1D"], 456 title="Interval", 457 ) 458 459 460class HttpRequestRegexMatcher(BaseModel): 461 class Config: 462 extra = Extra.allow 463 464 method: Optional[str] = Field( 465 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 466 ) 467 url_base: Optional[str] = Field( 468 None, 469 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 470 title="URL Base", 471 ) 472 url_path_pattern: Optional[str] = Field( 473 None, 474 description="A regular expression pattern to match the URL path.", 475 title="URL Path Pattern", 476 ) 477 params: Optional[Dict[str, Any]] = Field( 478 None, description="The query parameters to match.", title="Parameters" 479 ) 480 headers: Optional[Dict[str, Any]] = Field( 481 None, description="The headers to match.", title="Headers" 482 ) 483 484 485class DpathExtractor(BaseModel): 486 type: Literal["DpathExtractor"] 487 field_path: List[str] = Field( 488 ..., 489 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 490 examples=[ 491 ["data"], 492 ["data", "records"], 493 ["data", "{{ parameters.name }}"], 494 ["data", "*", "record"], 495 ], 496 title="Field Path", 497 ) 498 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 499 500 501class ResponseToFileExtractor(BaseModel): 502 type: Literal["ResponseToFileExtractor"] 503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 504 505 506class ExponentialBackoffStrategy(BaseModel): 507 type: Literal["ExponentialBackoffStrategy"] 508 factor: Optional[Union[float, str]] = Field( 509 5, 510 description="Multiplicative constant applied on each retry.", 511 examples=[5, 5.5, "10"], 512 title="Factor", 513 ) 514 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 515 516 517class GroupByKeyMergeStrategy(BaseModel): 518 type: Literal["GroupByKeyMergeStrategy"] 519 key: Union[str, List[str]] = Field( 520 ..., 521 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 522 examples=["id", ["parent_id", "end_date"]], 523 title="Key", 524 ) 525 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 526 527 528class SessionTokenRequestBearerAuthenticator(BaseModel): 529 type: Literal["Bearer"] 530 531 532class HttpMethod(Enum): 533 GET = "GET" 534 POST = "POST" 535 536 537class Action(Enum): 538 SUCCESS = "SUCCESS" 539 FAIL = "FAIL" 540 RETRY = "RETRY" 541 IGNORE = "IGNORE" 542 RATE_LIMITED = "RATE_LIMITED" 543 544 545class FailureType(Enum): 546 system_error = "system_error" 547 config_error = "config_error" 548 transient_error = "transient_error" 549 550 551class HttpResponseFilter(BaseModel): 552 type: Literal["HttpResponseFilter"] 553 action: Optional[Action] = Field( 554 None, 555 description="Action to execute if a response matches the filter.", 556 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 557 title="Action", 558 ) 559 failure_type: Optional[FailureType] = Field( 560 None, 561 description="Failure type of traced exception if a response matches the filter.", 562 examples=["system_error", "config_error", "transient_error"], 563 title="Failure Type", 564 ) 565 error_message: Optional[str] = Field( 566 None, 567 description="Error Message to display if the response matches the filter.", 568 title="Error Message", 569 ) 570 error_message_contains: Optional[str] = Field( 571 None, 572 description="Match the response if its error message contains the substring.", 573 example=["This API operation is not enabled for this site"], 574 title="Error Message Substring", 575 ) 576 http_codes: Optional[List[int]] = Field( 577 None, 578 description="Match the response if its HTTP code is included in this list.", 579 examples=[[420, 429], [500]], 580 title="HTTP Codes", 581 unique_items=True, 582 ) 583 predicate: Optional[str] = Field( 584 None, 585 description="Match the response if the predicate evaluates to true.", 586 examples=[ 587 "{{ 'Too much requests' in response }}", 588 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 589 ], 590 title="Predicate", 591 ) 592 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 593 594 595class ComplexFieldType(BaseModel): 596 field_type: str 597 items: Optional[Union[str, ComplexFieldType]] = None 598 599 600class TypesMap(BaseModel): 601 target_type: Union[str, List[str], ComplexFieldType] 602 current_type: Union[str, List[str]] 603 condition: Optional[str] = None 604 605 606class SchemaTypeIdentifier(BaseModel): 607 type: Optional[Literal["SchemaTypeIdentifier"]] = None 608 schema_pointer: Optional[List[str]] = Field( 609 [], 610 description="List of nested fields defining the schema field path to extract. Defaults to [].", 611 title="Schema Path", 612 ) 613 key_pointer: List[str] = Field( 614 ..., 615 description="List of potentially nested fields describing the full path of the field key to extract.", 616 title="Key Path", 617 ) 618 type_pointer: Optional[List[str]] = Field( 619 None, 620 description="List of potentially nested fields describing the full path of the field type to extract.", 621 title="Type Path", 622 ) 623 types_mapping: Optional[List[TypesMap]] = None 624 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 625 626 627class InlineSchemaLoader(BaseModel): 628 type: Literal["InlineSchemaLoader"] 629 schema_: Optional[Dict[str, Any]] = Field( 630 None, 631 alias="schema", 632 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 633 title="Schema", 634 ) 635 636 637class JsonFileSchemaLoader(BaseModel): 638 type: Literal["JsonFileSchemaLoader"] 639 file_path: Optional[str] = Field( 640 None, 641 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 642 example=["./schemas/users.json"], 643 title="File Path", 644 ) 645 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 646 647 648class JsonDecoder(BaseModel): 649 type: Literal["JsonDecoder"] 650 651 652class JsonlDecoder(BaseModel): 653 type: Literal["JsonlDecoder"] 654 655 656class KeysToLower(BaseModel): 657 type: Literal["KeysToLower"] 658 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 659 660 661class KeysToSnakeCase(BaseModel): 662 type: Literal["KeysToSnakeCase"] 663 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 664 665 666class FlattenFields(BaseModel): 667 type: Literal["FlattenFields"] 668 flatten_lists: Optional[bool] = Field( 669 True, 670 description="Whether to flatten lists or leave it as is. Default is True.", 671 title="Flatten Lists", 672 ) 673 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 674 675 676class KeyTransformation(BaseModel): 677 type: Literal["KeyTransformation"] 678 prefix: Optional[str] = Field( 679 None, 680 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 681 examples=["flattened_"], 682 title="Key Prefix", 683 ) 684 suffix: Optional[str] = Field( 685 None, 686 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 687 examples=["_flattened"], 688 title="Key Suffix", 689 ) 690 691 692class DpathFlattenFields(BaseModel): 693 type: Literal["DpathFlattenFields"] 694 field_path: List[str] = Field( 695 ..., 696 description="A path to field that needs to be flattened.", 697 examples=[["data"], ["data", "*", "field"]], 698 title="Field Path", 699 ) 700 delete_origin_value: Optional[bool] = Field( 701 None, 702 description="Whether to delete the origin value or keep it. Default is False.", 703 title="Delete Origin Value", 704 ) 705 replace_record: Optional[bool] = Field( 706 None, 707 description="Whether to replace the origin record or not. Default is False.", 708 title="Replace Origin Record", 709 ) 710 key_transformation: Optional[KeyTransformation] = Field( 711 None, 712 description="Transformation for object keys. If not provided, original key will be used.", 713 title="Key transformation", 714 ) 715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 716 717 718class KeysReplace(BaseModel): 719 type: Literal["KeysReplace"] 720 old: str = Field( 721 ..., 722 description="Old value to replace.", 723 examples=[ 724 " ", 725 "{{ record.id }}", 726 "{{ config['id'] }}", 727 "{{ stream_slice['id'] }}", 728 ], 729 title="Old value", 730 ) 731 new: str = Field( 732 ..., 733 description="New value to set.", 734 examples=[ 735 "_", 736 "{{ record.id }}", 737 "{{ config['id'] }}", 738 "{{ stream_slice['id'] }}", 739 ], 740 title="New value", 741 ) 742 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 743 744 745class IterableDecoder(BaseModel): 746 type: Literal["IterableDecoder"] 747 748 749class XmlDecoder(BaseModel): 750 type: Literal["XmlDecoder"] 751 752 753class CustomDecoder(BaseModel): 754 class Config: 755 extra = Extra.allow 756 757 type: Literal["CustomDecoder"] 758 class_name: str = Field( 759 ..., 760 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 761 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 762 title="Class Name", 763 ) 764 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 765 766 767class MinMaxDatetime(BaseModel): 768 type: Literal["MinMaxDatetime"] 769 datetime: str = Field( 770 ..., 771 description="Datetime value.", 772 examples=[ 773 "2021-01-01", 774 "2021-01-01T00:00:00Z", 775 "{{ config['start_time'] }}", 776 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 777 ], 778 title="Datetime", 779 ) 780 datetime_format: Optional[str] = Field( 781 "", 782 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 783 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 784 title="Datetime Format", 785 ) 786 max_datetime: Optional[str] = Field( 787 None, 788 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 789 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 790 title="Max Datetime", 791 ) 792 min_datetime: Optional[str] = Field( 793 None, 794 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 795 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 796 title="Min Datetime", 797 ) 798 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 799 800 801class NoAuth(BaseModel): 802 type: Literal["NoAuth"] 803 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 804 805 806class NoPagination(BaseModel): 807 type: Literal["NoPagination"] 808 809 810class State(BaseModel): 811 class Config: 812 extra = Extra.allow 813 814 min: int 815 max: int 816 817 818class OauthConnectorInputSpecification(BaseModel): 819 class Config: 820 extra = Extra.allow 821 822 consent_url: str = Field( 823 ..., 824 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 825 examples=[ 826 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 827 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 828 ], 829 title="Consent URL", 830 ) 831 scope: Optional[str] = Field( 832 None, 833 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 834 examples=["user:read user:read_orders workspaces:read"], 835 title="Scopes", 836 ) 837 access_token_url: str = Field( 838 ..., 839 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 840 examples=[ 841 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 842 ], 843 title="Access Token URL", 844 ) 845 access_token_headers: Optional[Dict[str, Any]] = Field( 846 None, 847 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 848 examples=[ 849 { 850 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 851 } 852 ], 853 title="Access Token Headers", 854 ) 855 access_token_params: Optional[Dict[str, Any]] = Field( 856 None, 857 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 858 examples=[ 859 { 860 "{{ auth_code_key }}": "{{ auth_code_value }}", 861 "{{ client_id_key }}": "{{ client_id_value }}", 862 "{{ client_secret_key }}": "{{ client_secret_value }}", 863 } 864 ], 865 title="Access Token Query Params (Json Encoded)", 866 ) 867 extract_output: Optional[List[str]] = Field( 868 None, 869 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 870 examples=[["access_token", "refresh_token", "other_field"]], 871 title="Extract Output", 872 ) 873 state: Optional[State] = Field( 874 None, 875 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 876 examples=[{"min": 7, "max": 128}], 877 title="Configurable State Query Param", 878 ) 879 client_id_key: Optional[str] = Field( 880 None, 881 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 882 examples=["my_custom_client_id_key_name"], 883 title="Client ID Key Override", 884 ) 885 client_secret_key: Optional[str] = Field( 886 None, 887 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 888 examples=["my_custom_client_secret_key_name"], 889 title="Client Secret Key Override", 890 ) 891 scope_key: Optional[str] = Field( 892 None, 893 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 894 examples=["my_custom_scope_key_key_name"], 895 title="Scopes Key Override", 896 ) 897 state_key: Optional[str] = Field( 898 None, 899 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 900 examples=["my_custom_state_key_key_name"], 901 title="State Key Override", 902 ) 903 auth_code_key: Optional[str] = Field( 904 None, 905 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 906 examples=["my_custom_auth_code_key_name"], 907 title="Auth Code Key Override", 908 ) 909 redirect_uri_key: Optional[str] = Field( 910 None, 911 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 912 examples=["my_custom_redirect_uri_key_name"], 913 title="Redirect URI Key Override", 914 ) 915 916 917class OAuthConfigSpecification(BaseModel): 918 class Config: 919 extra = Extra.allow 920 921 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 922 None, 923 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 924 examples=[ 925 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 926 { 927 "app_id": { 928 "type": "string", 929 "path_in_connector_config": ["info", "app_id"], 930 } 931 }, 932 ], 933 title="OAuth user input", 934 ) 935 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 936 None, 937 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 938 title="DeclarativeOAuth Connector Specification", 939 ) 940 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 941 None, 942 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 943 examples=[ 944 { 945 "refresh_token": { 946 "type": "string,", 947 "path_in_connector_config": ["credentials", "refresh_token"], 948 } 949 } 950 ], 951 title="OAuth output specification", 952 ) 953 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 954 None, 955 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 956 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 957 title="OAuth input specification", 958 ) 959 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 960 None, 961 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 962 examples=[ 963 { 964 "client_id": { 965 "type": "string,", 966 "path_in_connector_config": ["credentials", "client_id"], 967 }, 968 "client_secret": { 969 "type": "string,", 970 "path_in_connector_config": ["credentials", "client_secret"], 971 }, 972 } 973 ], 974 title="OAuth server output specification", 975 ) 976 977 978class OffsetIncrement(BaseModel): 979 type: Literal["OffsetIncrement"] 980 page_size: Optional[Union[int, str]] = Field( 981 None, 982 description="The number of records to include in each pages.", 983 examples=[100, "{{ config['page_size'] }}"], 984 title="Limit", 985 ) 986 inject_on_first_request: Optional[bool] = Field( 987 False, 988 description="Using the `offset` with value `0` during the first request", 989 title="Inject Offset on First Request", 990 ) 991 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 992 993 994class PageIncrement(BaseModel): 995 type: Literal["PageIncrement"] 996 page_size: Optional[Union[int, str]] = Field( 997 None, 998 description="The number of records to include in each pages.", 999 examples=[100, "100", "{{ config['page_size'] }}"], 1000 title="Page Size", 1001 ) 1002 start_from_page: Optional[int] = Field( 1003 0, 1004 description="Index of the first page to request.", 1005 examples=[0, 1], 1006 title="Start From Page", 1007 ) 1008 inject_on_first_request: Optional[bool] = Field( 1009 False, 1010 description="Using the `page number` with value defined by `start_from_page` during the first request", 1011 title="Inject Page Number on First Request", 1012 ) 1013 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1014 1015 1016class PrimaryKey(BaseModel): 1017 __root__: Union[str, List[str], List[List[str]]] = Field( 1018 ..., 1019 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1020 examples=["id", ["code", "type"]], 1021 title="Primary Key", 1022 ) 1023 1024 1025class PropertyLimitType(Enum): 1026 characters = "characters" 1027 property_count = "property_count" 1028 1029 1030class PropertyChunking(BaseModel): 1031 type: Literal["PropertyChunking"] 1032 property_limit_type: PropertyLimitType = Field( 1033 ..., 1034 description="The type used to determine the maximum number of properties per chunk", 1035 title="Property Limit Type", 1036 ) 1037 property_limit: Optional[int] = Field( 1038 None, 1039 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1040 title="Property Limit", 1041 ) 1042 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1043 None, 1044 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1045 title="Record Merge Strategy", 1046 ) 1047 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1048 1049 1050class RecordFilter(BaseModel): 1051 type: Literal["RecordFilter"] 1052 condition: Optional[str] = Field( 1053 "", 1054 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1055 examples=[ 1056 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1057 "{{ record.status in ['active', 'expired'] }}", 1058 ], 1059 title="Condition", 1060 ) 1061 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1062 1063 1064class SchemaNormalization(Enum): 1065 Default = "Default" 1066 None_ = "None" 1067 1068 1069class RemoveFields(BaseModel): 1070 type: Literal["RemoveFields"] 1071 condition: Optional[str] = Field( 1072 "", 1073 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1074 examples=[ 1075 "{{ property|string == '' }}", 1076 "{{ property is integer }}", 1077 "{{ property|length > 5 }}", 1078 "{{ property == 'some_string_to_match' }}", 1079 ], 1080 ) 1081 field_pointers: List[List[str]] = Field( 1082 ..., 1083 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1084 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1085 title="Field Paths", 1086 ) 1087 1088 1089class RequestPath(BaseModel): 1090 type: Literal["RequestPath"] 1091 1092 1093class InjectInto(Enum): 1094 request_parameter = "request_parameter" 1095 header = "header" 1096 body_data = "body_data" 1097 body_json = "body_json" 1098 1099 1100class RequestOption(BaseModel): 1101 type: Literal["RequestOption"] 1102 inject_into: InjectInto = Field( 1103 ..., 1104 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1105 examples=["request_parameter", "header", "body_data", "body_json"], 1106 title="Inject Into", 1107 ) 1108 field_name: Optional[str] = Field( 1109 None, 1110 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1111 examples=["segment_id"], 1112 title="Field Name", 1113 ) 1114 field_path: Optional[List[str]] = Field( 1115 None, 1116 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1117 examples=[["data", "viewer", "id"]], 1118 title="Field Path", 1119 ) 1120 1121 1122class Schemas(BaseModel): 1123 pass 1124 1125 class Config: 1126 extra = Extra.allow 1127 1128 1129class LegacySessionTokenAuthenticator(BaseModel): 1130 type: Literal["LegacySessionTokenAuthenticator"] 1131 header: str = Field( 1132 ..., 1133 description="The name of the session token header that will be injected in the request", 1134 examples=["X-Session"], 1135 title="Session Request Header", 1136 ) 1137 login_url: str = Field( 1138 ..., 1139 description="Path of the login URL (do not include the base URL)", 1140 examples=["session"], 1141 title="Login Path", 1142 ) 1143 session_token: Optional[str] = Field( 1144 None, 1145 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1146 example=["{{ config['session_token'] }}"], 1147 title="Session Token", 1148 ) 1149 session_token_response_key: str = Field( 1150 ..., 1151 description="Name of the key of the session token to be extracted from the response", 1152 examples=["id"], 1153 title="Response Token Response Key", 1154 ) 1155 username: Optional[str] = Field( 1156 None, 1157 description="Username used to authenticate and obtain a session token", 1158 examples=[" {{ config['username'] }}"], 1159 title="Username", 1160 ) 1161 password: Optional[str] = Field( 1162 "", 1163 description="Password used to authenticate and obtain a session token", 1164 examples=["{{ config['password'] }}", ""], 1165 title="Password", 1166 ) 1167 validate_session_url: str = Field( 1168 ..., 1169 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1170 examples=["user/current"], 1171 title="Validate Session Path", 1172 ) 1173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1174 1175 1176class CsvDecoder(BaseModel): 1177 type: Literal["CsvDecoder"] 1178 encoding: Optional[str] = "utf-8" 1179 delimiter: Optional[str] = "," 1180 set_values_to_none: Optional[List[str]] = None 1181 1182 1183class AsyncJobStatusMap(BaseModel): 1184 type: Optional[Literal["AsyncJobStatusMap"]] = None 1185 running: List[str] 1186 completed: List[str] 1187 failed: List[str] 1188 timeout: List[str] 1189 1190 1191class ValueType(Enum): 1192 string = "string" 1193 number = "number" 1194 integer = "integer" 1195 boolean = "boolean" 1196 1197 1198class WaitTimeFromHeader(BaseModel): 1199 type: Literal["WaitTimeFromHeader"] 1200 header: str = Field( 1201 ..., 1202 description="The name of the response header defining how long to wait before retrying.", 1203 examples=["Retry-After"], 1204 title="Response Header Name", 1205 ) 1206 regex: Optional[str] = Field( 1207 None, 1208 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1209 examples=["([-+]?\\d+)"], 1210 title="Extraction Regex", 1211 ) 1212 max_waiting_time_in_seconds: Optional[float] = Field( 1213 None, 1214 description="Given the value extracted from the header is greater than this value, stop the stream.", 1215 examples=[3600], 1216 title="Max Waiting Time in Seconds", 1217 ) 1218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1219 1220 1221class WaitUntilTimeFromHeader(BaseModel): 1222 type: Literal["WaitUntilTimeFromHeader"] 1223 header: str = Field( 1224 ..., 1225 description="The name of the response header defining how long to wait before retrying.", 1226 examples=["wait_time"], 1227 title="Response Header", 1228 ) 1229 min_wait: Optional[Union[float, str]] = Field( 1230 None, 1231 description="Minimum time to wait before retrying.", 1232 examples=[10, "60"], 1233 title="Minimum Wait Time", 1234 ) 1235 regex: Optional[str] = Field( 1236 None, 1237 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1238 examples=["([-+]?\\d+)"], 1239 title="Extraction Regex", 1240 ) 1241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1242 1243 1244class ComponentMappingDefinition(BaseModel): 1245 type: Literal["ComponentMappingDefinition"] 1246 field_path: List[str] = Field( 1247 ..., 1248 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1249 examples=[ 1250 ["name"], 1251 ["retriever", "requester", "url"], 1252 ["retriever", "requester", "{{ components_values.field }}"], 1253 ["*", "**", "name"], 1254 ], 1255 title="Field Path", 1256 ) 1257 value: str = Field( 1258 ..., 1259 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1260 examples=[ 1261 "{{ components_values['updates'] }}", 1262 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1263 "{{ config['segment_id'] }}", 1264 "{{ stream_slice['parent_id'] }}", 1265 "{{ stream_slice['extra_fields']['name'] }}", 1266 ], 1267 title="Value", 1268 ) 1269 value_type: Optional[ValueType] = Field( 1270 None, 1271 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1272 title="Value Type", 1273 ) 1274 create_or_update: Optional[bool] = Field( 1275 False, 1276 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1277 title="Create or Update", 1278 ) 1279 condition: Optional[str] = Field( 1280 None, 1281 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1282 examples=[ 1283 "{{ components_values.get('cursor_field', None) }}", 1284 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1285 ], 1286 title="Condition", 1287 ) 1288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1289 1290 1291class StreamConfig(BaseModel): 1292 type: Literal["StreamConfig"] 1293 configs_pointer: List[str] = Field( 1294 ..., 1295 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1296 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1297 title="Configs Pointer", 1298 ) 1299 default_values: Optional[List[Dict[str, Any]]] = Field( 1300 None, 1301 description="A list of default values, each matching the structure expected from the parsed component value.", 1302 title="Default Values", 1303 ) 1304 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1305 1306 1307class ConfigComponentsResolver(BaseModel): 1308 type: Literal["ConfigComponentsResolver"] 1309 stream_config: Union[List[StreamConfig], StreamConfig] 1310 components_mapping: List[ComponentMappingDefinition] 1311 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1312 1313 1314class StreamParametersDefinition(BaseModel): 1315 type: Literal["StreamParametersDefinition"] 1316 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1317 ..., 1318 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1319 examples=[ 1320 [ 1321 { 1322 "name": "test stream", 1323 "$parameters": {"entity": "test entity"}, 1324 "primary_key": "test key", 1325 } 1326 ] 1327 ], 1328 title="Stream Parameters", 1329 ) 1330 1331 1332class ParametrizedComponentsResolver(BaseModel): 1333 type: Literal["ParametrizedComponentsResolver"] 1334 stream_parameters: StreamParametersDefinition 1335 components_mapping: List[ComponentMappingDefinition] 1336 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1337 1338 1339class RequestBodyPlainText(BaseModel): 1340 type: Literal["RequestBodyPlainText"] 1341 value: str 1342 1343 1344class RequestBodyUrlEncodedForm(BaseModel): 1345 type: Literal["RequestBodyUrlEncodedForm"] 1346 value: Dict[str, str] 1347 1348 1349class RequestBodyJsonObject(BaseModel): 1350 type: Literal["RequestBodyJsonObject"] 1351 value: Dict[str, Any] 1352 1353 1354class RequestBodyGraphQlQuery(BaseModel): 1355 class Config: 1356 extra = Extra.allow 1357 1358 query: str = Field(..., description="The GraphQL query to be executed") 1359 1360 1361class ValidateAdheresToSchema(BaseModel): 1362 type: Literal["ValidateAdheresToSchema"] 1363 base_schema: Union[str, Dict[str, Any]] = Field( 1364 ..., 1365 description="The base JSON schema against which the user-provided schema will be validated.", 1366 examples=[ 1367 "{{ config['report_validation_schema'] }}", 1368 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1369 { 1370 "$schema": "http://json-schema.org/draft-07/schema#", 1371 "title": "Person", 1372 "type": "object", 1373 "properties": { 1374 "name": {"type": "string", "description": "The person's name"}, 1375 "age": { 1376 "type": "integer", 1377 "minimum": 0, 1378 "description": "The person's age", 1379 }, 1380 }, 1381 "required": ["name", "age"], 1382 }, 1383 ], 1384 title="Base JSON Schema", 1385 ) 1386 1387 1388class CustomValidationStrategy(BaseModel): 1389 class Config: 1390 extra = Extra.allow 1391 1392 type: Literal["CustomValidationStrategy"] 1393 class_name: str = Field( 1394 ..., 1395 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1396 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1397 title="Class Name", 1398 ) 1399 1400 1401class ConfigRemapField(BaseModel): 1402 type: Literal["ConfigRemapField"] 1403 map: Union[Dict[str, Any], str] = Field( 1404 ..., 1405 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1406 examples=[ 1407 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1408 "{{ config['status_mapping'] }}", 1409 ], 1410 title="Value Mapping", 1411 ) 1412 field_path: List[str] = Field( 1413 ..., 1414 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1415 examples=[ 1416 ["status"], 1417 ["data", "status"], 1418 ["data", "{{ config.name }}", "status"], 1419 ["data", "*", "status"], 1420 ], 1421 title="Field Path", 1422 ) 1423 1424 1425class ConfigRemoveFields(BaseModel): 1426 type: Literal["ConfigRemoveFields"] 1427 field_pointers: List[List[str]] = Field( 1428 ..., 1429 description="A list of field pointers to be removed from the config.", 1430 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1431 title="Field Pointers", 1432 ) 1433 condition: Optional[str] = Field( 1434 "", 1435 description="Fields will be removed if expression is evaluated to True.", 1436 examples=[ 1437 "{{ config['environemnt'] == 'sandbox' }}", 1438 "{{ property is integer }}", 1439 "{{ property|length > 5 }}", 1440 "{{ property == 'some_string_to_match' }}", 1441 ], 1442 ) 1443 1444 1445class CustomConfigTransformation(BaseModel): 1446 type: Literal["CustomConfigTransformation"] 1447 class_name: str = Field( 1448 ..., 1449 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1450 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1451 ) 1452 parameters: Optional[Dict[str, Any]] = Field( 1453 None, 1454 alias="$parameters", 1455 description="Additional parameters to be passed to the custom config transformation.", 1456 ) 1457 1458 1459class AddedFieldDefinition(BaseModel): 1460 type: Literal["AddedFieldDefinition"] 1461 path: List[str] = Field( 1462 ..., 1463 description="List of strings defining the path where to add the value on the record.", 1464 examples=[["segment_id"], ["metadata", "segment_id"]], 1465 title="Path", 1466 ) 1467 value: str = Field( 1468 ..., 1469 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1470 examples=[ 1471 "{{ record['updates'] }}", 1472 "{{ record['MetaData']['LastUpdatedTime'] }}", 1473 "{{ stream_partition['segment_id'] }}", 1474 ], 1475 title="Value", 1476 ) 1477 value_type: Optional[ValueType] = Field( 1478 None, 1479 description="Type of the value. If not specified, the type will be inferred from the value.", 1480 title="Value Type", 1481 ) 1482 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1483 1484 1485class AddFields(BaseModel): 1486 type: Literal["AddFields"] 1487 fields: List[AddedFieldDefinition] = Field( 1488 ..., 1489 description="List of transformations (path and corresponding value) that will be added to the record.", 1490 title="Fields", 1491 ) 1492 condition: Optional[str] = Field( 1493 "", 1494 description="Fields will be added if expression is evaluated to True.", 1495 examples=[ 1496 "{{ property|string == '' }}", 1497 "{{ property is integer }}", 1498 "{{ property|length > 5 }}", 1499 "{{ property == 'some_string_to_match' }}", 1500 ], 1501 ) 1502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1503 1504 1505class ApiKeyAuthenticator(BaseModel): 1506 type: Literal["ApiKeyAuthenticator"] 1507 api_token: Optional[str] = Field( 1508 None, 1509 description="The API key to inject in the request. Fill it in the user inputs.", 1510 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1511 title="API Key", 1512 ) 1513 header: Optional[str] = Field( 1514 None, 1515 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1516 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1517 title="Header Name", 1518 ) 1519 inject_into: Optional[RequestOption] = Field( 1520 None, 1521 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1522 examples=[ 1523 {"inject_into": "header", "field_name": "Authorization"}, 1524 {"inject_into": "request_parameter", "field_name": "authKey"}, 1525 ], 1526 title="Inject API Key Into Outgoing HTTP Request", 1527 ) 1528 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1529 1530 1531class AuthFlow(BaseModel): 1532 auth_flow_type: Optional[AuthFlowType] = Field( 1533 None, description="The type of auth to use", title="Auth flow type" 1534 ) 1535 predicate_key: Optional[List[str]] = Field( 1536 None, 1537 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1538 examples=[["credentials", "auth_type"]], 1539 title="Predicate key", 1540 ) 1541 predicate_value: Optional[str] = Field( 1542 None, 1543 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1544 examples=["Oauth"], 1545 title="Predicate value", 1546 ) 1547 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1548 1549 1550class CheckStream(BaseModel): 1551 type: Literal["CheckStream"] 1552 stream_names: Optional[List[str]] = Field( 1553 None, 1554 description="Names of the streams to try reading from when running a check operation.", 1555 examples=[["users"], ["users", "contacts"]], 1556 title="Stream Names", 1557 ) 1558 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1559 1560 1561class IncrementingCountCursor(BaseModel): 1562 type: Literal["IncrementingCountCursor"] 1563 cursor_field: str = Field( 1564 ..., 1565 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1566 examples=["created_at", "{{ config['record_cursor'] }}"], 1567 title="Cursor Field", 1568 ) 1569 start_value: Optional[Union[str, int]] = Field( 1570 None, 1571 description="The value that determines the earliest record that should be synced.", 1572 examples=[0, "{{ config['start_value'] }}"], 1573 title="Start Value", 1574 ) 1575 start_value_option: Optional[RequestOption] = Field( 1576 None, 1577 description="Optionally configures how the start value will be sent in requests to the source API.", 1578 title="Inject Start Value Into Outgoing HTTP Request", 1579 ) 1580 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1581 1582 1583class DatetimeBasedCursor(BaseModel): 1584 type: Literal["DatetimeBasedCursor"] 1585 clamping: Optional[Clamping] = Field( 1586 None, 1587 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1588 title="Date Range Clamping", 1589 ) 1590 cursor_field: str = Field( 1591 ..., 1592 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1593 examples=["created_at", "{{ config['record_cursor'] }}"], 1594 title="Cursor Field", 1595 ) 1596 cursor_datetime_formats: Optional[List[str]] = Field( 1597 None, 1598 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1599 examples=[ 1600 "%Y-%m-%d", 1601 "%Y-%m-%d %H:%M:%S", 1602 "%Y-%m-%dT%H:%M:%S", 1603 "%Y-%m-%dT%H:%M:%SZ", 1604 "%Y-%m-%dT%H:%M:%S%z", 1605 "%Y-%m-%dT%H:%M:%S.%fZ", 1606 "%Y-%m-%dT%H:%M:%S.%f%z", 1607 "%Y-%m-%d %H:%M:%S.%f+00:00", 1608 "%s", 1609 "%ms", 1610 ], 1611 title="Cursor Datetime Formats", 1612 ) 1613 start_datetime: Union[MinMaxDatetime, str] = Field( 1614 ..., 1615 description="The datetime that determines the earliest record that should be synced.", 1616 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1617 title="Start Datetime", 1618 ) 1619 start_time_option: Optional[RequestOption] = Field( 1620 None, 1621 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1622 title="Inject Start Time Into Outgoing HTTP Request", 1623 ) 1624 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1625 None, 1626 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1627 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1628 title="End Datetime", 1629 ) 1630 end_time_option: Optional[RequestOption] = Field( 1631 None, 1632 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1633 title="Inject End Time Into Outgoing HTTP Request", 1634 ) 1635 datetime_format: str = Field( 1636 ..., 1637 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1638 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1639 title="Outgoing Datetime Format", 1640 ) 1641 cursor_granularity: Optional[str] = Field( 1642 None, 1643 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1644 examples=["PT1S"], 1645 title="Cursor Granularity", 1646 ) 1647 is_data_feed: Optional[bool] = Field( 1648 None, 1649 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1650 title="Data Feed API", 1651 ) 1652 is_client_side_incremental: Optional[bool] = Field( 1653 None, 1654 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1655 title="Client-side Incremental Filtering", 1656 ) 1657 is_compare_strictly: Optional[bool] = Field( 1658 False, 1659 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1660 title="Strict Start-End Time Comparison", 1661 ) 1662 global_substream_cursor: Optional[bool] = Field( 1663 False, 1664 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1665 title="Global Substream Cursor", 1666 ) 1667 lookback_window: Optional[str] = Field( 1668 None, 1669 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1670 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1671 title="Lookback Window", 1672 ) 1673 partition_field_end: Optional[str] = Field( 1674 None, 1675 description="Name of the partition start time field.", 1676 examples=["ending_time"], 1677 title="Partition Field End", 1678 ) 1679 partition_field_start: Optional[str] = Field( 1680 None, 1681 description="Name of the partition end time field.", 1682 examples=["starting_time"], 1683 title="Partition Field Start", 1684 ) 1685 step: Optional[str] = Field( 1686 None, 1687 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1688 examples=["P1W", "{{ config['step_increment'] }}"], 1689 title="Step", 1690 ) 1691 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1692 1693 1694class JwtAuthenticator(BaseModel): 1695 type: Literal["JwtAuthenticator"] 1696 secret_key: str = Field( 1697 ..., 1698 description="Secret used to sign the JSON web token.", 1699 examples=["{{ config['secret_key'] }}"], 1700 title="Secret Key", 1701 ) 1702 base64_encode_secret_key: Optional[bool] = Field( 1703 False, 1704 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1705 title="Base64-encode Secret Key", 1706 ) 1707 algorithm: Algorithm = Field( 1708 ..., 1709 description="Algorithm used to sign the JSON web token.", 1710 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1711 title="Algorithm", 1712 ) 1713 token_duration: Optional[int] = Field( 1714 1200, 1715 description="The amount of time in seconds a JWT token can be valid after being issued.", 1716 examples=[1200, 3600], 1717 title="Token Duration", 1718 ) 1719 header_prefix: Optional[str] = Field( 1720 None, 1721 description="The prefix to be used within the Authentication header.", 1722 examples=["Bearer", "Basic"], 1723 title="Header Prefix", 1724 ) 1725 jwt_headers: Optional[JwtHeaders] = Field( 1726 None, 1727 description="JWT headers used when signing JSON web token.", 1728 title="JWT Headers", 1729 ) 1730 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1731 None, 1732 description="Additional headers to be included with the JWT headers object.", 1733 title="Additional JWT Headers", 1734 ) 1735 jwt_payload: Optional[JwtPayload] = Field( 1736 None, 1737 description="JWT Payload used when signing JSON web token.", 1738 title="JWT Payload", 1739 ) 1740 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1741 None, 1742 description="Additional properties to be added to the JWT payload.", 1743 title="Additional JWT Payload Properties", 1744 ) 1745 passphrase: Optional[str] = Field( 1746 None, 1747 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1748 examples=["{{ config['passphrase'] }}"], 1749 title="Passphrase", 1750 ) 1751 request_option: Optional[RequestOption] = Field( 1752 None, 1753 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1754 title="Request Option", 1755 ) 1756 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1757 1758 1759class OAuthAuthenticator(BaseModel): 1760 type: Literal["OAuthAuthenticator"] 1761 client_id_name: Optional[str] = Field( 1762 "client_id", 1763 description="The name of the property to use to refresh the `access_token`.", 1764 examples=["custom_app_id"], 1765 title="Client ID Property Name", 1766 ) 1767 client_id: Optional[str] = Field( 1768 None, 1769 description="The OAuth client ID. Fill it in the user inputs.", 1770 examples=[ 1771 "{{ config['client_id'] }}", 1772 "{{ config['credentials']['client_id }}", 1773 ], 1774 title="Client ID", 1775 ) 1776 client_secret_name: Optional[str] = Field( 1777 "client_secret", 1778 description="The name of the property to use to refresh the `access_token`.", 1779 examples=["custom_app_secret"], 1780 title="Client Secret Property Name", 1781 ) 1782 client_secret: Optional[str] = Field( 1783 None, 1784 description="The OAuth client secret. Fill it in the user inputs.", 1785 examples=[ 1786 "{{ config['client_secret'] }}", 1787 "{{ config['credentials']['client_secret }}", 1788 ], 1789 title="Client Secret", 1790 ) 1791 refresh_token_name: Optional[str] = Field( 1792 "refresh_token", 1793 description="The name of the property to use to refresh the `access_token`.", 1794 examples=["custom_app_refresh_value"], 1795 title="Refresh Token Property Name", 1796 ) 1797 refresh_token: Optional[str] = Field( 1798 None, 1799 description="Credential artifact used to get a new access token.", 1800 examples=[ 1801 "{{ config['refresh_token'] }}", 1802 "{{ config['credentials]['refresh_token'] }}", 1803 ], 1804 title="Refresh Token", 1805 ) 1806 token_refresh_endpoint: Optional[str] = Field( 1807 None, 1808 description="The full URL to call to obtain a new access token.", 1809 examples=["https://connect.squareup.com/oauth2/token"], 1810 title="Token Refresh Endpoint", 1811 ) 1812 access_token_name: Optional[str] = Field( 1813 "access_token", 1814 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1815 examples=["access_token"], 1816 title="Access Token Property Name", 1817 ) 1818 access_token_value: Optional[str] = Field( 1819 None, 1820 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1821 examples=["secret_access_token_value"], 1822 title="Access Token Value", 1823 ) 1824 expires_in_name: Optional[str] = Field( 1825 "expires_in", 1826 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1827 examples=["expires_in"], 1828 title="Token Expiry Property Name", 1829 ) 1830 grant_type_name: Optional[str] = Field( 1831 "grant_type", 1832 description="The name of the property to use to refresh the `access_token`.", 1833 examples=["custom_grant_type"], 1834 title="Grant Type Property Name", 1835 ) 1836 grant_type: Optional[str] = Field( 1837 "refresh_token", 1838 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1839 examples=["refresh_token", "client_credentials"], 1840 title="Grant Type", 1841 ) 1842 refresh_request_body: Optional[Dict[str, Any]] = Field( 1843 None, 1844 description="Body of the request sent to get a new access token.", 1845 examples=[ 1846 { 1847 "applicationId": "{{ config['application_id'] }}", 1848 "applicationSecret": "{{ config['application_secret'] }}", 1849 "token": "{{ config['token'] }}", 1850 } 1851 ], 1852 title="Refresh Request Body", 1853 ) 1854 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1855 None, 1856 description="Headers of the request sent to get a new access token.", 1857 examples=[ 1858 { 1859 "Authorization": "<AUTH_TOKEN>", 1860 "Content-Type": "application/x-www-form-urlencoded", 1861 } 1862 ], 1863 title="Refresh Request Headers", 1864 ) 1865 scopes: Optional[List[str]] = Field( 1866 None, 1867 description="List of scopes that should be granted to the access token.", 1868 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1869 title="Scopes", 1870 ) 1871 token_expiry_date: Optional[str] = Field( 1872 None, 1873 description="The access token expiry date.", 1874 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1875 title="Token Expiry Date", 1876 ) 1877 token_expiry_date_format: Optional[str] = Field( 1878 None, 1879 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1880 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1881 title="Token Expiry Date Format", 1882 ) 1883 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1884 None, 1885 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1886 title="Refresh Token Updater", 1887 ) 1888 profile_assertion: Optional[JwtAuthenticator] = Field( 1889 None, 1890 description="The authenticator being used to authenticate the client authenticator.", 1891 title="Profile Assertion", 1892 ) 1893 use_profile_assertion: Optional[bool] = Field( 1894 False, 1895 description="Enable using profile assertion as a flow for OAuth authorization.", 1896 title="Use Profile Assertion", 1897 ) 1898 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1899 1900 1901class FixedWindowCallRatePolicy(BaseModel): 1902 class Config: 1903 extra = Extra.allow 1904 1905 type: Literal["FixedWindowCallRatePolicy"] 1906 period: str = Field( 1907 ..., description="The time interval for the rate limit window.", title="Period" 1908 ) 1909 call_limit: int = Field( 1910 ..., 1911 description="The maximum number of calls allowed within the period.", 1912 title="Call Limit", 1913 ) 1914 matchers: List[HttpRequestRegexMatcher] = Field( 1915 ..., 1916 description="List of matchers that define which requests this policy applies to.", 1917 title="Matchers", 1918 ) 1919 1920 1921class MovingWindowCallRatePolicy(BaseModel): 1922 class Config: 1923 extra = Extra.allow 1924 1925 type: Literal["MovingWindowCallRatePolicy"] 1926 rates: List[Rate] = Field( 1927 ..., 1928 description="List of rates that define the call limits for different time intervals.", 1929 title="Rates", 1930 ) 1931 matchers: List[HttpRequestRegexMatcher] = Field( 1932 ..., 1933 description="List of matchers that define which requests this policy applies to.", 1934 title="Matchers", 1935 ) 1936 1937 1938class UnlimitedCallRatePolicy(BaseModel): 1939 class Config: 1940 extra = Extra.allow 1941 1942 type: Literal["UnlimitedCallRatePolicy"] 1943 matchers: List[HttpRequestRegexMatcher] = Field( 1944 ..., 1945 description="List of matchers that define which requests this policy applies to.", 1946 title="Matchers", 1947 ) 1948 1949 1950class DefaultErrorHandler(BaseModel): 1951 type: Literal["DefaultErrorHandler"] 1952 backoff_strategies: Optional[ 1953 List[ 1954 Union[ 1955 ConstantBackoffStrategy, 1956 ExponentialBackoffStrategy, 1957 WaitTimeFromHeader, 1958 WaitUntilTimeFromHeader, 1959 CustomBackoffStrategy, 1960 ] 1961 ] 1962 ] = Field( 1963 None, 1964 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1965 title="Backoff Strategies", 1966 ) 1967 max_retries: Optional[int] = Field( 1968 5, 1969 description="The maximum number of time to retry a retryable request before giving up and failing.", 1970 examples=[5, 0, 10], 1971 title="Max Retry Count", 1972 ) 1973 response_filters: Optional[List[HttpResponseFilter]] = Field( 1974 None, 1975 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1976 title="Response Filters", 1977 ) 1978 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1979 1980 1981class DefaultPaginator(BaseModel): 1982 type: Literal["DefaultPaginator"] 1983 pagination_strategy: Union[ 1984 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1985 ] = Field( 1986 ..., 1987 description="Strategy defining how records are paginated.", 1988 title="Pagination Strategy", 1989 ) 1990 page_size_option: Optional[RequestOption] = Field( 1991 None, title="Inject Page Size Into Outgoing HTTP Request" 1992 ) 1993 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1994 None, 1995 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 1996 title="Inject Page Token Into Outgoing HTTP Request", 1997 ) 1998 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1999 2000 2001class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2002 type: Literal["ApiKey"] 2003 inject_into: RequestOption = Field( 2004 ..., 2005 description="Configure how the API Key will be sent in requests to the source API.", 2006 examples=[ 2007 {"inject_into": "header", "field_name": "Authorization"}, 2008 {"inject_into": "request_parameter", "field_name": "authKey"}, 2009 ], 2010 title="Inject API Key Into Outgoing HTTP Request", 2011 ) 2012 2013 2014class ListPartitionRouter(BaseModel): 2015 type: Literal["ListPartitionRouter"] 2016 cursor_field: str = Field( 2017 ..., 2018 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2019 examples=["section", "{{ config['section_key'] }}"], 2020 title="Current Partition Value Identifier", 2021 ) 2022 values: Union[str, List[str]] = Field( 2023 ..., 2024 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2025 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2026 title="Partition Values", 2027 ) 2028 request_option: Optional[RequestOption] = Field( 2029 None, 2030 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2031 title="Inject Partition Value Into Outgoing HTTP Request", 2032 ) 2033 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2034 2035 2036class RecordSelector(BaseModel): 2037 type: Literal["RecordSelector"] 2038 extractor: Union[DpathExtractor, CustomRecordExtractor] 2039 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2040 None, 2041 description="Responsible for filtering records to be emitted by the Source.", 2042 title="Record Filter", 2043 ) 2044 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2045 None, 2046 description="Responsible for normalization according to the schema.", 2047 title="Schema Normalization", 2048 ) 2049 transform_before_filtering: Optional[bool] = Field( 2050 None, 2051 description="If true, transformation will be applied before record filtering.", 2052 title="Transform Before Filtering", 2053 ) 2054 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2055 2056 2057class GzipDecoder(BaseModel): 2058 type: Literal["GzipDecoder"] 2059 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] 2060 2061 2062class RequestBodyGraphQL(BaseModel): 2063 type: Literal["RequestBodyGraphQL"] 2064 value: RequestBodyGraphQlQuery 2065 2066 2067class DpathValidator(BaseModel): 2068 type: Literal["DpathValidator"] 2069 field_path: List[str] = Field( 2070 ..., 2071 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2072 examples=[ 2073 ["data"], 2074 ["data", "records"], 2075 ["data", "{{ parameters.name }}"], 2076 ["data", "*", "record"], 2077 ], 2078 title="Field Path", 2079 ) 2080 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2081 ..., 2082 description="The condition that the specified config value will be evaluated against", 2083 title="Validation Strategy", 2084 ) 2085 2086 2087class PredicateValidator(BaseModel): 2088 type: Literal["PredicateValidator"] 2089 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2090 ..., 2091 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2092 examples=[ 2093 "test-value", 2094 "{{ config['api_version'] }}", 2095 "{{ config['tenant_id'] }}", 2096 123, 2097 ], 2098 title="Value", 2099 ) 2100 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2101 ..., 2102 description="The validation strategy to apply to the value.", 2103 title="Validation Strategy", 2104 ) 2105 2106 2107class ConfigAddFields(BaseModel): 2108 type: Literal["ConfigAddFields"] 2109 fields: List[AddedFieldDefinition] = Field( 2110 ..., 2111 description="A list of transformations (path and corresponding value) that will be added to the config.", 2112 title="Fields", 2113 ) 2114 condition: Optional[str] = Field( 2115 "", 2116 description="Fields will be added if expression is evaluated to True.", 2117 examples=[ 2118 "{{ config['environemnt'] == 'sandbox' }}", 2119 "{{ property is integer }}", 2120 "{{ property|length > 5 }}", 2121 "{{ property == 'some_string_to_match' }}", 2122 ], 2123 ) 2124 2125 2126class CompositeErrorHandler(BaseModel): 2127 type: Literal["CompositeErrorHandler"] 2128 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2129 ..., 2130 description="List of error handlers to iterate on to determine how to handle a failed response.", 2131 title="Error Handlers", 2132 ) 2133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2134 2135 2136class HTTPAPIBudget(BaseModel): 2137 class Config: 2138 extra = Extra.allow 2139 2140 type: Literal["HTTPAPIBudget"] 2141 policies: List[ 2142 Union[ 2143 FixedWindowCallRatePolicy, 2144 MovingWindowCallRatePolicy, 2145 UnlimitedCallRatePolicy, 2146 ] 2147 ] = Field( 2148 ..., 2149 description="List of call rate policies that define how many calls are allowed.", 2150 title="Policies", 2151 ) 2152 ratelimit_reset_header: Optional[str] = Field( 2153 "ratelimit-reset", 2154 description="The HTTP response header name that indicates when the rate limit resets.", 2155 title="Rate Limit Reset Header", 2156 ) 2157 ratelimit_remaining_header: Optional[str] = Field( 2158 "ratelimit-remaining", 2159 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2160 title="Rate Limit Remaining Header", 2161 ) 2162 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2163 [429], 2164 description="List of HTTP status codes that indicate a rate limit has been hit.", 2165 title="Status Codes for Rate Limit Hit", 2166 ) 2167 2168 2169class ZipfileDecoder(BaseModel): 2170 class Config: 2171 extra = Extra.allow 2172 2173 type: Literal["ZipfileDecoder"] 2174 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2175 ..., 2176 description="Parser to parse the decompressed data from the zipfile(s).", 2177 title="Parser", 2178 ) 2179 2180 2181class ConfigMigration(BaseModel): 2182 type: Literal["ConfigMigration"] 2183 description: Optional[str] = Field( 2184 None, description="The description/purpose of the config migration." 2185 ) 2186 transformations: List[ 2187 Union[ 2188 ConfigRemapField, 2189 ConfigAddFields, 2190 ConfigRemoveFields, 2191 CustomConfigTransformation, 2192 ] 2193 ] = Field( 2194 ..., 2195 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2196 title="Transformations", 2197 ) 2198 2199 2200class ConfigNormalizationRules(BaseModel): 2201 class Config: 2202 extra = Extra.forbid 2203 2204 type: Literal["ConfigNormalizationRules"] 2205 config_migrations: Optional[List[ConfigMigration]] = Field( 2206 [], 2207 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2208 title="Config Migrations", 2209 ) 2210 transformations: Optional[ 2211 List[ 2212 Union[ 2213 ConfigRemapField, 2214 ConfigAddFields, 2215 ConfigRemoveFields, 2216 CustomConfigTransformation, 2217 ] 2218 ] 2219 ] = Field( 2220 [], 2221 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2222 title="Transformations", 2223 ) 2224 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2225 [], 2226 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2227 title="Validations", 2228 ) 2229 2230 2231class Spec(BaseModel): 2232 type: Literal["Spec"] 2233 connection_specification: Dict[str, Any] = Field( 2234 ..., 2235 description="A connection specification describing how a the connector can be configured.", 2236 title="Connection Specification", 2237 ) 2238 documentation_url: Optional[str] = Field( 2239 None, 2240 description="URL of the connector's documentation page.", 2241 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2242 title="Documentation URL", 2243 ) 2244 advanced_auth: Optional[AuthFlow] = Field( 2245 None, 2246 description="Advanced specification for configuring the authentication flow.", 2247 title="Advanced Auth", 2248 ) 2249 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2250 None, title="Config Normalization Rules" 2251 ) 2252 2253 2254class DeclarativeSource1(BaseModel): 2255 class Config: 2256 extra = Extra.forbid 2257 2258 type: Literal["DeclarativeSource"] 2259 check: Union[CheckStream, CheckDynamicStream] 2260 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2261 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2262 version: str = Field( 2263 ..., 2264 description="The version of the Airbyte CDK used to build and test the source.", 2265 ) 2266 schemas: Optional[Schemas] = None 2267 definitions: Optional[Dict[str, Any]] = None 2268 spec: Optional[Spec] = None 2269 concurrency_level: Optional[ConcurrencyLevel] = None 2270 api_budget: Optional[HTTPAPIBudget] = None 2271 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2272 None, 2273 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2274 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2275 title="Maximum Concurrent Asynchronous Jobs", 2276 ) 2277 metadata: Optional[Dict[str, Any]] = Field( 2278 None, 2279 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2280 ) 2281 description: Optional[str] = Field( 2282 None, 2283 description="A description of the connector. It will be presented on the Source documentation page.", 2284 ) 2285 2286 2287class DeclarativeSource2(BaseModel): 2288 class Config: 2289 extra = Extra.forbid 2290 2291 type: Literal["DeclarativeSource"] 2292 check: Union[CheckStream, CheckDynamicStream] 2293 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2294 None 2295 ) 2296 dynamic_streams: List[DynamicDeclarativeStream] 2297 version: str = Field( 2298 ..., 2299 description="The version of the Airbyte CDK used to build and test the source.", 2300 ) 2301 schemas: Optional[Schemas] = None 2302 definitions: Optional[Dict[str, Any]] = None 2303 spec: Optional[Spec] = None 2304 concurrency_level: Optional[ConcurrencyLevel] = None 2305 api_budget: Optional[HTTPAPIBudget] = None 2306 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2307 None, 2308 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2309 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2310 title="Maximum Concurrent Asynchronous Jobs", 2311 ) 2312 metadata: Optional[Dict[str, Any]] = Field( 2313 None, 2314 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2315 ) 2316 description: Optional[str] = Field( 2317 None, 2318 description="A description of the connector. It will be presented on the Source documentation page.", 2319 ) 2320 2321 2322class DeclarativeSource(BaseModel): 2323 class Config: 2324 extra = Extra.forbid 2325 2326 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2327 ..., 2328 description="An API source that extracts data according to its declarative components.", 2329 title="DeclarativeSource", 2330 ) 2331 2332 2333class SelectiveAuthenticator(BaseModel): 2334 class Config: 2335 extra = Extra.allow 2336 2337 type: Literal["SelectiveAuthenticator"] 2338 authenticator_selection_path: List[str] = Field( 2339 ..., 2340 description="Path of the field in config with selected authenticator name", 2341 examples=[["auth"], ["auth", "type"]], 2342 title="Authenticator Selection Path", 2343 ) 2344 authenticators: Dict[ 2345 str, 2346 Union[ 2347 ApiKeyAuthenticator, 2348 BasicHttpAuthenticator, 2349 BearerAuthenticator, 2350 OAuthAuthenticator, 2351 JwtAuthenticator, 2352 SessionTokenAuthenticator, 2353 LegacySessionTokenAuthenticator, 2354 CustomAuthenticator, 2355 NoAuth, 2356 ], 2357 ] = Field( 2358 ..., 2359 description="Authenticators to select from.", 2360 examples=[ 2361 { 2362 "authenticators": { 2363 "token": "#/definitions/ApiKeyAuthenticator", 2364 "oauth": "#/definitions/OAuthAuthenticator", 2365 "jwt": "#/definitions/JwtAuthenticator", 2366 } 2367 } 2368 ], 2369 title="Authenticators", 2370 ) 2371 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2372 2373 2374class ConditionalStreams(BaseModel): 2375 type: Literal["ConditionalStreams"] 2376 condition: str = Field( 2377 ..., 2378 description="Condition that will be evaluated to determine if a set of streams should be available.", 2379 examples=["{{ config['is_sandbox'] }}"], 2380 title="Condition", 2381 ) 2382 streams: List[DeclarativeStream] = Field( 2383 ..., 2384 description="Streams that will be used during an operation based on the condition.", 2385 title="Streams", 2386 ) 2387 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2388 2389 2390class FileUploader(BaseModel): 2391 type: Literal["FileUploader"] 2392 requester: Union[HttpRequester, CustomRequester] = Field( 2393 ..., 2394 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2395 ) 2396 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2397 ..., 2398 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2399 ) 2400 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2401 None, 2402 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2403 ) 2404 filename_extractor: Optional[str] = Field( 2405 None, 2406 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2407 examples=[ 2408 "{{ record.id }}/{{ record.file_name }}/", 2409 "{{ record.id }}_{{ record.file_name }}/", 2410 ], 2411 ) 2412 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2413 2414 2415class DeclarativeStream(BaseModel): 2416 class Config: 2417 extra = Extra.allow 2418 2419 type: Literal["DeclarativeStream"] 2420 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2421 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2422 ..., 2423 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2424 title="Retriever", 2425 ) 2426 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2427 None, 2428 description="Component used to fetch data incrementally based on a time field in the data.", 2429 title="Incremental Sync", 2430 ) 2431 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2432 schema_loader: Optional[ 2433 Union[ 2434 InlineSchemaLoader, 2435 DynamicSchemaLoader, 2436 JsonFileSchemaLoader, 2437 List[ 2438 Union[ 2439 InlineSchemaLoader, 2440 DynamicSchemaLoader, 2441 JsonFileSchemaLoader, 2442 CustomSchemaLoader, 2443 ] 2444 ], 2445 CustomSchemaLoader, 2446 ] 2447 ] = Field( 2448 None, 2449 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2450 title="Schema Loader", 2451 ) 2452 transformations: Optional[ 2453 List[ 2454 Union[ 2455 AddFields, 2456 RemoveFields, 2457 KeysToLower, 2458 KeysToSnakeCase, 2459 FlattenFields, 2460 DpathFlattenFields, 2461 KeysReplace, 2462 CustomTransformation, 2463 ] 2464 ] 2465 ] = Field( 2466 None, 2467 description="A list of transformations to be applied to each output record.", 2468 title="Transformations", 2469 ) 2470 state_migrations: Optional[ 2471 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2472 ] = Field( 2473 [], 2474 description="Array of state migrations to be applied on the input state", 2475 title="State Migrations", 2476 ) 2477 file_uploader: Optional[FileUploader] = Field( 2478 None, 2479 description="(experimental) Describes how to fetch a file", 2480 title="File Uploader", 2481 ) 2482 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2483 2484 2485class SessionTokenAuthenticator(BaseModel): 2486 type: Literal["SessionTokenAuthenticator"] 2487 login_requester: HttpRequester = Field( 2488 ..., 2489 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2490 examples=[ 2491 { 2492 "type": "HttpRequester", 2493 "url_base": "https://my_api.com", 2494 "path": "/login", 2495 "authenticator": { 2496 "type": "BasicHttpAuthenticator", 2497 "username": "{{ config.username }}", 2498 "password": "{{ config.password }}", 2499 }, 2500 } 2501 ], 2502 title="Login Requester", 2503 ) 2504 session_token_path: List[str] = Field( 2505 ..., 2506 description="The path in the response body returned from the login requester to the session token.", 2507 examples=[["access_token"], ["result", "token"]], 2508 title="Session Token Path", 2509 ) 2510 expiration_duration: Optional[str] = Field( 2511 None, 2512 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2513 examples=["PT1H", "P1D"], 2514 title="Expiration Duration", 2515 ) 2516 request_authentication: Union[ 2517 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2518 ] = Field( 2519 ..., 2520 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2521 title="Data Request Authentication", 2522 ) 2523 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2524 None, description="Component used to decode the response.", title="Decoder" 2525 ) 2526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2527 2528 2529class HttpRequester(BaseModelWithDeprecations): 2530 type: Literal["HttpRequester"] 2531 url_base: Optional[str] = Field( 2532 None, 2533 deprecated=True, 2534 deprecation_message="Use `url` field instead.", 2535 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2536 examples=[ 2537 "https://connect.squareup.com/v2", 2538 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2539 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2540 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2541 ], 2542 title="API Base URL", 2543 ) 2544 url: Optional[str] = Field( 2545 None, 2546 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2547 examples=[ 2548 "https://connect.squareup.com/v2", 2549 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2550 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2551 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2552 ], 2553 title="API Endpoint URL", 2554 ) 2555 path: Optional[str] = Field( 2556 None, 2557 deprecated=True, 2558 deprecation_message="Use `url` field instead.", 2559 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2560 examples=[ 2561 "/products", 2562 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2563 "/trades/{{ config['symbol_id'] }}/history", 2564 ], 2565 title="URL Path", 2566 ) 2567 http_method: Optional[HttpMethod] = Field( 2568 HttpMethod.GET, 2569 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2570 examples=["GET", "POST"], 2571 title="HTTP Method", 2572 ) 2573 authenticator: Optional[ 2574 Union[ 2575 ApiKeyAuthenticator, 2576 BasicHttpAuthenticator, 2577 BearerAuthenticator, 2578 OAuthAuthenticator, 2579 JwtAuthenticator, 2580 SessionTokenAuthenticator, 2581 SelectiveAuthenticator, 2582 CustomAuthenticator, 2583 NoAuth, 2584 LegacySessionTokenAuthenticator, 2585 ] 2586 ] = Field( 2587 None, 2588 description="Authentication method to use for requests sent to the API.", 2589 title="Authenticator", 2590 ) 2591 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2592 None, 2593 deprecated=True, 2594 deprecation_message="Use `query_properties` field instead.", 2595 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2596 title="Fetch Properties from Endpoint", 2597 ) 2598 query_properties: Optional[QueryProperties] = Field( 2599 None, 2600 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2601 title="Query Properties", 2602 ) 2603 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2604 None, 2605 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2606 examples=[ 2607 {"unit": "day"}, 2608 { 2609 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2610 }, 2611 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2612 {"sort_by[asc]": "updated_at"}, 2613 ], 2614 title="Query Parameters", 2615 ) 2616 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2617 None, 2618 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2619 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2620 title="Request Headers", 2621 ) 2622 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2623 None, 2624 deprecated=True, 2625 deprecation_message="Use `request_body` field instead.", 2626 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2627 examples=[ 2628 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2629 ], 2630 title="Request Body Payload (Non-JSON)", 2631 ) 2632 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2633 None, 2634 deprecated=True, 2635 deprecation_message="Use `request_body` field instead.", 2636 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2637 examples=[ 2638 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2639 {"key": "{{ config['value'] }}"}, 2640 {"sort": {"field": "updated_at", "order": "ascending"}}, 2641 ], 2642 title="Request Body JSON Payload", 2643 ) 2644 request_body: Optional[ 2645 Union[ 2646 RequestBodyPlainText, 2647 RequestBodyUrlEncodedForm, 2648 RequestBodyJsonObject, 2649 RequestBodyGraphQL, 2650 ] 2651 ] = Field( 2652 None, 2653 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2654 title="Request Body", 2655 ) 2656 error_handler: Optional[ 2657 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2658 ] = Field( 2659 None, 2660 description="Error handler component that defines how to handle errors.", 2661 title="Error Handler", 2662 ) 2663 use_cache: Optional[bool] = Field( 2664 False, 2665 description="Enables stream requests caching. This field is automatically set by the CDK.", 2666 title="Use Cache", 2667 ) 2668 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2669 2670 2671class DynamicSchemaLoader(BaseModel): 2672 type: Literal["DynamicSchemaLoader"] 2673 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2674 ..., 2675 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2676 title="Retriever", 2677 ) 2678 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2679 None, 2680 description="Responsible for filtering fields to be added to json schema.", 2681 title="Schema Filter", 2682 ) 2683 schema_transformations: Optional[ 2684 List[ 2685 Union[ 2686 AddFields, 2687 RemoveFields, 2688 KeysToLower, 2689 KeysToSnakeCase, 2690 FlattenFields, 2691 DpathFlattenFields, 2692 KeysReplace, 2693 CustomTransformation, 2694 ] 2695 ] 2696 ] = Field( 2697 None, 2698 description="A list of transformations to be applied to the schema.", 2699 title="Schema Transformations", 2700 ) 2701 schema_type_identifier: SchemaTypeIdentifier 2702 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2703 2704 2705class ParentStreamConfig(BaseModel): 2706 type: Literal["ParentStreamConfig"] 2707 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2708 ..., description="Reference to the parent stream.", title="Parent Stream" 2709 ) 2710 parent_key: str = Field( 2711 ..., 2712 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2713 examples=["id", "{{ config['parent_record_id'] }}"], 2714 title="Parent Key", 2715 ) 2716 partition_field: str = Field( 2717 ..., 2718 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2719 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2720 title="Current Parent Key Value Identifier", 2721 ) 2722 request_option: Optional[RequestOption] = Field( 2723 None, 2724 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2725 title="Request Option", 2726 ) 2727 incremental_dependency: Optional[bool] = Field( 2728 False, 2729 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2730 title="Incremental Dependency", 2731 ) 2732 lazy_read_pointer: Optional[List[str]] = Field( 2733 [], 2734 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2735 title="Lazy Read Pointer", 2736 ) 2737 extra_fields: Optional[List[List[str]]] = Field( 2738 None, 2739 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2740 title="Extra Fields", 2741 ) 2742 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2743 2744 2745class PropertiesFromEndpoint(BaseModel): 2746 type: Literal["PropertiesFromEndpoint"] 2747 property_field_path: List[str] = Field( 2748 ..., 2749 description="Describes the path to the field that should be extracted", 2750 examples=[["name"]], 2751 ) 2752 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2753 ..., 2754 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2755 ) 2756 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2757 2758 2759class QueryProperties(BaseModel): 2760 type: Literal["QueryProperties"] 2761 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2762 ..., 2763 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2764 title="Property List", 2765 ) 2766 always_include_properties: Optional[List[str]] = Field( 2767 None, 2768 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2769 title="Always Include Properties", 2770 ) 2771 property_chunking: Optional[PropertyChunking] = Field( 2772 None, 2773 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2774 title="Property Chunking", 2775 ) 2776 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2777 2778 2779class StateDelegatingStream(BaseModel): 2780 type: Literal["StateDelegatingStream"] 2781 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2782 full_refresh_stream: DeclarativeStream = Field( 2783 ..., 2784 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2785 title="Full Refresh Stream", 2786 ) 2787 incremental_stream: DeclarativeStream = Field( 2788 ..., 2789 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2790 title="Incremental Stream", 2791 ) 2792 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2793 2794 2795class SimpleRetriever(BaseModel): 2796 type: Literal["SimpleRetriever"] 2797 requester: Union[HttpRequester, CustomRequester] = Field( 2798 ..., 2799 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2800 ) 2801 decoder: Optional[ 2802 Union[ 2803 JsonDecoder, 2804 XmlDecoder, 2805 CsvDecoder, 2806 JsonlDecoder, 2807 GzipDecoder, 2808 IterableDecoder, 2809 ZipfileDecoder, 2810 CustomDecoder, 2811 ] 2812 ] = Field( 2813 None, 2814 description="Component decoding the response so records can be extracted.", 2815 title="HTTP Response Format", 2816 ) 2817 record_selector: RecordSelector = Field( 2818 ..., 2819 description="Component that describes how to extract records from a HTTP response.", 2820 ) 2821 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2822 None, 2823 description="Paginator component that describes how to navigate through the API's pages.", 2824 ) 2825 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2826 False, 2827 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2828 ) 2829 partition_router: Optional[ 2830 Union[ 2831 SubstreamPartitionRouter, 2832 ListPartitionRouter, 2833 GroupingPartitionRouter, 2834 CustomPartitionRouter, 2835 List[ 2836 Union[ 2837 SubstreamPartitionRouter, 2838 ListPartitionRouter, 2839 GroupingPartitionRouter, 2840 CustomPartitionRouter, 2841 ] 2842 ], 2843 ] 2844 ] = Field( 2845 None, 2846 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2847 title="Partition Router", 2848 ) 2849 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2850 2851 2852class AsyncRetriever(BaseModel): 2853 type: Literal["AsyncRetriever"] 2854 record_selector: RecordSelector = Field( 2855 ..., 2856 description="Component that describes how to extract records from a HTTP response.", 2857 ) 2858 status_mapping: AsyncJobStatusMap = Field( 2859 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2860 ) 2861 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2862 ..., description="Responsible for fetching the actual status of the async job." 2863 ) 2864 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2865 None, 2866 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2867 ) 2868 download_extractor: Optional[ 2869 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2870 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2871 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2872 ..., 2873 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2874 ) 2875 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2876 ..., 2877 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2878 ) 2879 polling_job_timeout: Optional[Union[int, str]] = Field( 2880 None, 2881 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2882 ) 2883 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2884 None, 2885 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2886 ) 2887 download_requester: Union[HttpRequester, CustomRequester] = Field( 2888 ..., 2889 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2890 ) 2891 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2892 None, 2893 description="Paginator component that describes how to navigate through the API's pages during download.", 2894 ) 2895 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2896 None, 2897 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2898 ) 2899 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2900 None, 2901 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2902 ) 2903 partition_router: Optional[ 2904 Union[ 2905 ListPartitionRouter, 2906 SubstreamPartitionRouter, 2907 GroupingPartitionRouter, 2908 CustomPartitionRouter, 2909 List[ 2910 Union[ 2911 ListPartitionRouter, 2912 SubstreamPartitionRouter, 2913 GroupingPartitionRouter, 2914 CustomPartitionRouter, 2915 ] 2916 ], 2917 ] 2918 ] = Field( 2919 [], 2920 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2921 title="Partition Router", 2922 ) 2923 decoder: Optional[ 2924 Union[ 2925 CsvDecoder, 2926 GzipDecoder, 2927 JsonDecoder, 2928 JsonlDecoder, 2929 IterableDecoder, 2930 XmlDecoder, 2931 ZipfileDecoder, 2932 CustomDecoder, 2933 ] 2934 ] = Field( 2935 None, 2936 description="Component decoding the response so records can be extracted.", 2937 title="HTTP Response Format", 2938 ) 2939 download_decoder: Optional[ 2940 Union[ 2941 CsvDecoder, 2942 GzipDecoder, 2943 JsonDecoder, 2944 JsonlDecoder, 2945 IterableDecoder, 2946 XmlDecoder, 2947 ZipfileDecoder, 2948 CustomDecoder, 2949 ] 2950 ] = Field( 2951 None, 2952 description="Component decoding the download response so records can be extracted.", 2953 title="Download HTTP Response Format", 2954 ) 2955 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2956 2957 2958class SubstreamPartitionRouter(BaseModel): 2959 type: Literal["SubstreamPartitionRouter"] 2960 parent_stream_configs: List[ParentStreamConfig] = Field( 2961 ..., 2962 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2963 title="Parent Stream Configs", 2964 ) 2965 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2966 2967 2968class GroupingPartitionRouter(BaseModel): 2969 type: Literal["GroupingPartitionRouter"] 2970 group_size: int = Field( 2971 ..., 2972 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2973 examples=[10, 50], 2974 title="Group Size", 2975 ) 2976 underlying_partition_router: Union[ 2977 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2978 ] = Field( 2979 ..., 2980 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2981 title="Underlying Partition Router", 2982 ) 2983 deduplicate: Optional[bool] = Field( 2984 True, 2985 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2986 title="Deduplicate Partitions", 2987 ) 2988 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2989 2990 2991class HttpComponentsResolver(BaseModel): 2992 type: Literal["HttpComponentsResolver"] 2993 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2994 ..., 2995 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2996 title="Retriever", 2997 ) 2998 components_mapping: List[ComponentMappingDefinition] 2999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3000 3001 3002class DynamicDeclarativeStream(BaseModel): 3003 type: Literal["DynamicDeclarativeStream"] 3004 name: Optional[str] = Field( 3005 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3006 ) 3007 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3008 ..., description="Reference to the stream template.", title="Stream Template" 3009 ) 3010 components_resolver: Union[ 3011 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3012 ] = Field( 3013 ..., 3014 description="Component resolve and populates stream templates with components values.", 3015 title="Components Resolver", 3016 ) 3017 use_parent_parameters: Optional[bool] = Field( 3018 True, 3019 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3020 title="Use Parent Parameters", 3021 ) 3022 3023 3024ComplexFieldType.update_forward_refs() 3025GzipDecoder.update_forward_refs() 3026CompositeErrorHandler.update_forward_refs() 3027DeclarativeSource1.update_forward_refs() 3028DeclarativeSource2.update_forward_refs() 3029SelectiveAuthenticator.update_forward_refs() 3030ConditionalStreams.update_forward_refs() 3031FileUploader.update_forward_refs() 3032DeclarativeStream.update_forward_refs() 3033SessionTokenAuthenticator.update_forward_refs() 3034HttpRequester.update_forward_refs() 3035DynamicSchemaLoader.update_forward_refs() 3036ParentStreamConfig.update_forward_refs() 3037PropertiesFromEndpoint.update_forward_refs() 3038SimpleRetriever.update_forward_refs() 3039AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
50class DynamicStreamCheckConfig(BaseModel): 51 type: Literal["DynamicStreamCheckConfig"] 52 dynamic_stream_name: str = Field( 53 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 54 ) 55 stream_count: Optional[int] = Field( 56 0, 57 description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.", 58 title="Stream Count", 59 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
62class CheckDynamicStream(BaseModel): 63 type: Literal["CheckDynamicStream"] 64 stream_count: int = Field( 65 ..., 66 description="Numbers of the streams to try reading from when running a check operation.", 67 title="Stream Count", 68 ) 69 use_check_availability: Optional[bool] = Field( 70 True, 71 description="Enables stream check availability. This field is automatically set by the CDK.", 72 title="Use Check Availability", 73 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
76class ConcurrencyLevel(BaseModel): 77 type: Optional[Literal["ConcurrencyLevel"]] = None 78 default_concurrency: Union[int, str] = Field( 79 ..., 80 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 81 examples=[10, "{{ config['num_workers'] or 10 }}"], 82 title="Default Concurrency", 83 ) 84 max_concurrency: Optional[int] = Field( 85 None, 86 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 87 examples=[20, 100], 88 title="Max Concurrency", 89 ) 90 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
93class ConstantBackoffStrategy(BaseModel): 94 type: Literal["ConstantBackoffStrategy"] 95 backoff_time_in_seconds: Union[float, str] = Field( 96 ..., 97 description="Backoff time in seconds.", 98 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 99 title="Backoff Time", 100 ) 101 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
104class CursorPagination(BaseModel): 105 type: Literal["CursorPagination"] 106 cursor_value: str = Field( 107 ..., 108 description="Value of the cursor defining the next page to fetch.", 109 examples=[ 110 "{{ headers.link.next.cursor }}", 111 "{{ last_record['key'] }}", 112 "{{ response['nextPage'] }}", 113 ], 114 title="Cursor Value", 115 ) 116 page_size: Optional[int] = Field( 117 None, 118 description="The number of records to include in each pages.", 119 examples=[100], 120 title="Page Size", 121 ) 122 stop_condition: Optional[str] = Field( 123 None, 124 description="Template string evaluating when to stop paginating.", 125 examples=[ 126 "{{ response.data.has_more is false }}", 127 "{{ 'next' not in headers['link'] }}", 128 ], 129 title="Stop Condition", 130 ) 131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
134class CustomAuthenticator(BaseModel): 135 class Config: 136 extra = Extra.allow 137 138 type: Literal["CustomAuthenticator"] 139 class_name: str = Field( 140 ..., 141 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 142 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 143 title="Class Name", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
148class CustomBackoffStrategy(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomBackoffStrategy"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.MyCustomBackoffStrategy"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
162class CustomErrorHandler(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomErrorHandler"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomErrorHandler"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
176class CustomPaginationStrategy(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomPaginationStrategy"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomPaginationStrategy"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
190class CustomRecordExtractor(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomRecordExtractor"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomRecordExtractor"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
204class CustomRecordFilter(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordFilter"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomCustomRecordFilter"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
218class CustomRequester(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRequester"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomRecordExtractor"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
232class CustomRetriever(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRetriever"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRetriever"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
246class CustomPartitionRouter(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomPartitionRouter"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomPartitionRouter"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
260class CustomSchemaLoader(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomSchemaLoader"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomSchemaLoader"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
274class CustomSchemaNormalization(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaNormalization"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 282 examples=[ 283 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 284 ], 285 title="Class Name", 286 ) 287 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
290class CustomStateMigration(BaseModel): 291 class Config: 292 extra = Extra.allow 293 294 type: Literal["CustomStateMigration"] 295 class_name: str = Field( 296 ..., 297 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 298 examples=["source_railz.components.MyCustomStateMigration"], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
304class CustomTransformation(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomTransformation"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomTransformation"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
335class Algorithm(Enum): 336 HS256 = "HS256" 337 HS384 = "HS384" 338 HS512 = "HS512" 339 ES256 = "ES256" 340 ES256K = "ES256K" 341 ES384 = "ES384" 342 ES512 = "ES512" 343 RS256 = "RS256" 344 RS384 = "RS384" 345 RS512 = "RS512" 346 PS256 = "PS256" 347 PS384 = "PS384" 348 PS512 = "PS512" 349 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
352class JwtHeaders(BaseModel): 353 class Config: 354 extra = Extra.forbid 355 356 kid: Optional[str] = Field( 357 None, 358 description="Private key ID for user account.", 359 examples=["{{ config['kid'] }}"], 360 title="Key Identifier", 361 ) 362 typ: Optional[str] = Field( 363 "JWT", 364 description="The media type of the complete JWT.", 365 examples=["JWT"], 366 title="Type", 367 ) 368 cty: Optional[str] = Field( 369 None, 370 description="Content type of JWT header.", 371 examples=["JWT"], 372 title="Content Type", 373 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
376class JwtPayload(BaseModel): 377 class Config: 378 extra = Extra.forbid 379 380 iss: Optional[str] = Field( 381 None, 382 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 383 examples=["{{ config['iss'] }}"], 384 title="Issuer", 385 ) 386 sub: Optional[str] = Field( 387 None, 388 description="The subject of the JWT. Commonly defined by the API.", 389 title="Subject", 390 ) 391 aud: Optional[str] = Field( 392 None, 393 description="The recipient that the JWT is intended for. Commonly defined by the API.", 394 examples=["appstoreconnect-v1"], 395 title="Audience", 396 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
399class RefreshTokenUpdater(BaseModel): 400 refresh_token_name: Optional[str] = Field( 401 "refresh_token", 402 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 403 examples=["refresh_token"], 404 title="Refresh Token Property Name", 405 ) 406 access_token_config_path: Optional[List[str]] = Field( 407 ["credentials", "access_token"], 408 description="Config path to the access token. Make sure the field actually exists in the config.", 409 examples=[["credentials", "access_token"], ["access_token"]], 410 title="Config Path To Access Token", 411 ) 412 refresh_token_config_path: Optional[List[str]] = Field( 413 ["credentials", "refresh_token"], 414 description="Config path to the access token. Make sure the field actually exists in the config.", 415 examples=[["credentials", "refresh_token"], ["refresh_token"]], 416 title="Config Path To Refresh Token", 417 ) 418 token_expiry_date_config_path: Optional[List[str]] = Field( 419 ["credentials", "token_expiry_date"], 420 description="Config path to the expiry date. Make sure actually exists in the config.", 421 examples=[["credentials", "token_expiry_date"]], 422 title="Config Path To Expiry Date", 423 ) 424 refresh_token_error_status_codes: Optional[List[int]] = Field( 425 [], 426 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 427 examples=[[400, 500]], 428 title="Refresh Token Error Status Codes", 429 ) 430 refresh_token_error_key: Optional[str] = Field( 431 "", 432 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 433 examples=["error"], 434 title="Refresh Token Error Key", 435 ) 436 refresh_token_error_values: Optional[List[str]] = Field( 437 [], 438 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 439 examples=[["invalid_grant", "invalid_permissions"]], 440 title="Refresh Token Error Values", 441 )
class
Rate(pydantic.v1.main.BaseModel):
444class Rate(BaseModel): 445 class Config: 446 extra = Extra.allow 447 448 limit: Union[int, str] = Field( 449 ..., 450 description="The maximum number of calls allowed within the interval.", 451 title="Limit", 452 ) 453 interval: str = Field( 454 ..., 455 description="The time interval for the rate limit.", 456 examples=["PT1H", "P1D"], 457 title="Interval", 458 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
461class HttpRequestRegexMatcher(BaseModel): 462 class Config: 463 extra = Extra.allow 464 465 method: Optional[str] = Field( 466 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 467 ) 468 url_base: Optional[str] = Field( 469 None, 470 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 471 title="URL Base", 472 ) 473 url_path_pattern: Optional[str] = Field( 474 None, 475 description="A regular expression pattern to match the URL path.", 476 title="URL Path Pattern", 477 ) 478 params: Optional[Dict[str, Any]] = Field( 479 None, description="The query parameters to match.", title="Parameters" 480 ) 481 headers: Optional[Dict[str, Any]] = Field( 482 None, description="The headers to match.", title="Headers" 483 )
class
HttpRequestRegexMatcher.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
486class DpathExtractor(BaseModel): 487 type: Literal["DpathExtractor"] 488 field_path: List[str] = Field( 489 ..., 490 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 491 examples=[ 492 ["data"], 493 ["data", "records"], 494 ["data", "{{ parameters.name }}"], 495 ["data", "*", "record"], 496 ], 497 title="Field Path", 498 ) 499 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
507class ExponentialBackoffStrategy(BaseModel): 508 type: Literal["ExponentialBackoffStrategy"] 509 factor: Optional[Union[float, str]] = Field( 510 5, 511 description="Multiplicative constant applied on each retry.", 512 examples=[5, 5.5, "10"], 513 title="Factor", 514 ) 515 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
518class GroupByKeyMergeStrategy(BaseModel): 519 type: Literal["GroupByKeyMergeStrategy"] 520 key: Union[str, List[str]] = Field( 521 ..., 522 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 523 examples=["id", ["parent_id", "end_date"]], 524 title="Key", 525 ) 526 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
Action(enum.Enum):
538class Action(Enum): 539 SUCCESS = "SUCCESS" 540 FAIL = "FAIL" 541 RETRY = "RETRY" 542 IGNORE = "IGNORE" 543 RATE_LIMITED = "RATE_LIMITED"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
class
FailureType(enum.Enum):
546class FailureType(Enum): 547 system_error = "system_error" 548 config_error = "config_error" 549 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
552class HttpResponseFilter(BaseModel): 553 type: Literal["HttpResponseFilter"] 554 action: Optional[Action] = Field( 555 None, 556 description="Action to execute if a response matches the filter.", 557 examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], 558 title="Action", 559 ) 560 failure_type: Optional[FailureType] = Field( 561 None, 562 description="Failure type of traced exception if a response matches the filter.", 563 examples=["system_error", "config_error", "transient_error"], 564 title="Failure Type", 565 ) 566 error_message: Optional[str] = Field( 567 None, 568 description="Error Message to display if the response matches the filter.", 569 title="Error Message", 570 ) 571 error_message_contains: Optional[str] = Field( 572 None, 573 description="Match the response if its error message contains the substring.", 574 example=["This API operation is not enabled for this site"], 575 title="Error Message Substring", 576 ) 577 http_codes: Optional[List[int]] = Field( 578 None, 579 description="Match the response if its HTTP code is included in this list.", 580 examples=[[420, 429], [500]], 581 title="HTTP Codes", 582 unique_items=True, 583 ) 584 predicate: Optional[str] = Field( 585 None, 586 description="Match the response if the predicate evaluates to true.", 587 examples=[ 588 "{{ 'Too much requests' in response }}", 589 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 590 ], 591 title="Predicate", 592 ) 593 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
596class ComplexFieldType(BaseModel): 597 field_type: str 598 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
601class TypesMap(BaseModel): 602 target_type: Union[str, List[str], ComplexFieldType] 603 current_type: Union[str, List[str]] 604 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
607class SchemaTypeIdentifier(BaseModel): 608 type: Optional[Literal["SchemaTypeIdentifier"]] = None 609 schema_pointer: Optional[List[str]] = Field( 610 [], 611 description="List of nested fields defining the schema field path to extract. Defaults to [].", 612 title="Schema Path", 613 ) 614 key_pointer: List[str] = Field( 615 ..., 616 description="List of potentially nested fields describing the full path of the field key to extract.", 617 title="Key Path", 618 ) 619 type_pointer: Optional[List[str]] = Field( 620 None, 621 description="List of potentially nested fields describing the full path of the field type to extract.", 622 title="Type Path", 623 ) 624 types_mapping: Optional[List[TypesMap]] = None 625 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
628class InlineSchemaLoader(BaseModel): 629 type: Literal["InlineSchemaLoader"] 630 schema_: Optional[Dict[str, Any]] = Field( 631 None, 632 alias="schema", 633 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 634 title="Schema", 635 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
638class JsonFileSchemaLoader(BaseModel): 639 type: Literal["JsonFileSchemaLoader"] 640 file_path: Optional[str] = Field( 641 None, 642 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 643 example=["./schemas/users.json"], 644 title="File Path", 645 ) 646 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
667class FlattenFields(BaseModel): 668 type: Literal["FlattenFields"] 669 flatten_lists: Optional[bool] = Field( 670 True, 671 description="Whether to flatten lists or leave it as is. Default is True.", 672 title="Flatten Lists", 673 ) 674 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
677class KeyTransformation(BaseModel): 678 type: Literal["KeyTransformation"] 679 prefix: Optional[str] = Field( 680 None, 681 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 682 examples=["flattened_"], 683 title="Key Prefix", 684 ) 685 suffix: Optional[str] = Field( 686 None, 687 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 688 examples=["_flattened"], 689 title="Key Suffix", 690 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
693class DpathFlattenFields(BaseModel): 694 type: Literal["DpathFlattenFields"] 695 field_path: List[str] = Field( 696 ..., 697 description="A path to field that needs to be flattened.", 698 examples=[["data"], ["data", "*", "field"]], 699 title="Field Path", 700 ) 701 delete_origin_value: Optional[bool] = Field( 702 None, 703 description="Whether to delete the origin value or keep it. Default is False.", 704 title="Delete Origin Value", 705 ) 706 replace_record: Optional[bool] = Field( 707 None, 708 description="Whether to replace the origin record or not. Default is False.", 709 title="Replace Origin Record", 710 ) 711 key_transformation: Optional[KeyTransformation] = Field( 712 None, 713 description="Transformation for object keys. If not provided, original key will be used.", 714 title="Key transformation", 715 ) 716 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
719class KeysReplace(BaseModel): 720 type: Literal["KeysReplace"] 721 old: str = Field( 722 ..., 723 description="Old value to replace.", 724 examples=[ 725 " ", 726 "{{ record.id }}", 727 "{{ config['id'] }}", 728 "{{ stream_slice['id'] }}", 729 ], 730 title="Old value", 731 ) 732 new: str = Field( 733 ..., 734 description="New value to set.", 735 examples=[ 736 "_", 737 "{{ record.id }}", 738 "{{ config['id'] }}", 739 "{{ stream_slice['id'] }}", 740 ], 741 title="New value", 742 ) 743 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
754class CustomDecoder(BaseModel): 755 class Config: 756 extra = Extra.allow 757 758 type: Literal["CustomDecoder"] 759 class_name: str = Field( 760 ..., 761 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 762 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 763 title="Class Name", 764 ) 765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
768class MinMaxDatetime(BaseModel): 769 type: Literal["MinMaxDatetime"] 770 datetime: str = Field( 771 ..., 772 description="Datetime value.", 773 examples=[ 774 "2021-01-01", 775 "2021-01-01T00:00:00Z", 776 "{{ config['start_time'] }}", 777 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 778 ], 779 title="Datetime", 780 ) 781 datetime_format: Optional[str] = Field( 782 "", 783 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 784 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 785 title="Datetime Format", 786 ) 787 max_datetime: Optional[str] = Field( 788 None, 789 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 790 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 791 title="Max Datetime", 792 ) 793 min_datetime: Optional[str] = Field( 794 None, 795 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 796 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 797 title="Min Datetime", 798 ) 799 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
819class OauthConnectorInputSpecification(BaseModel): 820 class Config: 821 extra = Extra.allow 822 823 consent_url: str = Field( 824 ..., 825 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 826 examples=[ 827 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 828 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 829 ], 830 title="Consent URL", 831 ) 832 scope: Optional[str] = Field( 833 None, 834 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 835 examples=["user:read user:read_orders workspaces:read"], 836 title="Scopes", 837 ) 838 access_token_url: str = Field( 839 ..., 840 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 841 examples=[ 842 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 843 ], 844 title="Access Token URL", 845 ) 846 access_token_headers: Optional[Dict[str, Any]] = Field( 847 None, 848 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 849 examples=[ 850 { 851 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 852 } 853 ], 854 title="Access Token Headers", 855 ) 856 access_token_params: Optional[Dict[str, Any]] = Field( 857 None, 858 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 859 examples=[ 860 { 861 "{{ auth_code_key }}": "{{ auth_code_value }}", 862 "{{ client_id_key }}": "{{ client_id_value }}", 863 "{{ client_secret_key }}": "{{ client_secret_value }}", 864 } 865 ], 866 title="Access Token Query Params (Json Encoded)", 867 ) 868 extract_output: Optional[List[str]] = Field( 869 None, 870 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 871 examples=[["access_token", "refresh_token", "other_field"]], 872 title="Extract Output", 873 ) 874 state: Optional[State] = Field( 875 None, 876 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 877 examples=[{"min": 7, "max": 128}], 878 title="Configurable State Query Param", 879 ) 880 client_id_key: Optional[str] = Field( 881 None, 882 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 883 examples=["my_custom_client_id_key_name"], 884 title="Client ID Key Override", 885 ) 886 client_secret_key: Optional[str] = Field( 887 None, 888 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 889 examples=["my_custom_client_secret_key_name"], 890 title="Client Secret Key Override", 891 ) 892 scope_key: Optional[str] = Field( 893 None, 894 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 895 examples=["my_custom_scope_key_key_name"], 896 title="Scopes Key Override", 897 ) 898 state_key: Optional[str] = Field( 899 None, 900 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 901 examples=["my_custom_state_key_key_name"], 902 title="State Key Override", 903 ) 904 auth_code_key: Optional[str] = Field( 905 None, 906 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 907 examples=["my_custom_auth_code_key_name"], 908 title="Auth Code Key Override", 909 ) 910 redirect_uri_key: Optional[str] = Field( 911 None, 912 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 913 examples=["my_custom_redirect_uri_key_name"], 914 title="Redirect URI Key Override", 915 )
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
918class OAuthConfigSpecification(BaseModel): 919 class Config: 920 extra = Extra.allow 921 922 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 923 None, 924 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 925 examples=[ 926 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 927 { 928 "app_id": { 929 "type": "string", 930 "path_in_connector_config": ["info", "app_id"], 931 } 932 }, 933 ], 934 title="OAuth user input", 935 ) 936 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 937 None, 938 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 939 title="DeclarativeOAuth Connector Specification", 940 ) 941 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 942 None, 943 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 944 examples=[ 945 { 946 "refresh_token": { 947 "type": "string,", 948 "path_in_connector_config": ["credentials", "refresh_token"], 949 } 950 } 951 ], 952 title="OAuth output specification", 953 ) 954 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 955 None, 956 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 957 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 958 title="OAuth input specification", 959 ) 960 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 961 None, 962 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 963 examples=[ 964 { 965 "client_id": { 966 "type": "string,", 967 "path_in_connector_config": ["credentials", "client_id"], 968 }, 969 "client_secret": { 970 "type": "string,", 971 "path_in_connector_config": ["credentials", "client_secret"], 972 }, 973 } 974 ], 975 title="OAuth server output specification", 976 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
979class OffsetIncrement(BaseModel): 980 type: Literal["OffsetIncrement"] 981 page_size: Optional[Union[int, str]] = Field( 982 None, 983 description="The number of records to include in each pages.", 984 examples=[100, "{{ config['page_size'] }}"], 985 title="Limit", 986 ) 987 inject_on_first_request: Optional[bool] = Field( 988 False, 989 description="Using the `offset` with value `0` during the first request", 990 title="Inject Offset on First Request", 991 ) 992 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
995class PageIncrement(BaseModel): 996 type: Literal["PageIncrement"] 997 page_size: Optional[Union[int, str]] = Field( 998 None, 999 description="The number of records to include in each pages.", 1000 examples=[100, "100", "{{ config['page_size'] }}"], 1001 title="Page Size", 1002 ) 1003 start_from_page: Optional[int] = Field( 1004 0, 1005 description="Index of the first page to request.", 1006 examples=[0, 1], 1007 title="Start From Page", 1008 ) 1009 inject_on_first_request: Optional[bool] = Field( 1010 False, 1011 description="Using the `page number` with value defined by `start_from_page` during the first request", 1012 title="Inject Page Number on First Request", 1013 ) 1014 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1017class PrimaryKey(BaseModel): 1018 __root__: Union[str, List[str], List[List[str]]] = Field( 1019 ..., 1020 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1021 examples=["id", ["code", "type"]], 1022 title="Primary Key", 1023 )
class
PropertyLimitType(enum.Enum):
1026class PropertyLimitType(Enum): 1027 characters = "characters" 1028 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1031class PropertyChunking(BaseModel): 1032 type: Literal["PropertyChunking"] 1033 property_limit_type: PropertyLimitType = Field( 1034 ..., 1035 description="The type used to determine the maximum number of properties per chunk", 1036 title="Property Limit Type", 1037 ) 1038 property_limit: Optional[int] = Field( 1039 None, 1040 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1041 title="Property Limit", 1042 ) 1043 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1044 None, 1045 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1046 title="Record Merge Strategy", 1047 ) 1048 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1051class RecordFilter(BaseModel): 1052 type: Literal["RecordFilter"] 1053 condition: Optional[str] = Field( 1054 "", 1055 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1056 examples=[ 1057 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1058 "{{ record.status in ['active', 'expired'] }}", 1059 ], 1060 title="Condition", 1061 ) 1062 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1070class RemoveFields(BaseModel): 1071 type: Literal["RemoveFields"] 1072 condition: Optional[str] = Field( 1073 "", 1074 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1075 examples=[ 1076 "{{ property|string == '' }}", 1077 "{{ property is integer }}", 1078 "{{ property|length > 5 }}", 1079 "{{ property == 'some_string_to_match' }}", 1080 ], 1081 ) 1082 field_pointers: List[List[str]] = Field( 1083 ..., 1084 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1085 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1086 title="Field Paths", 1087 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1094class InjectInto(Enum): 1095 request_parameter = "request_parameter" 1096 header = "header" 1097 body_data = "body_data" 1098 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1101class RequestOption(BaseModel): 1102 type: Literal["RequestOption"] 1103 inject_into: InjectInto = Field( 1104 ..., 1105 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1106 examples=["request_parameter", "header", "body_data", "body_json"], 1107 title="Inject Into", 1108 ) 1109 field_name: Optional[str] = Field( 1110 None, 1111 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1112 examples=["segment_id"], 1113 title="Field Name", 1114 ) 1115 field_path: Optional[List[str]] = Field( 1116 None, 1117 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1118 examples=[["data", "viewer", "id"]], 1119 title="Field Path", 1120 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1130class LegacySessionTokenAuthenticator(BaseModel): 1131 type: Literal["LegacySessionTokenAuthenticator"] 1132 header: str = Field( 1133 ..., 1134 description="The name of the session token header that will be injected in the request", 1135 examples=["X-Session"], 1136 title="Session Request Header", 1137 ) 1138 login_url: str = Field( 1139 ..., 1140 description="Path of the login URL (do not include the base URL)", 1141 examples=["session"], 1142 title="Login Path", 1143 ) 1144 session_token: Optional[str] = Field( 1145 None, 1146 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1147 example=["{{ config['session_token'] }}"], 1148 title="Session Token", 1149 ) 1150 session_token_response_key: str = Field( 1151 ..., 1152 description="Name of the key of the session token to be extracted from the response", 1153 examples=["id"], 1154 title="Response Token Response Key", 1155 ) 1156 username: Optional[str] = Field( 1157 None, 1158 description="Username used to authenticate and obtain a session token", 1159 examples=[" {{ config['username'] }}"], 1160 title="Username", 1161 ) 1162 password: Optional[str] = Field( 1163 "", 1164 description="Password used to authenticate and obtain a session token", 1165 examples=["{{ config['password'] }}", ""], 1166 title="Password", 1167 ) 1168 validate_session_url: str = Field( 1169 ..., 1170 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1171 examples=["user/current"], 1172 title="Validate Session Path", 1173 ) 1174 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1192class ValueType(Enum): 1193 string = "string" 1194 number = "number" 1195 integer = "integer" 1196 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1199class WaitTimeFromHeader(BaseModel): 1200 type: Literal["WaitTimeFromHeader"] 1201 header: str = Field( 1202 ..., 1203 description="The name of the response header defining how long to wait before retrying.", 1204 examples=["Retry-After"], 1205 title="Response Header Name", 1206 ) 1207 regex: Optional[str] = Field( 1208 None, 1209 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1210 examples=["([-+]?\\d+)"], 1211 title="Extraction Regex", 1212 ) 1213 max_waiting_time_in_seconds: Optional[float] = Field( 1214 None, 1215 description="Given the value extracted from the header is greater than this value, stop the stream.", 1216 examples=[3600], 1217 title="Max Waiting Time in Seconds", 1218 ) 1219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1222class WaitUntilTimeFromHeader(BaseModel): 1223 type: Literal["WaitUntilTimeFromHeader"] 1224 header: str = Field( 1225 ..., 1226 description="The name of the response header defining how long to wait before retrying.", 1227 examples=["wait_time"], 1228 title="Response Header", 1229 ) 1230 min_wait: Optional[Union[float, str]] = Field( 1231 None, 1232 description="Minimum time to wait before retrying.", 1233 examples=[10, "60"], 1234 title="Minimum Wait Time", 1235 ) 1236 regex: Optional[str] = Field( 1237 None, 1238 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1239 examples=["([-+]?\\d+)"], 1240 title="Extraction Regex", 1241 ) 1242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1245class ComponentMappingDefinition(BaseModel): 1246 type: Literal["ComponentMappingDefinition"] 1247 field_path: List[str] = Field( 1248 ..., 1249 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1250 examples=[ 1251 ["name"], 1252 ["retriever", "requester", "url"], 1253 ["retriever", "requester", "{{ components_values.field }}"], 1254 ["*", "**", "name"], 1255 ], 1256 title="Field Path", 1257 ) 1258 value: str = Field( 1259 ..., 1260 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1261 examples=[ 1262 "{{ components_values['updates'] }}", 1263 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1264 "{{ config['segment_id'] }}", 1265 "{{ stream_slice['parent_id'] }}", 1266 "{{ stream_slice['extra_fields']['name'] }}", 1267 ], 1268 title="Value", 1269 ) 1270 value_type: Optional[ValueType] = Field( 1271 None, 1272 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1273 title="Value Type", 1274 ) 1275 create_or_update: Optional[bool] = Field( 1276 False, 1277 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1278 title="Create or Update", 1279 ) 1280 condition: Optional[str] = Field( 1281 None, 1282 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1283 examples=[ 1284 "{{ components_values.get('cursor_field', None) }}", 1285 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1286 ], 1287 title="Condition", 1288 ) 1289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1292class StreamConfig(BaseModel): 1293 type: Literal["StreamConfig"] 1294 configs_pointer: List[str] = Field( 1295 ..., 1296 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1297 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1298 title="Configs Pointer", 1299 ) 1300 default_values: Optional[List[Dict[str, Any]]] = Field( 1301 None, 1302 description="A list of default values, each matching the structure expected from the parsed component value.", 1303 title="Default Values", 1304 ) 1305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1308class ConfigComponentsResolver(BaseModel): 1309 type: Literal["ConfigComponentsResolver"] 1310 stream_config: Union[List[StreamConfig], StreamConfig] 1311 components_mapping: List[ComponentMappingDefinition] 1312 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1315class StreamParametersDefinition(BaseModel): 1316 type: Literal["StreamParametersDefinition"] 1317 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1318 ..., 1319 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1320 examples=[ 1321 [ 1322 { 1323 "name": "test stream", 1324 "$parameters": {"entity": "test entity"}, 1325 "primary_key": "test key", 1326 } 1327 ] 1328 ], 1329 title="Stream Parameters", 1330 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1333class ParametrizedComponentsResolver(BaseModel): 1334 type: Literal["ParametrizedComponentsResolver"] 1335 stream_parameters: StreamParametersDefinition 1336 components_mapping: List[ComponentMappingDefinition] 1337 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1362class ValidateAdheresToSchema(BaseModel): 1363 type: Literal["ValidateAdheresToSchema"] 1364 base_schema: Union[str, Dict[str, Any]] = Field( 1365 ..., 1366 description="The base JSON schema against which the user-provided schema will be validated.", 1367 examples=[ 1368 "{{ config['report_validation_schema'] }}", 1369 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1370 { 1371 "$schema": "http://json-schema.org/draft-07/schema#", 1372 "title": "Person", 1373 "type": "object", 1374 "properties": { 1375 "name": {"type": "string", "description": "The person's name"}, 1376 "age": { 1377 "type": "integer", 1378 "minimum": 0, 1379 "description": "The person's age", 1380 }, 1381 }, 1382 "required": ["name", "age"], 1383 }, 1384 ], 1385 title="Base JSON Schema", 1386 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1389class CustomValidationStrategy(BaseModel): 1390 class Config: 1391 extra = Extra.allow 1392 1393 type: Literal["CustomValidationStrategy"] 1394 class_name: str = Field( 1395 ..., 1396 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1397 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1398 title="Class Name", 1399 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1402class ConfigRemapField(BaseModel): 1403 type: Literal["ConfigRemapField"] 1404 map: Union[Dict[str, Any], str] = Field( 1405 ..., 1406 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1407 examples=[ 1408 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1409 "{{ config['status_mapping'] }}", 1410 ], 1411 title="Value Mapping", 1412 ) 1413 field_path: List[str] = Field( 1414 ..., 1415 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1416 examples=[ 1417 ["status"], 1418 ["data", "status"], 1419 ["data", "{{ config.name }}", "status"], 1420 ["data", "*", "status"], 1421 ], 1422 title="Field Path", 1423 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1426class ConfigRemoveFields(BaseModel): 1427 type: Literal["ConfigRemoveFields"] 1428 field_pointers: List[List[str]] = Field( 1429 ..., 1430 description="A list of field pointers to be removed from the config.", 1431 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1432 title="Field Pointers", 1433 ) 1434 condition: Optional[str] = Field( 1435 "", 1436 description="Fields will be removed if expression is evaluated to True.", 1437 examples=[ 1438 "{{ config['environemnt'] == 'sandbox' }}", 1439 "{{ property is integer }}", 1440 "{{ property|length > 5 }}", 1441 "{{ property == 'some_string_to_match' }}", 1442 ], 1443 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1446class CustomConfigTransformation(BaseModel): 1447 type: Literal["CustomConfigTransformation"] 1448 class_name: str = Field( 1449 ..., 1450 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1451 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1452 ) 1453 parameters: Optional[Dict[str, Any]] = Field( 1454 None, 1455 alias="$parameters", 1456 description="Additional parameters to be passed to the custom config transformation.", 1457 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1460class AddedFieldDefinition(BaseModel): 1461 type: Literal["AddedFieldDefinition"] 1462 path: List[str] = Field( 1463 ..., 1464 description="List of strings defining the path where to add the value on the record.", 1465 examples=[["segment_id"], ["metadata", "segment_id"]], 1466 title="Path", 1467 ) 1468 value: str = Field( 1469 ..., 1470 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1471 examples=[ 1472 "{{ record['updates'] }}", 1473 "{{ record['MetaData']['LastUpdatedTime'] }}", 1474 "{{ stream_partition['segment_id'] }}", 1475 ], 1476 title="Value", 1477 ) 1478 value_type: Optional[ValueType] = Field( 1479 None, 1480 description="Type of the value. If not specified, the type will be inferred from the value.", 1481 title="Value Type", 1482 ) 1483 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1486class AddFields(BaseModel): 1487 type: Literal["AddFields"] 1488 fields: List[AddedFieldDefinition] = Field( 1489 ..., 1490 description="List of transformations (path and corresponding value) that will be added to the record.", 1491 title="Fields", 1492 ) 1493 condition: Optional[str] = Field( 1494 "", 1495 description="Fields will be added if expression is evaluated to True.", 1496 examples=[ 1497 "{{ property|string == '' }}", 1498 "{{ property is integer }}", 1499 "{{ property|length > 5 }}", 1500 "{{ property == 'some_string_to_match' }}", 1501 ], 1502 ) 1503 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1506class ApiKeyAuthenticator(BaseModel): 1507 type: Literal["ApiKeyAuthenticator"] 1508 api_token: Optional[str] = Field( 1509 None, 1510 description="The API key to inject in the request. Fill it in the user inputs.", 1511 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1512 title="API Key", 1513 ) 1514 header: Optional[str] = Field( 1515 None, 1516 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1517 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1518 title="Header Name", 1519 ) 1520 inject_into: Optional[RequestOption] = Field( 1521 None, 1522 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1523 examples=[ 1524 {"inject_into": "header", "field_name": "Authorization"}, 1525 {"inject_into": "request_parameter", "field_name": "authKey"}, 1526 ], 1527 title="Inject API Key Into Outgoing HTTP Request", 1528 ) 1529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1532class AuthFlow(BaseModel): 1533 auth_flow_type: Optional[AuthFlowType] = Field( 1534 None, description="The type of auth to use", title="Auth flow type" 1535 ) 1536 predicate_key: Optional[List[str]] = Field( 1537 None, 1538 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1539 examples=[["credentials", "auth_type"]], 1540 title="Predicate key", 1541 ) 1542 predicate_value: Optional[str] = Field( 1543 None, 1544 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1545 examples=["Oauth"], 1546 title="Predicate value", 1547 ) 1548 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1551class CheckStream(BaseModel): 1552 type: Literal["CheckStream"] 1553 stream_names: Optional[List[str]] = Field( 1554 None, 1555 description="Names of the streams to try reading from when running a check operation.", 1556 examples=[["users"], ["users", "contacts"]], 1557 title="Stream Names", 1558 ) 1559 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1562class IncrementingCountCursor(BaseModel): 1563 type: Literal["IncrementingCountCursor"] 1564 cursor_field: str = Field( 1565 ..., 1566 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1567 examples=["created_at", "{{ config['record_cursor'] }}"], 1568 title="Cursor Field", 1569 ) 1570 start_value: Optional[Union[str, int]] = Field( 1571 None, 1572 description="The value that determines the earliest record that should be synced.", 1573 examples=[0, "{{ config['start_value'] }}"], 1574 title="Start Value", 1575 ) 1576 start_value_option: Optional[RequestOption] = Field( 1577 None, 1578 description="Optionally configures how the start value will be sent in requests to the source API.", 1579 title="Inject Start Value Into Outgoing HTTP Request", 1580 ) 1581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1584class DatetimeBasedCursor(BaseModel): 1585 type: Literal["DatetimeBasedCursor"] 1586 clamping: Optional[Clamping] = Field( 1587 None, 1588 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1589 title="Date Range Clamping", 1590 ) 1591 cursor_field: str = Field( 1592 ..., 1593 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1594 examples=["created_at", "{{ config['record_cursor'] }}"], 1595 title="Cursor Field", 1596 ) 1597 cursor_datetime_formats: Optional[List[str]] = Field( 1598 None, 1599 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1600 examples=[ 1601 "%Y-%m-%d", 1602 "%Y-%m-%d %H:%M:%S", 1603 "%Y-%m-%dT%H:%M:%S", 1604 "%Y-%m-%dT%H:%M:%SZ", 1605 "%Y-%m-%dT%H:%M:%S%z", 1606 "%Y-%m-%dT%H:%M:%S.%fZ", 1607 "%Y-%m-%dT%H:%M:%S.%f%z", 1608 "%Y-%m-%d %H:%M:%S.%f+00:00", 1609 "%s", 1610 "%ms", 1611 ], 1612 title="Cursor Datetime Formats", 1613 ) 1614 start_datetime: Union[MinMaxDatetime, str] = Field( 1615 ..., 1616 description="The datetime that determines the earliest record that should be synced.", 1617 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1618 title="Start Datetime", 1619 ) 1620 start_time_option: Optional[RequestOption] = Field( 1621 None, 1622 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1623 title="Inject Start Time Into Outgoing HTTP Request", 1624 ) 1625 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1626 None, 1627 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1628 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1629 title="End Datetime", 1630 ) 1631 end_time_option: Optional[RequestOption] = Field( 1632 None, 1633 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1634 title="Inject End Time Into Outgoing HTTP Request", 1635 ) 1636 datetime_format: str = Field( 1637 ..., 1638 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1639 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1640 title="Outgoing Datetime Format", 1641 ) 1642 cursor_granularity: Optional[str] = Field( 1643 None, 1644 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1645 examples=["PT1S"], 1646 title="Cursor Granularity", 1647 ) 1648 is_data_feed: Optional[bool] = Field( 1649 None, 1650 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1651 title="Data Feed API", 1652 ) 1653 is_client_side_incremental: Optional[bool] = Field( 1654 None, 1655 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1656 title="Client-side Incremental Filtering", 1657 ) 1658 is_compare_strictly: Optional[bool] = Field( 1659 False, 1660 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1661 title="Strict Start-End Time Comparison", 1662 ) 1663 global_substream_cursor: Optional[bool] = Field( 1664 False, 1665 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1666 title="Global Substream Cursor", 1667 ) 1668 lookback_window: Optional[str] = Field( 1669 None, 1670 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1671 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1672 title="Lookback Window", 1673 ) 1674 partition_field_end: Optional[str] = Field( 1675 None, 1676 description="Name of the partition start time field.", 1677 examples=["ending_time"], 1678 title="Partition Field End", 1679 ) 1680 partition_field_start: Optional[str] = Field( 1681 None, 1682 description="Name of the partition end time field.", 1683 examples=["starting_time"], 1684 title="Partition Field Start", 1685 ) 1686 step: Optional[str] = Field( 1687 None, 1688 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1689 examples=["P1W", "{{ config['step_increment'] }}"], 1690 title="Step", 1691 ) 1692 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1695class JwtAuthenticator(BaseModel): 1696 type: Literal["JwtAuthenticator"] 1697 secret_key: str = Field( 1698 ..., 1699 description="Secret used to sign the JSON web token.", 1700 examples=["{{ config['secret_key'] }}"], 1701 title="Secret Key", 1702 ) 1703 base64_encode_secret_key: Optional[bool] = Field( 1704 False, 1705 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1706 title="Base64-encode Secret Key", 1707 ) 1708 algorithm: Algorithm = Field( 1709 ..., 1710 description="Algorithm used to sign the JSON web token.", 1711 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1712 title="Algorithm", 1713 ) 1714 token_duration: Optional[int] = Field( 1715 1200, 1716 description="The amount of time in seconds a JWT token can be valid after being issued.", 1717 examples=[1200, 3600], 1718 title="Token Duration", 1719 ) 1720 header_prefix: Optional[str] = Field( 1721 None, 1722 description="The prefix to be used within the Authentication header.", 1723 examples=["Bearer", "Basic"], 1724 title="Header Prefix", 1725 ) 1726 jwt_headers: Optional[JwtHeaders] = Field( 1727 None, 1728 description="JWT headers used when signing JSON web token.", 1729 title="JWT Headers", 1730 ) 1731 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1732 None, 1733 description="Additional headers to be included with the JWT headers object.", 1734 title="Additional JWT Headers", 1735 ) 1736 jwt_payload: Optional[JwtPayload] = Field( 1737 None, 1738 description="JWT Payload used when signing JSON web token.", 1739 title="JWT Payload", 1740 ) 1741 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1742 None, 1743 description="Additional properties to be added to the JWT payload.", 1744 title="Additional JWT Payload Properties", 1745 ) 1746 passphrase: Optional[str] = Field( 1747 None, 1748 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1749 examples=["{{ config['passphrase'] }}"], 1750 title="Passphrase", 1751 ) 1752 request_option: Optional[RequestOption] = Field( 1753 None, 1754 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1755 title="Request Option", 1756 ) 1757 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1760class OAuthAuthenticator(BaseModel): 1761 type: Literal["OAuthAuthenticator"] 1762 client_id_name: Optional[str] = Field( 1763 "client_id", 1764 description="The name of the property to use to refresh the `access_token`.", 1765 examples=["custom_app_id"], 1766 title="Client ID Property Name", 1767 ) 1768 client_id: Optional[str] = Field( 1769 None, 1770 description="The OAuth client ID. Fill it in the user inputs.", 1771 examples=[ 1772 "{{ config['client_id'] }}", 1773 "{{ config['credentials']['client_id }}", 1774 ], 1775 title="Client ID", 1776 ) 1777 client_secret_name: Optional[str] = Field( 1778 "client_secret", 1779 description="The name of the property to use to refresh the `access_token`.", 1780 examples=["custom_app_secret"], 1781 title="Client Secret Property Name", 1782 ) 1783 client_secret: Optional[str] = Field( 1784 None, 1785 description="The OAuth client secret. Fill it in the user inputs.", 1786 examples=[ 1787 "{{ config['client_secret'] }}", 1788 "{{ config['credentials']['client_secret }}", 1789 ], 1790 title="Client Secret", 1791 ) 1792 refresh_token_name: Optional[str] = Field( 1793 "refresh_token", 1794 description="The name of the property to use to refresh the `access_token`.", 1795 examples=["custom_app_refresh_value"], 1796 title="Refresh Token Property Name", 1797 ) 1798 refresh_token: Optional[str] = Field( 1799 None, 1800 description="Credential artifact used to get a new access token.", 1801 examples=[ 1802 "{{ config['refresh_token'] }}", 1803 "{{ config['credentials]['refresh_token'] }}", 1804 ], 1805 title="Refresh Token", 1806 ) 1807 token_refresh_endpoint: Optional[str] = Field( 1808 None, 1809 description="The full URL to call to obtain a new access token.", 1810 examples=["https://connect.squareup.com/oauth2/token"], 1811 title="Token Refresh Endpoint", 1812 ) 1813 access_token_name: Optional[str] = Field( 1814 "access_token", 1815 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 1816 examples=["access_token"], 1817 title="Access Token Property Name", 1818 ) 1819 access_token_value: Optional[str] = Field( 1820 None, 1821 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 1822 examples=["secret_access_token_value"], 1823 title="Access Token Value", 1824 ) 1825 expires_in_name: Optional[str] = Field( 1826 "expires_in", 1827 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 1828 examples=["expires_in"], 1829 title="Token Expiry Property Name", 1830 ) 1831 grant_type_name: Optional[str] = Field( 1832 "grant_type", 1833 description="The name of the property to use to refresh the `access_token`.", 1834 examples=["custom_grant_type"], 1835 title="Grant Type Property Name", 1836 ) 1837 grant_type: Optional[str] = Field( 1838 "refresh_token", 1839 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 1840 examples=["refresh_token", "client_credentials"], 1841 title="Grant Type", 1842 ) 1843 refresh_request_body: Optional[Dict[str, Any]] = Field( 1844 None, 1845 description="Body of the request sent to get a new access token.", 1846 examples=[ 1847 { 1848 "applicationId": "{{ config['application_id'] }}", 1849 "applicationSecret": "{{ config['application_secret'] }}", 1850 "token": "{{ config['token'] }}", 1851 } 1852 ], 1853 title="Refresh Request Body", 1854 ) 1855 refresh_request_headers: Optional[Dict[str, Any]] = Field( 1856 None, 1857 description="Headers of the request sent to get a new access token.", 1858 examples=[ 1859 { 1860 "Authorization": "<AUTH_TOKEN>", 1861 "Content-Type": "application/x-www-form-urlencoded", 1862 } 1863 ], 1864 title="Refresh Request Headers", 1865 ) 1866 scopes: Optional[List[str]] = Field( 1867 None, 1868 description="List of scopes that should be granted to the access token.", 1869 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 1870 title="Scopes", 1871 ) 1872 token_expiry_date: Optional[str] = Field( 1873 None, 1874 description="The access token expiry date.", 1875 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 1876 title="Token Expiry Date", 1877 ) 1878 token_expiry_date_format: Optional[str] = Field( 1879 None, 1880 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 1881 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 1882 title="Token Expiry Date Format", 1883 ) 1884 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 1885 None, 1886 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 1887 title="Refresh Token Updater", 1888 ) 1889 profile_assertion: Optional[JwtAuthenticator] = Field( 1890 None, 1891 description="The authenticator being used to authenticate the client authenticator.", 1892 title="Profile Assertion", 1893 ) 1894 use_profile_assertion: Optional[bool] = Field( 1895 False, 1896 description="Enable using profile assertion as a flow for OAuth authorization.", 1897 title="Use Profile Assertion", 1898 ) 1899 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1902class FixedWindowCallRatePolicy(BaseModel): 1903 class Config: 1904 extra = Extra.allow 1905 1906 type: Literal["FixedWindowCallRatePolicy"] 1907 period: str = Field( 1908 ..., description="The time interval for the rate limit window.", title="Period" 1909 ) 1910 call_limit: int = Field( 1911 ..., 1912 description="The maximum number of calls allowed within the period.", 1913 title="Call Limit", 1914 ) 1915 matchers: List[HttpRequestRegexMatcher] = Field( 1916 ..., 1917 description="List of matchers that define which requests this policy applies to.", 1918 title="Matchers", 1919 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
1922class MovingWindowCallRatePolicy(BaseModel): 1923 class Config: 1924 extra = Extra.allow 1925 1926 type: Literal["MovingWindowCallRatePolicy"] 1927 rates: List[Rate] = Field( 1928 ..., 1929 description="List of rates that define the call limits for different time intervals.", 1930 title="Rates", 1931 ) 1932 matchers: List[HttpRequestRegexMatcher] = Field( 1933 ..., 1934 description="List of matchers that define which requests this policy applies to.", 1935 title="Matchers", 1936 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
1939class UnlimitedCallRatePolicy(BaseModel): 1940 class Config: 1941 extra = Extra.allow 1942 1943 type: Literal["UnlimitedCallRatePolicy"] 1944 matchers: List[HttpRequestRegexMatcher] = Field( 1945 ..., 1946 description="List of matchers that define which requests this policy applies to.", 1947 title="Matchers", 1948 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
1951class DefaultErrorHandler(BaseModel): 1952 type: Literal["DefaultErrorHandler"] 1953 backoff_strategies: Optional[ 1954 List[ 1955 Union[ 1956 ConstantBackoffStrategy, 1957 ExponentialBackoffStrategy, 1958 WaitTimeFromHeader, 1959 WaitUntilTimeFromHeader, 1960 CustomBackoffStrategy, 1961 ] 1962 ] 1963 ] = Field( 1964 None, 1965 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 1966 title="Backoff Strategies", 1967 ) 1968 max_retries: Optional[int] = Field( 1969 5, 1970 description="The maximum number of time to retry a retryable request before giving up and failing.", 1971 examples=[5, 0, 10], 1972 title="Max Retry Count", 1973 ) 1974 response_filters: Optional[List[HttpResponseFilter]] = Field( 1975 None, 1976 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 1977 title="Response Filters", 1978 ) 1979 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
1982class DefaultPaginator(BaseModel): 1983 type: Literal["DefaultPaginator"] 1984 pagination_strategy: Union[ 1985 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 1986 ] = Field( 1987 ..., 1988 description="Strategy defining how records are paginated.", 1989 title="Pagination Strategy", 1990 ) 1991 page_size_option: Optional[RequestOption] = Field( 1992 None, title="Inject Page Size Into Outgoing HTTP Request" 1993 ) 1994 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 1995 None, 1996 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 1997 title="Inject Page Token Into Outgoing HTTP Request", 1998 ) 1999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2002class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2003 type: Literal["ApiKey"] 2004 inject_into: RequestOption = Field( 2005 ..., 2006 description="Configure how the API Key will be sent in requests to the source API.", 2007 examples=[ 2008 {"inject_into": "header", "field_name": "Authorization"}, 2009 {"inject_into": "request_parameter", "field_name": "authKey"}, 2010 ], 2011 title="Inject API Key Into Outgoing HTTP Request", 2012 )
inject_into: RequestOption
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2015class ListPartitionRouter(BaseModel): 2016 type: Literal["ListPartitionRouter"] 2017 cursor_field: str = Field( 2018 ..., 2019 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2020 examples=["section", "{{ config['section_key'] }}"], 2021 title="Current Partition Value Identifier", 2022 ) 2023 values: Union[str, List[str]] = Field( 2024 ..., 2025 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2026 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2027 title="Partition Values", 2028 ) 2029 request_option: Optional[RequestOption] = Field( 2030 None, 2031 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2032 title="Inject Partition Value Into Outgoing HTTP Request", 2033 ) 2034 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
RecordSelector(pydantic.v1.main.BaseModel):
2037class RecordSelector(BaseModel): 2038 type: Literal["RecordSelector"] 2039 extractor: Union[DpathExtractor, CustomRecordExtractor] 2040 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2041 None, 2042 description="Responsible for filtering records to be emitted by the Source.", 2043 title="Record Filter", 2044 ) 2045 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2046 None, 2047 description="Responsible for normalization according to the schema.", 2048 title="Schema Normalization", 2049 ) 2050 transform_before_filtering: Optional[bool] = Field( 2051 None, 2052 description="If true, transformation will be applied before record filtering.", 2053 title="Transform Before Filtering", 2054 ) 2055 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2058class GzipDecoder(BaseModel): 2059 type: Literal["GzipDecoder"] 2060 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2063class RequestBodyGraphQL(BaseModel): 2064 type: Literal["RequestBodyGraphQL"] 2065 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2068class DpathValidator(BaseModel): 2069 type: Literal["DpathValidator"] 2070 field_path: List[str] = Field( 2071 ..., 2072 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2073 examples=[ 2074 ["data"], 2075 ["data", "records"], 2076 ["data", "{{ parameters.name }}"], 2077 ["data", "*", "record"], 2078 ], 2079 title="Field Path", 2080 ) 2081 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2082 ..., 2083 description="The condition that the specified config value will be evaluated against", 2084 title="Validation Strategy", 2085 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2088class PredicateValidator(BaseModel): 2089 type: Literal["PredicateValidator"] 2090 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2091 ..., 2092 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2093 examples=[ 2094 "test-value", 2095 "{{ config['api_version'] }}", 2096 "{{ config['tenant_id'] }}", 2097 123, 2098 ], 2099 title="Value", 2100 ) 2101 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2102 ..., 2103 description="The validation strategy to apply to the value.", 2104 title="Validation Strategy", 2105 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2108class ConfigAddFields(BaseModel): 2109 type: Literal["ConfigAddFields"] 2110 fields: List[AddedFieldDefinition] = Field( 2111 ..., 2112 description="A list of transformations (path and corresponding value) that will be added to the config.", 2113 title="Fields", 2114 ) 2115 condition: Optional[str] = Field( 2116 "", 2117 description="Fields will be added if expression is evaluated to True.", 2118 examples=[ 2119 "{{ config['environemnt'] == 'sandbox' }}", 2120 "{{ property is integer }}", 2121 "{{ property|length > 5 }}", 2122 "{{ property == 'some_string_to_match' }}", 2123 ], 2124 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2127class CompositeErrorHandler(BaseModel): 2128 type: Literal["CompositeErrorHandler"] 2129 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( 2130 ..., 2131 description="List of error handlers to iterate on to determine how to handle a failed response.", 2132 title="Error Handlers", 2133 ) 2134 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2137class HTTPAPIBudget(BaseModel): 2138 class Config: 2139 extra = Extra.allow 2140 2141 type: Literal["HTTPAPIBudget"] 2142 policies: List[ 2143 Union[ 2144 FixedWindowCallRatePolicy, 2145 MovingWindowCallRatePolicy, 2146 UnlimitedCallRatePolicy, 2147 ] 2148 ] = Field( 2149 ..., 2150 description="List of call rate policies that define how many calls are allowed.", 2151 title="Policies", 2152 ) 2153 ratelimit_reset_header: Optional[str] = Field( 2154 "ratelimit-reset", 2155 description="The HTTP response header name that indicates when the rate limit resets.", 2156 title="Rate Limit Reset Header", 2157 ) 2158 ratelimit_remaining_header: Optional[str] = Field( 2159 "ratelimit-remaining", 2160 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2161 title="Rate Limit Remaining Header", 2162 ) 2163 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2164 [429], 2165 description="List of HTTP status codes that indicate a rate limit has been hit.", 2166 title="Status Codes for Rate Limit Hit", 2167 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2170class ZipfileDecoder(BaseModel): 2171 class Config: 2172 extra = Extra.allow 2173 2174 type: Literal["ZipfileDecoder"] 2175 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field( 2176 ..., 2177 description="Parser to parse the decompressed data from the zipfile(s).", 2178 title="Parser", 2179 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
ConfigMigration(pydantic.v1.main.BaseModel):
2182class ConfigMigration(BaseModel): 2183 type: Literal["ConfigMigration"] 2184 description: Optional[str] = Field( 2185 None, description="The description/purpose of the config migration." 2186 ) 2187 transformations: List[ 2188 Union[ 2189 ConfigRemapField, 2190 ConfigAddFields, 2191 ConfigRemoveFields, 2192 CustomConfigTransformation, 2193 ] 2194 ] = Field( 2195 ..., 2196 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2197 title="Transformations", 2198 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2201class ConfigNormalizationRules(BaseModel): 2202 class Config: 2203 extra = Extra.forbid 2204 2205 type: Literal["ConfigNormalizationRules"] 2206 config_migrations: Optional[List[ConfigMigration]] = Field( 2207 [], 2208 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2209 title="Config Migrations", 2210 ) 2211 transformations: Optional[ 2212 List[ 2213 Union[ 2214 ConfigRemapField, 2215 ConfigAddFields, 2216 ConfigRemoveFields, 2217 CustomConfigTransformation, 2218 ] 2219 ] 2220 ] = Field( 2221 [], 2222 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2223 title="Transformations", 2224 ) 2225 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2226 [], 2227 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2228 title="Validations", 2229 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2232class Spec(BaseModel): 2233 type: Literal["Spec"] 2234 connection_specification: Dict[str, Any] = Field( 2235 ..., 2236 description="A connection specification describing how a the connector can be configured.", 2237 title="Connection Specification", 2238 ) 2239 documentation_url: Optional[str] = Field( 2240 None, 2241 description="URL of the connector's documentation page.", 2242 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2243 title="Documentation URL", 2244 ) 2245 advanced_auth: Optional[AuthFlow] = Field( 2246 None, 2247 description="Advanced specification for configuring the authentication flow.", 2248 title="Advanced Auth", 2249 ) 2250 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2251 None, title="Config Normalization Rules" 2252 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2255class DeclarativeSource1(BaseModel): 2256 class Config: 2257 extra = Extra.forbid 2258 2259 type: Literal["DeclarativeSource"] 2260 check: Union[CheckStream, CheckDynamicStream] 2261 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2262 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2263 version: str = Field( 2264 ..., 2265 description="The version of the Airbyte CDK used to build and test the source.", 2266 ) 2267 schemas: Optional[Schemas] = None 2268 definitions: Optional[Dict[str, Any]] = None 2269 spec: Optional[Spec] = None 2270 concurrency_level: Optional[ConcurrencyLevel] = None 2271 api_budget: Optional[HTTPAPIBudget] = None 2272 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2273 None, 2274 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2275 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2276 title="Maximum Concurrent Asynchronous Jobs", 2277 ) 2278 metadata: Optional[Dict[str, Any]] = Field( 2279 None, 2280 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2281 ) 2282 description: Optional[str] = Field( 2283 None, 2284 description="A description of the connector. It will be presented on the Source documentation page.", 2285 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2288class DeclarativeSource2(BaseModel): 2289 class Config: 2290 extra = Extra.forbid 2291 2292 type: Literal["DeclarativeSource"] 2293 check: Union[CheckStream, CheckDynamicStream] 2294 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2295 None 2296 ) 2297 dynamic_streams: List[DynamicDeclarativeStream] 2298 version: str = Field( 2299 ..., 2300 description="The version of the Airbyte CDK used to build and test the source.", 2301 ) 2302 schemas: Optional[Schemas] = None 2303 definitions: Optional[Dict[str, Any]] = None 2304 spec: Optional[Spec] = None 2305 concurrency_level: Optional[ConcurrencyLevel] = None 2306 api_budget: Optional[HTTPAPIBudget] = None 2307 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2308 None, 2309 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2310 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2311 title="Maximum Concurrent Asynchronous Jobs", 2312 ) 2313 metadata: Optional[Dict[str, Any]] = Field( 2314 None, 2315 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2316 ) 2317 description: Optional[str] = Field( 2318 None, 2319 description="A description of the connector. It will be presented on the Source documentation page.", 2320 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2323class DeclarativeSource(BaseModel): 2324 class Config: 2325 extra = Extra.forbid 2326 2327 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2328 ..., 2329 description="An API source that extracts data according to its declarative components.", 2330 title="DeclarativeSource", 2331 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2334class SelectiveAuthenticator(BaseModel): 2335 class Config: 2336 extra = Extra.allow 2337 2338 type: Literal["SelectiveAuthenticator"] 2339 authenticator_selection_path: List[str] = Field( 2340 ..., 2341 description="Path of the field in config with selected authenticator name", 2342 examples=[["auth"], ["auth", "type"]], 2343 title="Authenticator Selection Path", 2344 ) 2345 authenticators: Dict[ 2346 str, 2347 Union[ 2348 ApiKeyAuthenticator, 2349 BasicHttpAuthenticator, 2350 BearerAuthenticator, 2351 OAuthAuthenticator, 2352 JwtAuthenticator, 2353 SessionTokenAuthenticator, 2354 LegacySessionTokenAuthenticator, 2355 CustomAuthenticator, 2356 NoAuth, 2357 ], 2358 ] = Field( 2359 ..., 2360 description="Authenticators to select from.", 2361 examples=[ 2362 { 2363 "authenticators": { 2364 "token": "#/definitions/ApiKeyAuthenticator", 2365 "oauth": "#/definitions/OAuthAuthenticator", 2366 "jwt": "#/definitions/JwtAuthenticator", 2367 } 2368 } 2369 ], 2370 title="Authenticators", 2371 ) 2372 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
authenticators: Dict[str, Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, CustomAuthenticator, NoAuth]]
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2375class ConditionalStreams(BaseModel): 2376 type: Literal["ConditionalStreams"] 2377 condition: str = Field( 2378 ..., 2379 description="Condition that will be evaluated to determine if a set of streams should be available.", 2380 examples=["{{ config['is_sandbox'] }}"], 2381 title="Condition", 2382 ) 2383 streams: List[DeclarativeStream] = Field( 2384 ..., 2385 description="Streams that will be used during an operation based on the condition.", 2386 title="Streams", 2387 ) 2388 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2391class FileUploader(BaseModel): 2392 type: Literal["FileUploader"] 2393 requester: Union[HttpRequester, CustomRequester] = Field( 2394 ..., 2395 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2396 ) 2397 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2398 ..., 2399 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2400 ) 2401 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2402 None, 2403 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2404 ) 2405 filename_extractor: Optional[str] = Field( 2406 None, 2407 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2408 examples=[ 2409 "{{ record.id }}/{{ record.file_name }}/", 2410 "{{ record.id }}_{{ record.file_name }}/", 2411 ], 2412 ) 2413 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2416class DeclarativeStream(BaseModel): 2417 class Config: 2418 extra = Extra.allow 2419 2420 type: Literal["DeclarativeStream"] 2421 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2422 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2423 ..., 2424 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2425 title="Retriever", 2426 ) 2427 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2428 None, 2429 description="Component used to fetch data incrementally based on a time field in the data.", 2430 title="Incremental Sync", 2431 ) 2432 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2433 schema_loader: Optional[ 2434 Union[ 2435 InlineSchemaLoader, 2436 DynamicSchemaLoader, 2437 JsonFileSchemaLoader, 2438 List[ 2439 Union[ 2440 InlineSchemaLoader, 2441 DynamicSchemaLoader, 2442 JsonFileSchemaLoader, 2443 CustomSchemaLoader, 2444 ] 2445 ], 2446 CustomSchemaLoader, 2447 ] 2448 ] = Field( 2449 None, 2450 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2451 title="Schema Loader", 2452 ) 2453 transformations: Optional[ 2454 List[ 2455 Union[ 2456 AddFields, 2457 RemoveFields, 2458 KeysToLower, 2459 KeysToSnakeCase, 2460 FlattenFields, 2461 DpathFlattenFields, 2462 KeysReplace, 2463 CustomTransformation, 2464 ] 2465 ] 2466 ] = Field( 2467 None, 2468 description="A list of transformations to be applied to each output record.", 2469 title="Transformations", 2470 ) 2471 state_migrations: Optional[ 2472 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2473 ] = Field( 2474 [], 2475 description="Array of state migrations to be applied on the input state", 2476 title="State Migrations", 2477 ) 2478 file_uploader: Optional[FileUploader] = Field( 2479 None, 2480 description="(experimental) Describes how to fetch a file", 2481 title="File Uploader", 2482 ) 2483 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2486class SessionTokenAuthenticator(BaseModel): 2487 type: Literal["SessionTokenAuthenticator"] 2488 login_requester: HttpRequester = Field( 2489 ..., 2490 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2491 examples=[ 2492 { 2493 "type": "HttpRequester", 2494 "url_base": "https://my_api.com", 2495 "path": "/login", 2496 "authenticator": { 2497 "type": "BasicHttpAuthenticator", 2498 "username": "{{ config.username }}", 2499 "password": "{{ config.password }}", 2500 }, 2501 } 2502 ], 2503 title="Login Requester", 2504 ) 2505 session_token_path: List[str] = Field( 2506 ..., 2507 description="The path in the response body returned from the login requester to the session token.", 2508 examples=[["access_token"], ["result", "token"]], 2509 title="Session Token Path", 2510 ) 2511 expiration_duration: Optional[str] = Field( 2512 None, 2513 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2514 examples=["PT1H", "P1D"], 2515 title="Expiration Duration", 2516 ) 2517 request_authentication: Union[ 2518 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2519 ] = Field( 2520 ..., 2521 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2522 title="Data Request Authentication", 2523 ) 2524 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2525 None, description="Component used to decode the response.", title="Decoder" 2526 ) 2527 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2530class HttpRequester(BaseModelWithDeprecations): 2531 type: Literal["HttpRequester"] 2532 url_base: Optional[str] = Field( 2533 None, 2534 deprecated=True, 2535 deprecation_message="Use `url` field instead.", 2536 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2537 examples=[ 2538 "https://connect.squareup.com/v2", 2539 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2540 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2541 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2542 ], 2543 title="API Base URL", 2544 ) 2545 url: Optional[str] = Field( 2546 None, 2547 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2548 examples=[ 2549 "https://connect.squareup.com/v2", 2550 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2551 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2552 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2553 ], 2554 title="API Endpoint URL", 2555 ) 2556 path: Optional[str] = Field( 2557 None, 2558 deprecated=True, 2559 deprecation_message="Use `url` field instead.", 2560 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2561 examples=[ 2562 "/products", 2563 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2564 "/trades/{{ config['symbol_id'] }}/history", 2565 ], 2566 title="URL Path", 2567 ) 2568 http_method: Optional[HttpMethod] = Field( 2569 HttpMethod.GET, 2570 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2571 examples=["GET", "POST"], 2572 title="HTTP Method", 2573 ) 2574 authenticator: Optional[ 2575 Union[ 2576 ApiKeyAuthenticator, 2577 BasicHttpAuthenticator, 2578 BearerAuthenticator, 2579 OAuthAuthenticator, 2580 JwtAuthenticator, 2581 SessionTokenAuthenticator, 2582 SelectiveAuthenticator, 2583 CustomAuthenticator, 2584 NoAuth, 2585 LegacySessionTokenAuthenticator, 2586 ] 2587 ] = Field( 2588 None, 2589 description="Authentication method to use for requests sent to the API.", 2590 title="Authenticator", 2591 ) 2592 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2593 None, 2594 deprecated=True, 2595 deprecation_message="Use `query_properties` field instead.", 2596 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2597 title="Fetch Properties from Endpoint", 2598 ) 2599 query_properties: Optional[QueryProperties] = Field( 2600 None, 2601 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2602 title="Query Properties", 2603 ) 2604 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2605 None, 2606 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2607 examples=[ 2608 {"unit": "day"}, 2609 { 2610 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2611 }, 2612 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2613 {"sort_by[asc]": "updated_at"}, 2614 ], 2615 title="Query Parameters", 2616 ) 2617 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2618 None, 2619 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2620 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2621 title="Request Headers", 2622 ) 2623 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2624 None, 2625 deprecated=True, 2626 deprecation_message="Use `request_body` field instead.", 2627 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2628 examples=[ 2629 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2630 ], 2631 title="Request Body Payload (Non-JSON)", 2632 ) 2633 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2634 None, 2635 deprecated=True, 2636 deprecation_message="Use `request_body` field instead.", 2637 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2638 examples=[ 2639 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2640 {"key": "{{ config['value'] }}"}, 2641 {"sort": {"field": "updated_at", "order": "ascending"}}, 2642 ], 2643 title="Request Body JSON Payload", 2644 ) 2645 request_body: Optional[ 2646 Union[ 2647 RequestBodyPlainText, 2648 RequestBodyUrlEncodedForm, 2649 RequestBodyJsonObject, 2650 RequestBodyGraphQL, 2651 ] 2652 ] = Field( 2653 None, 2654 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2655 title="Request Body", 2656 ) 2657 error_handler: Optional[ 2658 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2659 ] = Field( 2660 None, 2661 description="Error handler component that defines how to handle errors.", 2662 title="Error Handler", 2663 ) 2664 use_cache: Optional[bool] = Field( 2665 False, 2666 description="Enables stream requests caching. This field is automatically set by the CDK.", 2667 title="Use Cache", 2668 ) 2669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
2672class DynamicSchemaLoader(BaseModel): 2673 type: Literal["DynamicSchemaLoader"] 2674 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2675 ..., 2676 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2677 title="Retriever", 2678 ) 2679 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2680 None, 2681 description="Responsible for filtering fields to be added to json schema.", 2682 title="Schema Filter", 2683 ) 2684 schema_transformations: Optional[ 2685 List[ 2686 Union[ 2687 AddFields, 2688 RemoveFields, 2689 KeysToLower, 2690 KeysToSnakeCase, 2691 FlattenFields, 2692 DpathFlattenFields, 2693 KeysReplace, 2694 CustomTransformation, 2695 ] 2696 ] 2697 ] = Field( 2698 None, 2699 description="A list of transformations to be applied to the schema.", 2700 title="Schema Transformations", 2701 ) 2702 schema_type_identifier: SchemaTypeIdentifier 2703 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
2706class ParentStreamConfig(BaseModel): 2707 type: Literal["ParentStreamConfig"] 2708 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 2709 ..., description="Reference to the parent stream.", title="Parent Stream" 2710 ) 2711 parent_key: str = Field( 2712 ..., 2713 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 2714 examples=["id", "{{ config['parent_record_id'] }}"], 2715 title="Parent Key", 2716 ) 2717 partition_field: str = Field( 2718 ..., 2719 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 2720 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 2721 title="Current Parent Key Value Identifier", 2722 ) 2723 request_option: Optional[RequestOption] = Field( 2724 None, 2725 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 2726 title="Request Option", 2727 ) 2728 incremental_dependency: Optional[bool] = Field( 2729 False, 2730 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 2731 title="Incremental Dependency", 2732 ) 2733 lazy_read_pointer: Optional[List[str]] = Field( 2734 [], 2735 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 2736 title="Lazy Read Pointer", 2737 ) 2738 extra_fields: Optional[List[List[str]]] = Field( 2739 None, 2740 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 2741 title="Extra Fields", 2742 ) 2743 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
2746class PropertiesFromEndpoint(BaseModel): 2747 type: Literal["PropertiesFromEndpoint"] 2748 property_field_path: List[str] = Field( 2749 ..., 2750 description="Describes the path to the field that should be extracted", 2751 examples=[["name"]], 2752 ) 2753 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 2754 ..., 2755 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 2756 ) 2757 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
2760class QueryProperties(BaseModel): 2761 type: Literal["QueryProperties"] 2762 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 2763 ..., 2764 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 2765 title="Property List", 2766 ) 2767 always_include_properties: Optional[List[str]] = Field( 2768 None, 2769 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 2770 title="Always Include Properties", 2771 ) 2772 property_chunking: Optional[PropertyChunking] = Field( 2773 None, 2774 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 2775 title="Property Chunking", 2776 ) 2777 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
2780class StateDelegatingStream(BaseModel): 2781 type: Literal["StateDelegatingStream"] 2782 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 2783 full_refresh_stream: DeclarativeStream = Field( 2784 ..., 2785 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 2786 title="Full Refresh Stream", 2787 ) 2788 incremental_stream: DeclarativeStream = Field( 2789 ..., 2790 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 2791 title="Incremental Stream", 2792 ) 2793 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
2796class SimpleRetriever(BaseModel): 2797 type: Literal["SimpleRetriever"] 2798 requester: Union[HttpRequester, CustomRequester] = Field( 2799 ..., 2800 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2801 ) 2802 decoder: Optional[ 2803 Union[ 2804 JsonDecoder, 2805 XmlDecoder, 2806 CsvDecoder, 2807 JsonlDecoder, 2808 GzipDecoder, 2809 IterableDecoder, 2810 ZipfileDecoder, 2811 CustomDecoder, 2812 ] 2813 ] = Field( 2814 None, 2815 description="Component decoding the response so records can be extracted.", 2816 title="HTTP Response Format", 2817 ) 2818 record_selector: RecordSelector = Field( 2819 ..., 2820 description="Component that describes how to extract records from a HTTP response.", 2821 ) 2822 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2823 None, 2824 description="Paginator component that describes how to navigate through the API's pages.", 2825 ) 2826 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 2827 False, 2828 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 2829 ) 2830 partition_router: Optional[ 2831 Union[ 2832 SubstreamPartitionRouter, 2833 ListPartitionRouter, 2834 GroupingPartitionRouter, 2835 CustomPartitionRouter, 2836 List[ 2837 Union[ 2838 SubstreamPartitionRouter, 2839 ListPartitionRouter, 2840 GroupingPartitionRouter, 2841 CustomPartitionRouter, 2842 ] 2843 ], 2844 ] 2845 ] = Field( 2846 None, 2847 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 2848 title="Partition Router", 2849 ) 2850 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
2853class AsyncRetriever(BaseModel): 2854 type: Literal["AsyncRetriever"] 2855 record_selector: RecordSelector = Field( 2856 ..., 2857 description="Component that describes how to extract records from a HTTP response.", 2858 ) 2859 status_mapping: AsyncJobStatusMap = Field( 2860 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 2861 ) 2862 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2863 ..., description="Responsible for fetching the actual status of the async job." 2864 ) 2865 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2866 None, 2867 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 2868 ) 2869 download_extractor: Optional[ 2870 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 2871 ] = Field(None, description="Responsible for fetching the records from provided urls.") 2872 creation_requester: Union[HttpRequester, CustomRequester] = Field( 2873 ..., 2874 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 2875 ) 2876 polling_requester: Union[HttpRequester, CustomRequester] = Field( 2877 ..., 2878 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 2879 ) 2880 polling_job_timeout: Optional[Union[int, str]] = Field( 2881 None, 2882 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 2883 ) 2884 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2885 None, 2886 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 2887 ) 2888 download_requester: Union[HttpRequester, CustomRequester] = Field( 2889 ..., 2890 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 2891 ) 2892 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 2893 None, 2894 description="Paginator component that describes how to navigate through the API's pages during download.", 2895 ) 2896 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2897 None, 2898 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 2899 ) 2900 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 2901 None, 2902 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 2903 ) 2904 partition_router: Optional[ 2905 Union[ 2906 ListPartitionRouter, 2907 SubstreamPartitionRouter, 2908 GroupingPartitionRouter, 2909 CustomPartitionRouter, 2910 List[ 2911 Union[ 2912 ListPartitionRouter, 2913 SubstreamPartitionRouter, 2914 GroupingPartitionRouter, 2915 CustomPartitionRouter, 2916 ] 2917 ], 2918 ] 2919 ] = Field( 2920 [], 2921 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 2922 title="Partition Router", 2923 ) 2924 decoder: Optional[ 2925 Union[ 2926 CsvDecoder, 2927 GzipDecoder, 2928 JsonDecoder, 2929 JsonlDecoder, 2930 IterableDecoder, 2931 XmlDecoder, 2932 ZipfileDecoder, 2933 CustomDecoder, 2934 ] 2935 ] = Field( 2936 None, 2937 description="Component decoding the response so records can be extracted.", 2938 title="HTTP Response Format", 2939 ) 2940 download_decoder: Optional[ 2941 Union[ 2942 CsvDecoder, 2943 GzipDecoder, 2944 JsonDecoder, 2945 JsonlDecoder, 2946 IterableDecoder, 2947 XmlDecoder, 2948 ZipfileDecoder, 2949 CustomDecoder, 2950 ] 2951 ] = Field( 2952 None, 2953 description="Component decoding the download response so records can be extracted.", 2954 title="Download HTTP Response Format", 2955 ) 2956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
2959class SubstreamPartitionRouter(BaseModel): 2960 type: Literal["SubstreamPartitionRouter"] 2961 parent_stream_configs: List[ParentStreamConfig] = Field( 2962 ..., 2963 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 2964 title="Parent Stream Configs", 2965 ) 2966 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
2969class GroupingPartitionRouter(BaseModel): 2970 type: Literal["GroupingPartitionRouter"] 2971 group_size: int = Field( 2972 ..., 2973 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 2974 examples=[10, 50], 2975 title="Group Size", 2976 ) 2977 underlying_partition_router: Union[ 2978 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 2979 ] = Field( 2980 ..., 2981 description="The partition router whose output will be grouped. This can be any valid partition router component.", 2982 title="Underlying Partition Router", 2983 ) 2984 deduplicate: Optional[bool] = Field( 2985 True, 2986 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 2987 title="Deduplicate Partitions", 2988 ) 2989 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
2992class HttpComponentsResolver(BaseModel): 2993 type: Literal["HttpComponentsResolver"] 2994 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2995 ..., 2996 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2997 title="Retriever", 2998 ) 2999 components_mapping: List[ComponentMappingDefinition] 3000 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3003class DynamicDeclarativeStream(BaseModel): 3004 type: Literal["DynamicDeclarativeStream"] 3005 name: Optional[str] = Field( 3006 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3007 ) 3008 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3009 ..., description="Reference to the stream template.", title="Stream Template" 3010 ) 3011 components_resolver: Union[ 3012 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3013 ] = Field( 3014 ..., 3015 description="Component resolve and populates stream templates with components values.", 3016 title="Components Resolver", 3017 ) 3018 use_parent_parameters: Optional[bool] = Field( 3019 True, 3020 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3021 title="Use Parent Parameters", 3022 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]