airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class HttpMethod(Enum): 50 GET = "GET" 51 POST = "POST" 52 53 54class QuotaStatusSource(BaseModel): 55 type: Literal["QuotaStatusSource"] 56 url: str = Field( 57 ..., 58 description="The full URL of the quota status endpoint.", 59 examples=[ 60 "https://api.github.com/rate_limit", 61 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 62 ], 63 title="URL", 64 ) 65 http_method: Optional[HttpMethod] = Field( 66 HttpMethod.GET, 67 description="The HTTP method used to fetch the quota status.", 68 title="HTTP Method", 69 ) 70 request_headers: Optional[Dict[str, str]] = Field( 71 None, 72 description="Additional headers to send with the quota status request.", 73 title="Request Headers", 74 ) 75 unavailable_status_codes: Optional[List[int]] = Field( 76 None, 77 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 78 examples=[[404]], 79 title="Unavailable Status Codes", 80 unique_items=True, 81 ) 82 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 83 84 85class DynamicStreamCheckConfig(BaseModel): 86 type: Literal["DynamicStreamCheckConfig"] 87 dynamic_stream_name: str = Field( 88 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 89 ) 90 stream_count: Optional[int] = Field( 91 None, 92 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 93 ge=1, 94 title="Stream Count", 95 ) 96 97 98class CheckDynamicStream(BaseModel): 99 type: Literal["CheckDynamicStream"] 100 stream_count: int = Field( 101 ..., 102 description="Numbers of the streams to try reading from when running a check operation.", 103 title="Stream Count", 104 ) 105 use_check_availability: Optional[bool] = Field( 106 True, 107 description="Enables stream check availability. This field is automatically set by the CDK.", 108 title="Use Check Availability", 109 ) 110 config_overrides: Optional[Dict[str, Any]] = Field( 111 None, 112 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 113 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 114 title="Config Overrides", 115 ) 116 117 118class ConcurrencyLevel(BaseModel): 119 type: Optional[Literal["ConcurrencyLevel"]] = None 120 default_concurrency: Union[int, str] = Field( 121 ..., 122 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 123 examples=[10, "{{ config['num_workers'] or 10 }}"], 124 title="Default Concurrency", 125 ) 126 max_concurrency: Optional[int] = Field( 127 None, 128 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 129 examples=[20, 100], 130 title="Max Concurrency", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class ConstantBackoffStrategy(BaseModel): 136 type: Literal["ConstantBackoffStrategy"] 137 backoff_time_in_seconds: Union[float, str] = Field( 138 ..., 139 description="Backoff time in seconds.", 140 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 141 title="Backoff Time", 142 ) 143 jitter_range_in_seconds: Optional[float] = Field( 144 None, 145 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 146 examples=[15], 147 ge=0, 148 title="Jitter Range", 149 ) 150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 151 152 153class CursorPagination(BaseModel): 154 type: Literal["CursorPagination"] 155 cursor_value: str = Field( 156 ..., 157 description="Value of the cursor defining the next page to fetch.", 158 examples=[ 159 "{{ headers.link.next.cursor }}", 160 "{{ last_record['key'] }}", 161 "{{ response['nextPage'] }}", 162 ], 163 title="Cursor Value", 164 ) 165 page_size: Optional[Union[int, str]] = Field( 166 None, 167 description="The number of records to include in each pages.", 168 examples=[100, "{{ config['page_size'] }}"], 169 title="Page Size", 170 ) 171 stop_condition: Optional[str] = Field( 172 None, 173 description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", 174 examples=[ 175 "{{ response.data.has_more is false }}", 176 "{{ 'next' not in headers['link'] }}", 177 "{{ last_page_size < page_size }}", 178 ], 179 title="Stop Condition", 180 ) 181 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 182 183 184class CustomAuthenticator(BaseModel): 185 class Config: 186 extra = Extra.allow 187 188 type: Literal["CustomAuthenticator"] 189 class_name: str = Field( 190 ..., 191 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 192 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 193 title="Class Name", 194 ) 195 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 196 197 198class CustomBackoffStrategy(BaseModel): 199 class Config: 200 extra = Extra.allow 201 202 type: Literal["CustomBackoffStrategy"] 203 class_name: str = Field( 204 ..., 205 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 206 examples=["source_railz.components.MyCustomBackoffStrategy"], 207 title="Class Name", 208 ) 209 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 210 211 212class CustomErrorHandler(BaseModel): 213 class Config: 214 extra = Extra.allow 215 216 type: Literal["CustomErrorHandler"] 217 class_name: str = Field( 218 ..., 219 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 220 examples=["source_railz.components.MyCustomErrorHandler"], 221 title="Class Name", 222 ) 223 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 224 225 226class CustomPaginationStrategy(BaseModel): 227 class Config: 228 extra = Extra.allow 229 230 type: Literal["CustomPaginationStrategy"] 231 class_name: str = Field( 232 ..., 233 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 234 examples=["source_railz.components.MyCustomPaginationStrategy"], 235 title="Class Name", 236 ) 237 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 238 239 240class CustomRecordExtractor(BaseModel): 241 class Config: 242 extra = Extra.allow 243 244 type: Literal["CustomRecordExtractor"] 245 class_name: str = Field( 246 ..., 247 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 248 examples=["source_railz.components.MyCustomRecordExtractor"], 249 title="Class Name", 250 ) 251 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 252 253 254class CustomRecordFilter(BaseModel): 255 class Config: 256 extra = Extra.allow 257 258 type: Literal["CustomRecordFilter"] 259 class_name: str = Field( 260 ..., 261 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 262 examples=["source_railz.components.MyCustomCustomRecordFilter"], 263 title="Class Name", 264 ) 265 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 266 267 268class CustomRequester(BaseModel): 269 class Config: 270 extra = Extra.allow 271 272 type: Literal["CustomRequester"] 273 class_name: str = Field( 274 ..., 275 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 276 examples=["source_railz.components.MyCustomRecordExtractor"], 277 title="Class Name", 278 ) 279 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 280 281 282class CustomRetriever(BaseModel): 283 class Config: 284 extra = Extra.allow 285 286 type: Literal["CustomRetriever"] 287 class_name: str = Field( 288 ..., 289 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 290 examples=["source_railz.components.MyCustomRetriever"], 291 title="Class Name", 292 ) 293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 294 295 296class CustomPartitionRouter(BaseModel): 297 class Config: 298 extra = Extra.allow 299 300 type: Literal["CustomPartitionRouter"] 301 class_name: str = Field( 302 ..., 303 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 304 examples=["source_railz.components.MyCustomPartitionRouter"], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 308 309 310class CustomSchemaLoader(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomSchemaLoader"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomSchemaLoader"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 322 323 324class CustomSchemaNormalization(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomSchemaNormalization"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 332 examples=[ 333 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 334 ], 335 title="Class Name", 336 ) 337 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 338 339 340class CustomStateMigration(BaseModel): 341 class Config: 342 extra = Extra.allow 343 344 type: Literal["CustomStateMigration"] 345 class_name: str = Field( 346 ..., 347 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 348 examples=["source_railz.components.MyCustomStateMigration"], 349 title="Class Name", 350 ) 351 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 352 353 354class CustomTransformation(BaseModel): 355 class Config: 356 extra = Extra.allow 357 358 type: Literal["CustomTransformation"] 359 class_name: str = Field( 360 ..., 361 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 362 examples=["source_railz.components.MyCustomTransformation"], 363 title="Class Name", 364 ) 365 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 366 367 368class LegacyToPerPartitionStateMigration(BaseModel): 369 class Config: 370 extra = Extra.allow 371 372 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 373 374 375class Clamping(BaseModel): 376 target: str = Field( 377 ..., 378 description="The period of time that datetime windows will be clamped by", 379 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 380 title="Target", 381 ) 382 target_details: Optional[Dict[str, Any]] = None 383 384 385class Algorithm(Enum): 386 HS256 = "HS256" 387 HS384 = "HS384" 388 HS512 = "HS512" 389 ES256 = "ES256" 390 ES256K = "ES256K" 391 ES384 = "ES384" 392 ES512 = "ES512" 393 RS256 = "RS256" 394 RS384 = "RS384" 395 RS512 = "RS512" 396 PS256 = "PS256" 397 PS384 = "PS384" 398 PS512 = "PS512" 399 EdDSA = "EdDSA" 400 401 402class JwtHeaders(BaseModel): 403 class Config: 404 extra = Extra.forbid 405 406 kid: Optional[str] = Field( 407 None, 408 description="Private key ID for user account.", 409 examples=["{{ config['kid'] }}"], 410 title="Key Identifier", 411 ) 412 typ: Optional[str] = Field( 413 "JWT", 414 description="The media type of the complete JWT.", 415 examples=["JWT"], 416 title="Type", 417 ) 418 cty: Optional[str] = Field( 419 None, 420 description="Content type of JWT header.", 421 examples=["JWT"], 422 title="Content Type", 423 ) 424 425 426class JwtPayload(BaseModel): 427 class Config: 428 extra = Extra.forbid 429 430 iss: Optional[str] = Field( 431 None, 432 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 433 examples=["{{ config['iss'] }}"], 434 title="Issuer", 435 ) 436 sub: Optional[str] = Field( 437 None, 438 description="The subject of the JWT. Commonly defined by the API.", 439 title="Subject", 440 ) 441 aud: Optional[str] = Field( 442 None, 443 description="The recipient that the JWT is intended for. Commonly defined by the API.", 444 examples=["appstoreconnect-v1"], 445 title="Audience", 446 ) 447 448 449class RefreshTokenUpdater(BaseModel): 450 refresh_token_name: Optional[str] = Field( 451 "refresh_token", 452 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 453 examples=["refresh_token"], 454 title="Refresh Token Property Name", 455 ) 456 access_token_config_path: Optional[List[str]] = Field( 457 ["credentials", "access_token"], 458 description="Config path to the access token. Make sure the field actually exists in the config.", 459 examples=[["credentials", "access_token"], ["access_token"]], 460 title="Config Path To Access Token", 461 ) 462 refresh_token_config_path: Optional[List[str]] = Field( 463 ["credentials", "refresh_token"], 464 description="Config path to the access token. Make sure the field actually exists in the config.", 465 examples=[["credentials", "refresh_token"], ["refresh_token"]], 466 title="Config Path To Refresh Token", 467 ) 468 token_expiry_date_config_path: Optional[List[str]] = Field( 469 ["credentials", "token_expiry_date"], 470 description="Config path to the expiry date. Make sure actually exists in the config.", 471 examples=[["credentials", "token_expiry_date"]], 472 title="Config Path To Expiry Date", 473 ) 474 refresh_token_error_status_codes: Optional[List[int]] = Field( 475 [], 476 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 477 examples=[[400, 500]], 478 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 479 ) 480 refresh_token_error_key: Optional[str] = Field( 481 "", 482 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 483 examples=["error"], 484 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 485 ) 486 refresh_token_error_values: Optional[List[str]] = Field( 487 [], 488 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 489 examples=[["invalid_grant", "invalid_permissions"]], 490 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 491 ) 492 493 494class Rate(BaseModel): 495 class Config: 496 extra = Extra.allow 497 498 limit: Union[int, str] = Field( 499 ..., 500 description="The maximum number of calls allowed within the interval.", 501 title="Limit", 502 ) 503 interval: str = Field( 504 ..., 505 description="The time interval for the rate limit.", 506 examples=["PT1H", "P1D"], 507 title="Interval", 508 ) 509 510 511class HttpRequestRegexMatcher(BaseModel): 512 class Config: 513 extra = Extra.allow 514 515 method: Optional[str] = Field( 516 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 517 ) 518 url_base: Optional[str] = Field( 519 None, 520 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 521 title="URL Base", 522 ) 523 url_path_pattern: Optional[str] = Field( 524 None, 525 description="A regular expression pattern to match the URL path.", 526 title="URL Path Pattern", 527 ) 528 params: Optional[Dict[str, Any]] = Field( 529 None, description="The query parameters to match.", title="Parameters" 530 ) 531 headers: Optional[Dict[str, Any]] = Field( 532 None, description="The headers to match.", title="Headers" 533 ) 534 weight: Optional[Union[int, str]] = Field( 535 None, 536 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", 537 title="Weight", 538 ) 539 540 541class CombineMode(Enum): 542 union = "union" 543 first_match = "first_match" 544 545 546class ResponseToFileExtractor(BaseModel): 547 type: Literal["ResponseToFileExtractor"] 548 preserve_na_values: Optional[bool] = Field( 549 False, 550 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 551 title="Preserve NA Values", 552 ) 553 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 554 555 556class OnNoRecords(Enum): 557 skip = "skip" 558 emit_parent = "emit_parent" 559 560 561class ParentFieldPath(BaseModel): 562 type: Literal["ParentFieldPath"] 563 parent_path: List[str] = Field( 564 ..., 565 description="Path to the value on the record being expanded.", 566 examples=[["url"], ["id"], ["repository", "name"]], 567 title="Parent Path", 568 ) 569 record_path: List[str] = Field( 570 ..., 571 description="Path on the expanded item to write the value to. An existing value is overwritten. Intermediate objects are created as needed, so a path may not pass through a value the item already holds as a scalar, and a numeric path segment creates an array rather than an object - the same behavior as `AddFields`.", 572 examples=[["pull_request_url"], ["comment_id"], ["parent", "id"]], 573 title="Record Path", 574 ) 575 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 576 577 578class ExponentialBackoffStrategy(BaseModel): 579 type: Literal["ExponentialBackoffStrategy"] 580 factor: Optional[Union[float, str]] = Field( 581 5, 582 description="Multiplicative constant applied on each retry.", 583 examples=[5, 5.5, "10"], 584 title="Factor", 585 ) 586 jitter_range_in_seconds: Optional[float] = Field( 587 None, 588 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 589 examples=[2], 590 ge=0, 591 title="Jitter Range", 592 ) 593 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 594 595 596class GroupByKeyMergeStrategy(BaseModel): 597 type: Literal["GroupByKeyMergeStrategy"] 598 key: Union[str, List[str]] = Field( 599 ..., 600 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 601 examples=["id", ["parent_id", "end_date"]], 602 title="Key", 603 ) 604 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 605 606 607class SessionTokenRequestBearerAuthenticator(BaseModel): 608 type: Literal["Bearer"] 609 610 611class Action(Enum): 612 SUCCESS = "SUCCESS" 613 FAIL = "FAIL" 614 RETRY = "RETRY" 615 IGNORE = "IGNORE" 616 RESET_PAGINATION = "RESET_PAGINATION" 617 RATE_LIMITED = "RATE_LIMITED" 618 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 619 REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" 620 SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW" 621 622 623class FailureType(Enum): 624 system_error = "system_error" 625 config_error = "config_error" 626 transient_error = "transient_error" 627 628 629class HttpResponseFilter(BaseModel): 630 type: Literal["HttpResponseFilter"] 631 action: Optional[Action] = Field( 632 None, 633 description="Action to execute if a response matches the filter.", 634 examples=[ 635 "SUCCESS", 636 "FAIL", 637 "RETRY", 638 "IGNORE", 639 "RESET_PAGINATION", 640 "RATE_LIMITED", 641 "REFRESH_TOKEN_THEN_RETRY", 642 "REDUCE_PAGE_SIZE", 643 "SPLIT_REQUEST_WINDOW", 644 ], 645 title="Action", 646 ) 647 failure_type: Optional[FailureType] = Field( 648 None, 649 description="Failure type of traced exception if a response matches the filter.", 650 examples=["system_error", "config_error", "transient_error"], 651 title="Failure Type", 652 ) 653 error_message: Optional[str] = Field( 654 None, 655 description="Error Message to display if the response matches the filter.", 656 title="Error Message", 657 ) 658 error_message_contains: Optional[str] = Field( 659 None, 660 description="Match the response if its error message contains the substring.", 661 example=["This API operation is not enabled for this site"], 662 title="Error Message Substring", 663 ) 664 http_codes: Optional[List[int]] = Field( 665 None, 666 description="Match the response if its HTTP code is included in this list.", 667 examples=[[420, 429], [500]], 668 title="HTTP Codes", 669 unique_items=True, 670 ) 671 predicate: Optional[str] = Field( 672 None, 673 description="Match the response if the predicate evaluates to true.", 674 examples=[ 675 "{{ 'Too much requests' in response }}", 676 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 677 ], 678 title="Predicate", 679 ) 680 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 681 682 683class ComplexFieldType(BaseModel): 684 field_type: str 685 items: Optional[Union[str, ComplexFieldType]] = None 686 687 688class TypesMap(BaseModel): 689 target_type: Union[str, List[str], ComplexFieldType] 690 current_type: Union[str, List[str]] 691 condition: Optional[str] = None 692 693 694class SchemaTypeIdentifier(BaseModel): 695 type: Optional[Literal["SchemaTypeIdentifier"]] = None 696 schema_pointer: Optional[List[str]] = Field( 697 [], 698 description="List of nested fields defining the schema field path to extract. Defaults to [].", 699 title="Schema Path", 700 ) 701 key_pointer: List[str] = Field( 702 ..., 703 description="List of potentially nested fields describing the full path of the field key to extract.", 704 title="Key Path", 705 ) 706 type_pointer: Optional[List[str]] = Field( 707 None, 708 description="List of potentially nested fields describing the full path of the field type to extract.", 709 title="Type Path", 710 ) 711 types_mapping: Optional[List[TypesMap]] = None 712 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 713 714 715class InlineSchemaLoader(BaseModel): 716 type: Literal["InlineSchemaLoader"] 717 schema_: Optional[Dict[str, Any]] = Field( 718 None, 719 alias="schema", 720 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 721 title="Schema", 722 ) 723 724 725class JsonFileSchemaLoader(BaseModel): 726 type: Literal["JsonFileSchemaLoader"] 727 file_path: Optional[str] = Field( 728 None, 729 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 730 example=["./schemas/users.json"], 731 title="File Path", 732 ) 733 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 734 735 736class JsonDecoder(BaseModel): 737 type: Literal["JsonDecoder"] 738 739 740class JsonItemsDecoder(BaseModel): 741 type: Literal["JsonItemsDecoder"] 742 items_path: str = Field( 743 ..., 744 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", 745 examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], 746 title="Items Path", 747 ) 748 encoding: Optional[str] = Field( 749 "utf-8", 750 description="Text encoding used to decode the streamed bytes before JSON parsing.", 751 title="Encoding", 752 ) 753 754 755class JsonlDecoder(BaseModel): 756 type: Literal["JsonlDecoder"] 757 758 759class KeysToLower(BaseModel): 760 type: Literal["KeysToLower"] 761 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 762 763 764class KeysToSnakeCase(BaseModel): 765 type: Literal["KeysToSnakeCase"] 766 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 767 768 769class FlattenFields(BaseModel): 770 type: Literal["FlattenFields"] 771 flatten_lists: Optional[bool] = Field( 772 True, 773 description="Whether to flatten lists or leave it as is. Default is True.", 774 title="Flatten Lists", 775 ) 776 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 777 778 779class KeyTransformation(BaseModel): 780 type: Literal["KeyTransformation"] 781 prefix: Optional[str] = Field( 782 None, 783 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 784 examples=["flattened_"], 785 title="Key Prefix", 786 ) 787 suffix: Optional[str] = Field( 788 None, 789 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 790 examples=["_flattened"], 791 title="Key Suffix", 792 ) 793 794 795class DpathFlattenFields(BaseModel): 796 type: Literal["DpathFlattenFields"] 797 field_path: List[str] = Field( 798 ..., 799 description="A path to field that needs to be flattened.", 800 examples=[["data"], ["data", "*", "field"]], 801 title="Field Path", 802 ) 803 delete_origin_value: Optional[bool] = Field( 804 None, 805 description="Whether to delete the origin value or keep it. Default is False.", 806 title="Delete Origin Value", 807 ) 808 replace_record: Optional[bool] = Field( 809 None, 810 description="Whether to replace the origin record or not. Default is False.", 811 title="Replace Origin Record", 812 ) 813 key_transformation: Optional[KeyTransformation] = Field( 814 None, 815 description="Transformation for object keys. If not provided, original key will be used.", 816 title="Key transformation", 817 ) 818 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 819 820 821class KeysReplace(BaseModel): 822 type: Literal["KeysReplace"] 823 old: str = Field( 824 ..., 825 description="Old value to replace.", 826 examples=[ 827 " ", 828 "{{ record.id }}", 829 "{{ config['id'] }}", 830 "{{ stream_slice['id'] }}", 831 ], 832 title="Old value", 833 ) 834 new: str = Field( 835 ..., 836 description="New value to set.", 837 examples=[ 838 "_", 839 "{{ record.id }}", 840 "{{ config['id'] }}", 841 "{{ stream_slice['id'] }}", 842 ], 843 title="New value", 844 ) 845 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 846 847 848class IterableDecoder(BaseModel): 849 type: Literal["IterableDecoder"] 850 851 852class XmlDecoder(BaseModel): 853 type: Literal["XmlDecoder"] 854 855 856class CustomDecoder(BaseModel): 857 class Config: 858 extra = Extra.allow 859 860 type: Literal["CustomDecoder"] 861 class_name: str = Field( 862 ..., 863 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 864 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 865 title="Class Name", 866 ) 867 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 868 869 870class MinMaxDatetime(BaseModel): 871 type: Literal["MinMaxDatetime"] 872 datetime: str = Field( 873 ..., 874 description="Datetime value.", 875 examples=[ 876 "2021-01-01", 877 "2021-01-01T00:00:00Z", 878 "{{ config['start_time'] }}", 879 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 880 ], 881 title="Datetime", 882 ) 883 datetime_format: Optional[str] = Field( 884 "", 885 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 886 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 887 title="Datetime Format", 888 ) 889 max_datetime: Optional[str] = Field( 890 None, 891 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 892 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 893 title="Max Datetime", 894 ) 895 min_datetime: Optional[str] = Field( 896 None, 897 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 898 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 899 title="Min Datetime", 900 ) 901 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 902 903 904class NoAuth(BaseModel): 905 type: Literal["NoAuth"] 906 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 907 908 909class NoPagination(BaseModel): 910 type: Literal["NoPagination"] 911 912 913class Scope(BaseModel): 914 class Config: 915 extra = Extra.allow 916 917 scope: str = Field(..., description="The OAuth scope string to request from the provider.") 918 919 920class OptionalScope(BaseModel): 921 class Config: 922 extra = Extra.allow 923 924 scope: str = Field(..., description="The OAuth scope string to request from the provider.") 925 926 927class ScopesJoinStrategy(Enum): 928 space = "space" 929 comma = "comma" 930 plus = "plus" 931 932 933class State(BaseModel): 934 class Config: 935 extra = Extra.allow 936 937 min: int 938 max: int 939 940 941class OauthConnectorInputSpecification(BaseModel): 942 class Config: 943 extra = Extra.allow 944 945 consent_url: str = Field( 946 ..., 947 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 948 examples=[ 949 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 950 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 951 ], 952 title="Consent URL", 953 ) 954 scope: Optional[str] = Field( 955 None, 956 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 957 examples=["user:read user:read_orders workspaces:read"], 958 title="Scopes", 959 ) 960 scopes: Optional[List[Scope]] = Field( 961 None, 962 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 963 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 964 title="Scopes", 965 ) 966 optional_scopes: Optional[List[OptionalScope]] = Field( 967 None, 968 description="Optional OAuth scope objects that may or may not be granted.", 969 examples=[[{"scope": "admin:read"}]], 970 title="Optional Scopes", 971 ) 972 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 973 ScopesJoinStrategy.space, 974 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 975 title="Scopes Join Strategy", 976 ) 977 access_token_url: str = Field( 978 ..., 979 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 980 examples=[ 981 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 982 ], 983 title="Access Token URL", 984 ) 985 access_token_headers: Optional[Dict[str, Any]] = Field( 986 None, 987 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 988 examples=[ 989 { 990 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 991 } 992 ], 993 title="Access Token Headers", 994 ) 995 access_token_params: Optional[Dict[str, Any]] = Field( 996 None, 997 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 998 examples=[ 999 { 1000 "{{ auth_code_key }}": "{{ auth_code_value }}", 1001 "{{ client_id_key }}": "{{ client_id_value }}", 1002 "{{ client_secret_key }}": "{{ client_secret_value }}", 1003 } 1004 ], 1005 title="Access Token Query Params (Json Encoded)", 1006 ) 1007 extract_output: Optional[List[str]] = Field( 1008 None, 1009 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1010 examples=[["access_token", "refresh_token", "other_field"]], 1011 title="Extract Output", 1012 ) 1013 state: Optional[State] = Field( 1014 None, 1015 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1016 examples=[{"min": 7, "max": 128}], 1017 title="Configurable State Query Param", 1018 ) 1019 client_id_key: Optional[str] = Field( 1020 None, 1021 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1022 examples=["my_custom_client_id_key_name"], 1023 title="Client ID Key Override", 1024 ) 1025 client_secret_key: Optional[str] = Field( 1026 None, 1027 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1028 examples=["my_custom_client_secret_key_name"], 1029 title="Client Secret Key Override", 1030 ) 1031 scope_key: Optional[str] = Field( 1032 None, 1033 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1034 examples=["my_custom_scope_key_key_name"], 1035 title="Scopes Key Override", 1036 ) 1037 state_key: Optional[str] = Field( 1038 None, 1039 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1040 examples=["my_custom_state_key_key_name"], 1041 title="State Key Override", 1042 ) 1043 auth_code_key: Optional[str] = Field( 1044 None, 1045 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1046 examples=["my_custom_auth_code_key_name"], 1047 title="Auth Code Key Override", 1048 ) 1049 redirect_uri_key: Optional[str] = Field( 1050 None, 1051 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1052 examples=["my_custom_redirect_uri_key_name"], 1053 title="Redirect URI Key Override", 1054 ) 1055 1056 1057class OAuthConfigSpecification(BaseModel): 1058 class Config: 1059 extra = Extra.allow 1060 1061 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1062 None, 1063 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1064 examples=[ 1065 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1066 { 1067 "app_id": { 1068 "type": "string", 1069 "path_in_connector_config": ["info", "app_id"], 1070 } 1071 }, 1072 ], 1073 title="OAuth user input", 1074 ) 1075 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1076 None, 1077 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1078 title="DeclarativeOAuth Connector Specification", 1079 ) 1080 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1081 None, 1082 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1083 examples=[ 1084 { 1085 "refresh_token": { 1086 "type": "string,", 1087 "path_in_connector_config": ["credentials", "refresh_token"], 1088 } 1089 } 1090 ], 1091 title="OAuth output specification", 1092 ) 1093 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1094 None, 1095 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1096 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1097 title="OAuth input specification", 1098 ) 1099 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1100 None, 1101 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1102 examples=[ 1103 { 1104 "client_id": { 1105 "type": "string,", 1106 "path_in_connector_config": ["credentials", "client_id"], 1107 }, 1108 "client_secret": { 1109 "type": "string,", 1110 "path_in_connector_config": ["credentials", "client_secret"], 1111 }, 1112 } 1113 ], 1114 title="OAuth server output specification", 1115 ) 1116 1117 1118class OffsetIncrement(BaseModel): 1119 type: Literal["OffsetIncrement"] 1120 page_size: Optional[Union[int, str]] = Field( 1121 None, 1122 description="The number of records to include in each pages.", 1123 examples=[100, "{{ config['page_size'] }}"], 1124 title="Limit", 1125 ) 1126 inject_on_first_request: Optional[bool] = Field( 1127 False, 1128 description="Using the `offset` with value `0` during the first request", 1129 title="Inject Offset on First Request", 1130 ) 1131 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1132 1133 1134class PageIncrement(BaseModel): 1135 type: Literal["PageIncrement"] 1136 page_size: Optional[Union[int, str]] = Field( 1137 None, 1138 description="The number of records to include in each pages.", 1139 examples=[100, "100", "{{ config['page_size'] }}"], 1140 title="Page Size", 1141 ) 1142 start_from_page: Optional[int] = Field( 1143 0, 1144 description="Index of the first page to request.", 1145 examples=[0, 1], 1146 title="Start From Page", 1147 ) 1148 inject_on_first_request: Optional[bool] = Field( 1149 False, 1150 description="Using the `page number` with value defined by `start_from_page` during the first request", 1151 title="Inject Page Number on First Request", 1152 ) 1153 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1154 1155 1156class PrimaryKey(BaseModel): 1157 __root__: Union[str, List[str], List[List[str]]] = Field( 1158 ..., 1159 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1160 examples=["id", ["code", "type"]], 1161 title="Primary Key", 1162 ) 1163 1164 1165class PropertyLimitType(Enum): 1166 characters = "characters" 1167 property_count = "property_count" 1168 1169 1170class PropertyChunking(BaseModel): 1171 type: Literal["PropertyChunking"] 1172 property_limit_type: PropertyLimitType = Field( 1173 ..., 1174 description="The type used to determine the maximum number of properties per chunk", 1175 title="Property Limit Type", 1176 ) 1177 property_limit: Optional[int] = Field( 1178 None, 1179 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1180 title="Property Limit", 1181 ) 1182 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1183 None, 1184 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1185 title="Record Merge Strategy", 1186 ) 1187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1188 1189 1190class RecordFilter(BaseModel): 1191 type: Literal["RecordFilter"] 1192 condition: Optional[str] = Field( 1193 "", 1194 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1195 examples=[ 1196 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1197 "{{ record.status in ['active', 'expired'] }}", 1198 ], 1199 title="Condition", 1200 ) 1201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1202 1203 1204class SchemaNormalization(Enum): 1205 Default = "Default" 1206 None_ = "None" 1207 1208 1209class RemoveFields(BaseModel): 1210 type: Literal["RemoveFields"] 1211 condition: Optional[str] = Field( 1212 "", 1213 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1214 examples=[ 1215 "{{ property|string == '' }}", 1216 "{{ property is integer }}", 1217 "{{ property|length > 5 }}", 1218 "{{ property == 'some_string_to_match' }}", 1219 ], 1220 ) 1221 field_pointers: List[List[str]] = Field( 1222 ..., 1223 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1224 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1225 title="Field Paths", 1226 ) 1227 1228 1229class RequestPath(BaseModel): 1230 type: Literal["RequestPath"] 1231 1232 1233class InjectInto(Enum): 1234 request_parameter = "request_parameter" 1235 header = "header" 1236 body_data = "body_data" 1237 body_json = "body_json" 1238 1239 1240class RequestOption(BaseModel): 1241 type: Literal["RequestOption"] 1242 inject_into: InjectInto = Field( 1243 ..., 1244 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1245 examples=["request_parameter", "header", "body_data", "body_json"], 1246 title="Inject Into", 1247 ) 1248 field_name: Optional[str] = Field( 1249 None, 1250 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1251 examples=["segment_id"], 1252 title="Field Name", 1253 ) 1254 field_path: Optional[List[str]] = Field( 1255 None, 1256 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1257 examples=[["data", "viewer", "id"]], 1258 title="Field Path", 1259 ) 1260 1261 1262class Schemas(BaseModel): 1263 pass 1264 1265 class Config: 1266 extra = Extra.allow 1267 1268 1269class LegacySessionTokenAuthenticator(BaseModel): 1270 type: Literal["LegacySessionTokenAuthenticator"] 1271 header: str = Field( 1272 ..., 1273 description="The name of the session token header that will be injected in the request", 1274 examples=["X-Session"], 1275 title="Session Request Header", 1276 ) 1277 login_url: str = Field( 1278 ..., 1279 description="Path of the login URL (do not include the base URL)", 1280 examples=["session"], 1281 title="Login Path", 1282 ) 1283 session_token: Optional[str] = Field( 1284 None, 1285 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1286 example=["{{ config['session_token'] }}"], 1287 title="Session Token", 1288 ) 1289 session_token_response_key: str = Field( 1290 ..., 1291 description="Name of the key of the session token to be extracted from the response", 1292 examples=["id"], 1293 title="Response Token Response Key", 1294 ) 1295 username: Optional[str] = Field( 1296 None, 1297 description="Username used to authenticate and obtain a session token", 1298 examples=[" {{ config['username'] }}"], 1299 title="Username", 1300 ) 1301 password: Optional[str] = Field( 1302 "", 1303 description="Password used to authenticate and obtain a session token", 1304 examples=["{{ config['password'] }}", ""], 1305 title="Password", 1306 ) 1307 validate_session_url: str = Field( 1308 ..., 1309 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1310 examples=["user/current"], 1311 title="Validate Session Path", 1312 ) 1313 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1314 1315 1316class Action1(Enum): 1317 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1318 RESET = "RESET" 1319 1320 1321class PaginationResetLimits(BaseModel): 1322 type: Literal["PaginationResetLimits"] 1323 number_of_records: Optional[int] = None 1324 1325 1326class ResetPolicy(Enum): 1327 NEVER = "NEVER" 1328 AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" 1329 1330 1331class PageSizeReduction(BaseModel): 1332 type: Literal["PageSizeReduction"] 1333 reduction_factor: Optional[float] = Field( 1334 2, 1335 description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", 1336 examples=[2, 4], 1337 gt=1, 1338 title="Reduction Factor", 1339 ) 1340 minimum_page_size: Optional[int] = Field( 1341 1, 1342 description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", 1343 examples=[1, 10], 1344 ge=1, 1345 title="Minimum Page Size", 1346 ) 1347 max_attempts: Optional[int] = Field( 1348 5, 1349 description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", 1350 examples=[5, 10], 1351 ge=1, 1352 title="Maximum Reduction Attempts", 1353 ) 1354 backoff_seconds: Optional[float] = Field( 1355 0.5, 1356 description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', 1357 examples=[0.5, 5], 1358 ge=0, 1359 title="Backoff Seconds", 1360 ) 1361 retries_at_minimum_page_size: Optional[int] = Field( 1362 0, 1363 description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", 1364 examples=[0, 3], 1365 ge=0, 1366 title="Retries At Minimum Page Size", 1367 ) 1368 failure_message: Optional[str] = Field( 1369 None, 1370 description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", 1371 examples=[ 1372 "Narrow the sync down by selecting fewer fields on this stream.", 1373 "Set a more recent start date so that each page covers less data.", 1374 ], 1375 title="Failure Message", 1376 ) 1377 reset_policy: Optional[ResetPolicy] = Field( 1378 ResetPolicy.NEVER, 1379 description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", 1380 title="Reset Policy", 1381 ) 1382 1383 1384class RequestWindowSplitting(BaseModel): 1385 type: Literal["RequestWindowSplitting"] 1386 failure_message: Optional[str] = Field( 1387 None, 1388 description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", 1389 examples=[ 1390 "Narrow the sync down by selecting fewer fields on this stream.", 1391 "Lower time_window so that each request covers less data.", 1392 ], 1393 title="Failure Message", 1394 ) 1395 min_split_window: Optional[str] = Field( 1396 None, 1397 description="Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or below this size is not split further, though the result can end up a bit smaller than this, not larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", 1398 examples=["P1D", "PT1H"], 1399 title="Minimum Split Window", 1400 ) 1401 1402 1403class CsvDecoder(BaseModel): 1404 type: Literal["CsvDecoder"] 1405 encoding: Optional[str] = "utf-8" 1406 delimiter: Optional[str] = "," 1407 set_values_to_none: Optional[List[str]] = None 1408 1409 1410class AsyncJobStatusMap(BaseModel): 1411 type: Optional[Literal["AsyncJobStatusMap"]] = None 1412 running: List[str] 1413 completed: List[str] 1414 failed: List[str] 1415 timeout: List[str] 1416 skipped: Optional[List[str]] = Field( 1417 None, 1418 description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", 1419 ) 1420 1421 1422class BlockSimultaneousSyncsAction(BaseModel): 1423 type: Literal["BlockSimultaneousSyncsAction"] 1424 1425 1426class ValueType(Enum): 1427 string = "string" 1428 number = "number" 1429 integer = "integer" 1430 boolean = "boolean" 1431 1432 1433class WaitTimeFromHeader(BaseModel): 1434 type: Literal["WaitTimeFromHeader"] 1435 header: str = Field( 1436 ..., 1437 description="The name of the response header defining how long to wait before retrying.", 1438 examples=["Retry-After"], 1439 title="Response Header Name", 1440 ) 1441 regex: Optional[str] = Field( 1442 None, 1443 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1444 examples=["([-+]?\\d+)"], 1445 title="Extraction Regex", 1446 ) 1447 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1448 None, 1449 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1450 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1451 title="Max Waiting Time in Seconds", 1452 ) 1453 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1454 1455 1456class WaitUntilTimeFromHeader(BaseModel): 1457 type: Literal["WaitUntilTimeFromHeader"] 1458 header: str = Field( 1459 ..., 1460 description="The name of the response header defining how long to wait before retrying.", 1461 examples=["wait_time"], 1462 title="Response Header", 1463 ) 1464 min_wait: Optional[Union[float, str]] = Field( 1465 None, 1466 description="Minimum time to wait before retrying.", 1467 examples=[10, "60"], 1468 title="Minimum Wait Time", 1469 ) 1470 regex: Optional[str] = Field( 1471 None, 1472 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1473 examples=["([-+]?\\d+)"], 1474 title="Extraction Regex", 1475 ) 1476 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1477 None, 1478 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1479 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1480 title="Max Waiting Time in Seconds", 1481 ) 1482 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1483 1484 1485class ComponentMappingDefinition(BaseModel): 1486 type: Literal["ComponentMappingDefinition"] 1487 field_path: List[str] = Field( 1488 ..., 1489 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1490 examples=[ 1491 ["name"], 1492 ["retriever", "requester", "url"], 1493 ["retriever", "requester", "{{ components_values.field }}"], 1494 ["*", "**", "name"], 1495 ], 1496 title="Field Path", 1497 ) 1498 value: str = Field( 1499 ..., 1500 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1501 examples=[ 1502 "{{ components_values['updates'] }}", 1503 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1504 "{{ config['segment_id'] }}", 1505 "{{ stream_slice['parent_id'] }}", 1506 "{{ stream_slice['extra_fields']['name'] }}", 1507 ], 1508 title="Value", 1509 ) 1510 value_type: Optional[ValueType] = Field( 1511 None, 1512 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1513 title="Value Type", 1514 ) 1515 create_or_update: Optional[bool] = Field( 1516 False, 1517 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1518 title="Create or Update", 1519 ) 1520 condition: Optional[str] = Field( 1521 None, 1522 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1523 examples=[ 1524 "{{ components_values.get('cursor_field', None) }}", 1525 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1526 ], 1527 title="Condition", 1528 ) 1529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1530 1531 1532class StreamConfig(BaseModel): 1533 type: Literal["StreamConfig"] 1534 configs_pointer: List[str] = Field( 1535 ..., 1536 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1537 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1538 title="Configs Pointer", 1539 ) 1540 default_values: Optional[List[Dict[str, Any]]] = Field( 1541 None, 1542 description="A list of default values, each matching the structure expected from the parsed component value.", 1543 title="Default Values", 1544 ) 1545 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1546 1547 1548class ConfigComponentsResolver(BaseModel): 1549 type: Literal["ConfigComponentsResolver"] 1550 stream_config: Union[List[StreamConfig], StreamConfig] 1551 components_mapping: List[ComponentMappingDefinition] 1552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1553 1554 1555class StreamParametersDefinition(BaseModel): 1556 type: Literal["StreamParametersDefinition"] 1557 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1558 ..., 1559 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1560 examples=[ 1561 [ 1562 { 1563 "name": "test stream", 1564 "$parameters": {"entity": "test entity"}, 1565 "primary_key": "test key", 1566 } 1567 ] 1568 ], 1569 title="Stream Parameters", 1570 ) 1571 1572 1573class ParametrizedComponentsResolver(BaseModel): 1574 type: Literal["ParametrizedComponentsResolver"] 1575 stream_parameters: StreamParametersDefinition 1576 components_mapping: List[ComponentMappingDefinition] 1577 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1578 1579 1580class RequestBodyPlainText(BaseModel): 1581 type: Literal["RequestBodyPlainText"] 1582 value: str 1583 1584 1585class RequestBodyUrlEncodedForm(BaseModel): 1586 type: Literal["RequestBodyUrlEncodedForm"] 1587 value: Dict[str, str] 1588 1589 1590class RequestBodyJsonObject(BaseModel): 1591 type: Literal["RequestBodyJsonObject"] 1592 value: Dict[str, Any] 1593 1594 1595class RequestBodyGraphQlQuery(BaseModel): 1596 class Config: 1597 extra = Extra.allow 1598 1599 query: str = Field(..., description="The GraphQL query to be executed") 1600 1601 1602class ValidateAdheresToSchema(BaseModel): 1603 type: Literal["ValidateAdheresToSchema"] 1604 base_schema: Union[str, Dict[str, Any]] = Field( 1605 ..., 1606 description="The base JSON schema against which the user-provided schema will be validated.", 1607 examples=[ 1608 "{{ config['report_validation_schema'] }}", 1609 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1610 { 1611 "$schema": "http://json-schema.org/draft-07/schema#", 1612 "title": "Person", 1613 "type": "object", 1614 "properties": { 1615 "name": {"type": "string", "description": "The person's name"}, 1616 "age": { 1617 "type": "integer", 1618 "minimum": 0, 1619 "description": "The person's age", 1620 }, 1621 }, 1622 "required": ["name", "age"], 1623 }, 1624 ], 1625 title="Base JSON Schema", 1626 ) 1627 1628 1629class CustomValidationStrategy(BaseModel): 1630 class Config: 1631 extra = Extra.allow 1632 1633 type: Literal["CustomValidationStrategy"] 1634 class_name: str = Field( 1635 ..., 1636 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1637 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1638 title="Class Name", 1639 ) 1640 1641 1642class ConfigRemapField(BaseModel): 1643 type: Literal["ConfigRemapField"] 1644 map: Union[Dict[str, Any], str] = Field( 1645 ..., 1646 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1647 examples=[ 1648 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1649 "{{ config['status_mapping'] }}", 1650 ], 1651 title="Value Mapping", 1652 ) 1653 field_path: List[str] = Field( 1654 ..., 1655 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1656 examples=[ 1657 ["status"], 1658 ["data", "status"], 1659 ["data", "{{ config.name }}", "status"], 1660 ["data", "*", "status"], 1661 ], 1662 title="Field Path", 1663 ) 1664 1665 1666class ConfigRemoveFields(BaseModel): 1667 type: Literal["ConfigRemoveFields"] 1668 field_pointers: List[List[str]] = Field( 1669 ..., 1670 description="A list of field pointers to be removed from the config.", 1671 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1672 title="Field Pointers", 1673 ) 1674 condition: Optional[str] = Field( 1675 "", 1676 description="Fields will be removed if expression is evaluated to True.", 1677 examples=[ 1678 "{{ config['environemnt'] == 'sandbox' }}", 1679 "{{ property is integer }}", 1680 "{{ property|length > 5 }}", 1681 "{{ property == 'some_string_to_match' }}", 1682 ], 1683 ) 1684 1685 1686class CustomConfigTransformation(BaseModel): 1687 type: Literal["CustomConfigTransformation"] 1688 class_name: str = Field( 1689 ..., 1690 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1691 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1692 ) 1693 parameters: Optional[Dict[str, Any]] = Field( 1694 None, 1695 alias="$parameters", 1696 description="Additional parameters to be passed to the custom config transformation.", 1697 ) 1698 1699 1700class AddedFieldDefinition(BaseModel): 1701 type: Literal["AddedFieldDefinition"] 1702 path: List[str] = Field( 1703 ..., 1704 description="List of strings defining the path where to add the value on the record.", 1705 examples=[["segment_id"], ["metadata", "segment_id"]], 1706 title="Path", 1707 ) 1708 value: str = Field( 1709 ..., 1710 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1711 examples=[ 1712 "{{ record['updates'] }}", 1713 "{{ record['MetaData']['LastUpdatedTime'] }}", 1714 "{{ stream_partition['segment_id'] }}", 1715 ], 1716 title="Value", 1717 ) 1718 value_type: Optional[ValueType] = Field( 1719 None, 1720 description="Type of the value. If not specified, the type will be inferred from the value.", 1721 title="Value Type", 1722 ) 1723 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1724 1725 1726class AddFields(BaseModel): 1727 type: Literal["AddFields"] 1728 fields: List[AddedFieldDefinition] = Field( 1729 ..., 1730 description="List of transformations (path and corresponding value) that will be added to the record.", 1731 title="Fields", 1732 ) 1733 condition: Optional[str] = Field( 1734 "", 1735 description="Fields will be added if expression is evaluated to True.", 1736 examples=[ 1737 "{{ property|string == '' }}", 1738 "{{ property is integer }}", 1739 "{{ property|length > 5 }}", 1740 "{{ property == 'some_string_to_match' }}", 1741 ], 1742 ) 1743 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1744 1745 1746class ApiKeyAuthenticator(BaseModel): 1747 type: Literal["ApiKeyAuthenticator"] 1748 api_token: Optional[str] = Field( 1749 None, 1750 description="The API key to inject in the request. Fill it in the user inputs.", 1751 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1752 title="API Key", 1753 ) 1754 header: Optional[str] = Field( 1755 None, 1756 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1757 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1758 title="Header Name", 1759 ) 1760 inject_into: Optional[RequestOption] = Field( 1761 None, 1762 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1763 examples=[ 1764 {"inject_into": "header", "field_name": "Authorization"}, 1765 {"inject_into": "request_parameter", "field_name": "authKey"}, 1766 ], 1767 title="Inject API Key Into Outgoing HTTP Request", 1768 ) 1769 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1770 1771 1772class AuthFlow(BaseModel): 1773 auth_flow_type: Optional[AuthFlowType] = Field( 1774 None, description="The type of auth to use", title="Auth flow type" 1775 ) 1776 predicate_key: Optional[List[str]] = Field( 1777 None, 1778 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1779 examples=[["credentials", "auth_type"]], 1780 title="Predicate key", 1781 ) 1782 predicate_value: Optional[str] = Field( 1783 None, 1784 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1785 examples=["Oauth"], 1786 title="Predicate value", 1787 ) 1788 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1789 1790 1791class TokenQuota(BaseModel): 1792 type: Literal["TokenQuota"] 1793 name: str = Field( 1794 ..., 1795 description="Name of the quota pool.", 1796 examples=["rest", "graphql"], 1797 title="Name", 1798 ) 1799 remaining_path: List[str] = Field( 1800 ..., 1801 description="Path to the remaining call count for this pool in the quota status response.", 1802 examples=[["resources", "core", "remaining"]], 1803 title="Remaining Path", 1804 ) 1805 reset_path: List[str] = Field( 1806 ..., 1807 description="Path to the quota reset timestamp for this pool in the quota status response.", 1808 examples=[["resources", "core", "reset"]], 1809 title="Reset Path", 1810 ) 1811 limit_path: Optional[List[str]] = Field( 1812 None, 1813 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 1814 examples=[["resources", "core", "limit"]], 1815 title="Limit Path", 1816 ) 1817 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 1818 None, 1819 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 1820 title="Matchers", 1821 ) 1822 remaining_header: Optional[str] = Field( 1823 None, 1824 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 1825 examples=["X-RateLimit-Remaining"], 1826 title="Remaining Header", 1827 ) 1828 reset_header: Optional[str] = Field( 1829 None, 1830 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 1831 examples=["X-RateLimit-Reset"], 1832 title="Reset Header", 1833 ) 1834 limit_header: Optional[str] = Field( 1835 None, 1836 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 1837 examples=["X-RateLimit-Limit"], 1838 title="Limit Header", 1839 ) 1840 exhaustion_status_codes: Optional[List[int]] = Field( 1841 None, 1842 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 1843 examples=[[429]], 1844 title="Exhaustion Status Codes", 1845 ) 1846 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1847 1848 1849class CheckStream(BaseModel): 1850 type: Literal["CheckStream"] 1851 stream_names: Optional[List[str]] = Field( 1852 None, 1853 description="Names of the streams to try reading from when running a check operation.", 1854 examples=[["users"], ["users", "contacts"]], 1855 title="Stream Names", 1856 ) 1857 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1858 config_overrides: Optional[Dict[str, Any]] = Field( 1859 None, 1860 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1861 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1862 title="Config Overrides", 1863 ) 1864 1865 1866class IncrementingCountCursor(BaseModel): 1867 type: Literal["IncrementingCountCursor"] 1868 cursor_field: str = Field( 1869 ..., 1870 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1871 examples=["created_at", "{{ config['record_cursor'] }}"], 1872 title="Cursor Field", 1873 ) 1874 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1875 None, 1876 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1877 title="Allow Catalog Defined Cursor Field", 1878 ) 1879 start_value: Optional[Union[str, int]] = Field( 1880 None, 1881 description="The value that determines the earliest record that should be synced.", 1882 examples=[0, "{{ config['start_value'] }}"], 1883 title="Start Value", 1884 ) 1885 start_value_option: Optional[RequestOption] = Field( 1886 None, 1887 description="Optionally configures how the start value will be sent in requests to the source API.", 1888 title="Inject Start Value Into Outgoing HTTP Request", 1889 ) 1890 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1891 1892 1893class DatetimeBasedCursor(BaseModel): 1894 type: Literal["DatetimeBasedCursor"] 1895 clamping: Optional[Clamping] = Field( 1896 None, 1897 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1898 title="Date Range Clamping", 1899 ) 1900 cursor_field: str = Field( 1901 ..., 1902 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1903 examples=["created_at", "{{ config['record_cursor'] }}"], 1904 title="Cursor Field", 1905 ) 1906 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1907 None, 1908 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1909 title="Allow Catalog Defined Cursor Field", 1910 ) 1911 cursor_datetime_formats: Optional[List[str]] = Field( 1912 None, 1913 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1914 examples=[ 1915 "%Y-%m-%d", 1916 "%Y-%m-%d %H:%M:%S", 1917 "%Y-%m-%dT%H:%M:%S", 1918 "%Y-%m-%dT%H:%M:%SZ", 1919 "%Y-%m-%dT%H:%M:%S%z", 1920 "%Y-%m-%dT%H:%M:%S.%fZ", 1921 "%Y-%m-%dT%H:%M:%S.%f%z", 1922 "%Y-%m-%d %H:%M:%S.%f+00:00", 1923 "%s", 1924 "%ms", 1925 ], 1926 title="Cursor Datetime Formats", 1927 ) 1928 start_datetime: Union[MinMaxDatetime, str] = Field( 1929 ..., 1930 description="The datetime that determines the earliest record that should be synced.", 1931 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1932 title="Start Datetime", 1933 ) 1934 start_time_option: Optional[RequestOption] = Field( 1935 None, 1936 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1937 title="Inject Start Time Into Outgoing HTTP Request", 1938 ) 1939 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1940 None, 1941 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1942 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1943 title="End Datetime", 1944 ) 1945 end_time_option: Optional[RequestOption] = Field( 1946 None, 1947 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1948 title="Inject End Time Into Outgoing HTTP Request", 1949 ) 1950 datetime_format: str = Field( 1951 ..., 1952 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1953 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1954 title="Outgoing Datetime Format", 1955 ) 1956 cursor_granularity: Optional[str] = Field( 1957 None, 1958 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1959 examples=["PT1S"], 1960 title="Cursor Granularity", 1961 ) 1962 is_data_feed: Optional[bool] = Field( 1963 None, 1964 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1965 title="Data Feed API", 1966 ) 1967 is_client_side_incremental: Optional[bool] = Field( 1968 None, 1969 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1970 title="Client-side Incremental Filtering", 1971 ) 1972 is_compare_strictly: Optional[bool] = Field( 1973 False, 1974 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1975 title="Strict Start-End Time Comparison", 1976 ) 1977 global_substream_cursor: Optional[bool] = Field( 1978 False, 1979 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1980 title="Global Substream Cursor", 1981 ) 1982 lookback_window: Optional[str] = Field( 1983 None, 1984 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1985 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1986 title="Lookback Window", 1987 ) 1988 partition_field_end: Optional[str] = Field( 1989 None, 1990 description="Name of the partition start time field.", 1991 examples=["ending_time"], 1992 title="Partition Field End", 1993 ) 1994 partition_field_start: Optional[str] = Field( 1995 None, 1996 description="Name of the partition end time field.", 1997 examples=["starting_time"], 1998 title="Partition Field Start", 1999 ) 2000 step: Optional[str] = Field( 2001 None, 2002 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2003 examples=["P1W", "{{ config['step_increment'] }}"], 2004 title="Step", 2005 ) 2006 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2007 2008 2009class JwtAuthenticator(BaseModel): 2010 type: Literal["JwtAuthenticator"] 2011 secret_key: str = Field( 2012 ..., 2013 description="Secret used to sign the JSON web token.", 2014 examples=["{{ config['secret_key'] }}"], 2015 title="Secret Key", 2016 ) 2017 base64_encode_secret_key: Optional[bool] = Field( 2018 False, 2019 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 2020 title="Base64-encode Secret Key", 2021 ) 2022 algorithm: Algorithm = Field( 2023 ..., 2024 description="Algorithm used to sign the JSON web token.", 2025 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 2026 title="Algorithm", 2027 ) 2028 token_duration: Optional[int] = Field( 2029 1200, 2030 description="The amount of time in seconds a JWT token can be valid after being issued.", 2031 examples=[1200, 3600], 2032 title="Token Duration", 2033 ) 2034 header_prefix: Optional[str] = Field( 2035 None, 2036 description="The prefix to be used within the Authentication header.", 2037 examples=["Bearer", "Basic"], 2038 title="Header Prefix", 2039 ) 2040 jwt_headers: Optional[JwtHeaders] = Field( 2041 None, 2042 description="JWT headers used when signing JSON web token.", 2043 title="JWT Headers", 2044 ) 2045 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 2046 None, 2047 description="Additional headers to be included with the JWT headers object.", 2048 title="Additional JWT Headers", 2049 ) 2050 jwt_payload: Optional[JwtPayload] = Field( 2051 None, 2052 description="JWT Payload used when signing JSON web token.", 2053 title="JWT Payload", 2054 ) 2055 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2056 None, 2057 description="Additional properties to be added to the JWT payload.", 2058 title="Additional JWT Payload Properties", 2059 ) 2060 passphrase: Optional[str] = Field( 2061 None, 2062 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2063 examples=["{{ config['passphrase'] }}"], 2064 title="Passphrase", 2065 ) 2066 request_option: Optional[RequestOption] = Field( 2067 None, 2068 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2069 title="Request Option", 2070 ) 2071 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2072 2073 2074class OAuthAuthenticator(BaseModel): 2075 type: Literal["OAuthAuthenticator"] 2076 client_id_name: Optional[str] = Field( 2077 "client_id", 2078 description="The name of the property to use to refresh the `access_token`.", 2079 examples=["custom_app_id"], 2080 title="Client ID Property Name", 2081 ) 2082 client_id: Optional[str] = Field( 2083 None, 2084 description="The OAuth client ID. Fill it in the user inputs.", 2085 examples=[ 2086 "{{ config['client_id'] }}", 2087 "{{ config['credentials']['client_id }}", 2088 ], 2089 title="Client ID", 2090 ) 2091 client_secret_name: Optional[str] = Field( 2092 "client_secret", 2093 description="The name of the property to use to refresh the `access_token`.", 2094 examples=["custom_app_secret"], 2095 title="Client Secret Property Name", 2096 ) 2097 client_secret: Optional[str] = Field( 2098 None, 2099 description="The OAuth client secret. Fill it in the user inputs.", 2100 examples=[ 2101 "{{ config['client_secret'] }}", 2102 "{{ config['credentials']['client_secret }}", 2103 ], 2104 title="Client Secret", 2105 ) 2106 refresh_token_name: Optional[str] = Field( 2107 "refresh_token", 2108 description="The name of the property to use to refresh the `access_token`.", 2109 examples=["custom_app_refresh_value"], 2110 title="Refresh Token Property Name", 2111 ) 2112 refresh_token: Optional[str] = Field( 2113 None, 2114 description="Credential artifact used to get a new access token.", 2115 examples=[ 2116 "{{ config['refresh_token'] }}", 2117 "{{ config['credentials]['refresh_token'] }}", 2118 ], 2119 title="Refresh Token", 2120 ) 2121 token_refresh_endpoint: Optional[str] = Field( 2122 None, 2123 description="The full URL to call to obtain a new access token.", 2124 examples=["https://connect.squareup.com/oauth2/token"], 2125 title="Token Refresh Endpoint", 2126 ) 2127 access_token_name: Optional[str] = Field( 2128 "access_token", 2129 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2130 examples=["access_token"], 2131 title="Access Token Property Name", 2132 ) 2133 access_token_value: Optional[str] = Field( 2134 None, 2135 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2136 examples=["secret_access_token_value"], 2137 title="Access Token Value", 2138 ) 2139 expires_in_name: Optional[str] = Field( 2140 "expires_in", 2141 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2142 examples=["expires_in"], 2143 title="Token Expiry Property Name", 2144 ) 2145 grant_type_name: Optional[str] = Field( 2146 "grant_type", 2147 description="The name of the property to use to refresh the `access_token`.", 2148 examples=["custom_grant_type"], 2149 title="Grant Type Property Name", 2150 ) 2151 grant_type: Optional[str] = Field( 2152 "refresh_token", 2153 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2154 examples=["refresh_token", "client_credentials"], 2155 title="Grant Type", 2156 ) 2157 refresh_request_body: Optional[Dict[str, Any]] = Field( 2158 None, 2159 description="Body of the request sent to get a new access token.", 2160 examples=[ 2161 { 2162 "applicationId": "{{ config['application_id'] }}", 2163 "applicationSecret": "{{ config['application_secret'] }}", 2164 "token": "{{ config['token'] }}", 2165 } 2166 ], 2167 title="Refresh Request Body", 2168 ) 2169 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2170 None, 2171 description="Headers of the request sent to get a new access token.", 2172 examples=[ 2173 { 2174 "Authorization": "<AUTH_TOKEN>", 2175 "Content-Type": "application/x-www-form-urlencoded", 2176 } 2177 ], 2178 title="Refresh Request Headers", 2179 ) 2180 send_refresh_request_as_query_params: Optional[bool] = Field( 2181 False, 2182 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2183 examples=[True], 2184 title="Send Refresh Request As Query Params", 2185 ) 2186 scopes: Optional[List[str]] = Field( 2187 None, 2188 description="List of scopes that should be granted to the access token.", 2189 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2190 title="Scopes", 2191 ) 2192 token_expiry_date: Optional[str] = Field( 2193 None, 2194 description="The access token expiry date.", 2195 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2196 title="Token Expiry Date", 2197 ) 2198 token_expiry_date_format: Optional[str] = Field( 2199 None, 2200 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2201 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2202 title="Token Expiry Date Format", 2203 ) 2204 refresh_token_error_status_codes: Optional[List[int]] = Field( 2205 None, 2206 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2207 examples=[[400, 500]], 2208 title="Refresh Token Error Status Codes", 2209 ) 2210 refresh_token_error_key: Optional[str] = Field( 2211 None, 2212 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2213 examples=["error"], 2214 title="Refresh Token Error Key", 2215 ) 2216 refresh_token_error_values: Optional[List[str]] = Field( 2217 None, 2218 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2219 examples=[["invalid_grant", "invalid_permissions"]], 2220 title="Refresh Token Error Values", 2221 ) 2222 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2223 None, 2224 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2225 title="Refresh Token Updater", 2226 ) 2227 profile_assertion: Optional[JwtAuthenticator] = Field( 2228 None, 2229 description="The authenticator being used to authenticate the client authenticator.", 2230 title="Profile Assertion", 2231 ) 2232 use_profile_assertion: Optional[bool] = Field( 2233 False, 2234 description="Enable using profile assertion as a flow for OAuth authorization.", 2235 title="Use Profile Assertion", 2236 ) 2237 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2238 2239 2240class FixedWindowCallRatePolicy(BaseModel): 2241 class Config: 2242 extra = Extra.allow 2243 2244 type: Literal["FixedWindowCallRatePolicy"] 2245 period: str = Field( 2246 ..., description="The time interval for the rate limit window.", title="Period" 2247 ) 2248 call_limit: int = Field( 2249 ..., 2250 description="The maximum number of calls allowed within the period.", 2251 title="Call Limit", 2252 ) 2253 matchers: List[HttpRequestRegexMatcher] = Field( 2254 ..., 2255 description="List of matchers that define which requests this policy applies to.", 2256 title="Matchers", 2257 ) 2258 2259 2260class MovingWindowCallRatePolicy(BaseModel): 2261 class Config: 2262 extra = Extra.allow 2263 2264 type: Literal["MovingWindowCallRatePolicy"] 2265 rates: List[Rate] = Field( 2266 ..., 2267 description="List of rates that define the call limits for different time intervals.", 2268 title="Rates", 2269 ) 2270 matchers: List[HttpRequestRegexMatcher] = Field( 2271 ..., 2272 description="List of matchers that define which requests this policy applies to.", 2273 title="Matchers", 2274 ) 2275 2276 2277class UnlimitedCallRatePolicy(BaseModel): 2278 class Config: 2279 extra = Extra.allow 2280 2281 type: Literal["UnlimitedCallRatePolicy"] 2282 matchers: List[HttpRequestRegexMatcher] = Field( 2283 ..., 2284 description="List of matchers that define which requests this policy applies to.", 2285 title="Matchers", 2286 ) 2287 2288 2289class DefaultErrorHandler(BaseModel): 2290 type: Literal["DefaultErrorHandler"] 2291 backoff_strategies: Optional[ 2292 List[ 2293 Union[ 2294 ConstantBackoffStrategy, 2295 ExponentialBackoffStrategy, 2296 WaitTimeFromHeader, 2297 WaitUntilTimeFromHeader, 2298 CustomBackoffStrategy, 2299 ] 2300 ] 2301 ] = Field( 2302 None, 2303 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2304 title="Backoff Strategies", 2305 ) 2306 max_retries: Optional[Union[int, str]] = Field( 2307 5, 2308 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2309 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2310 title="Max Retry Count", 2311 ) 2312 response_filters: Optional[List[HttpResponseFilter]] = Field( 2313 None, 2314 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2315 title="Response Filters", 2316 ) 2317 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2318 2319 2320class DefaultPaginator(BaseModel): 2321 type: Literal["DefaultPaginator"] 2322 pagination_strategy: Union[ 2323 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2324 ] = Field( 2325 ..., 2326 description="Strategy defining how records are paginated.", 2327 title="Pagination Strategy", 2328 ) 2329 page_size_option: Optional[RequestOption] = Field( 2330 None, title="Inject Page Size Into Outgoing HTTP Request" 2331 ) 2332 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2333 None, 2334 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2335 title="Inject Page Token Into Outgoing HTTP Request", 2336 ) 2337 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2338 2339 2340class RecordExpander(BaseModel): 2341 type: Literal["RecordExpander"] 2342 expand_records_from_field: List[str] = Field( 2343 ..., 2344 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2345 examples=[ 2346 ["lines", "data"], 2347 ["items"], 2348 ["nested", "array"], 2349 ["sections", "*", "items"], 2350 ], 2351 title="Expand Records From Field", 2352 ) 2353 remain_original_record: Optional[bool] = Field( 2354 False, 2355 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2356 title="Remain Original Record", 2357 ) 2358 parent_fields: Optional[List[ParentFieldPath]] = Field( 2359 None, 2360 description="Named values to copy from the record being expanded onto each expanded item. Use this instead of `remain_original_record` when only a few parent fields are needed: it copies the named values rather than deep-copying the whole parent once per item, which matters when the parent record is large and the nested list is long. An existing value at `record_path` is overwritten, and a `parent_path` the parent does not have copies null. A copied object or array is deep-copied, so a transformation that writes inside it affects only that item. Independent of `remain_original_record`; both may be set. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2361 title="Parent Fields", 2362 ) 2363 merge_parent: Optional[bool] = Field( 2364 False, 2365 description="If true, each expanded item is the parent record shallow-merged with the item, the item's own keys winning on collision, and the expanded list removed from the parent's copy. Only the value at `expand_records_from_field` is removed, so for a multi-segment path the top-level key stays with its other fields. Each item receives its own deep copy of the merged parent fields, so a transformation that writes into a nested value affects only that item. The merge happens first, then `parent_fields` are copied, then `original_record` is embedded when `remain_original_record` is set; all three may be combined. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2366 title="Merge Parent", 2367 ) 2368 on_no_records: Optional[OnNoRecords] = Field( 2369 OnNoRecords.skip, 2370 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2371 title="On No Records", 2372 ) 2373 truncation_indicator_path: Optional[List[str]] = Field( 2374 None, 2375 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2376 examples=[["data", "object", "lines", "has_more"]], 2377 title="Truncation Indicator Path", 2378 ) 2379 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2380 None, 2381 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2382 title="Truncated List Retriever", 2383 ) 2384 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2385 2386 2387class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2388 type: Literal["ApiKey"] 2389 inject_into: RequestOption = Field( 2390 ..., 2391 description="Configure how the API Key will be sent in requests to the source API.", 2392 examples=[ 2393 {"inject_into": "header", "field_name": "Authorization"}, 2394 {"inject_into": "request_parameter", "field_name": "authKey"}, 2395 ], 2396 title="Inject API Key Into Outgoing HTTP Request", 2397 ) 2398 api_token: Optional[str] = Field( 2399 "{{ session_token }}", 2400 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2401 examples=[ 2402 "{{ session_token }}", 2403 "Token {{ session_token }}", 2404 "Bearer {{ session_token }}", 2405 ], 2406 title="API Token Template", 2407 ) 2408 2409 2410class JsonSchemaPropertySelector(BaseModel): 2411 type: Literal["JsonSchemaPropertySelector"] 2412 transformations: Optional[ 2413 List[ 2414 Union[ 2415 AddFields, 2416 RemoveFields, 2417 KeysToLower, 2418 KeysToSnakeCase, 2419 FlattenFields, 2420 DpathFlattenFields, 2421 KeysReplace, 2422 CustomTransformation, 2423 ] 2424 ] 2425 ] = Field( 2426 None, 2427 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2428 title="Transformations", 2429 ) 2430 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2431 2432 2433class ListPartitionRouter(BaseModel): 2434 type: Literal["ListPartitionRouter"] 2435 cursor_field: str = Field( 2436 ..., 2437 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2438 examples=["section", "{{ config['section_key'] }}"], 2439 title="Current Partition Value Identifier", 2440 ) 2441 values: Union[str, List[str]] = Field( 2442 ..., 2443 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2444 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2445 title="Partition Values", 2446 ) 2447 request_option: Optional[RequestOption] = Field( 2448 None, 2449 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2450 title="Inject Partition Value Into Outgoing HTTP Request", 2451 ) 2452 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2453 2454 2455class PaginationReset(BaseModel): 2456 type: Literal["PaginationReset"] 2457 action: Action1 2458 limits: Optional[PaginationResetLimits] = None 2459 2460 2461class GzipDecoder(BaseModel): 2462 type: Literal["GzipDecoder"] 2463 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2464 2465 2466class RequestBodyGraphQL(BaseModel): 2467 type: Literal["RequestBodyGraphQL"] 2468 value: RequestBodyGraphQlQuery 2469 2470 2471class DpathValidator(BaseModel): 2472 type: Literal["DpathValidator"] 2473 field_path: List[str] = Field( 2474 ..., 2475 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2476 examples=[ 2477 ["data"], 2478 ["data", "records"], 2479 ["data", "{{ parameters.name }}"], 2480 ["data", "*", "record"], 2481 ], 2482 title="Field Path", 2483 ) 2484 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2485 ..., 2486 description="The condition that the specified config value will be evaluated against", 2487 title="Validation Strategy", 2488 ) 2489 2490 2491class PredicateValidator(BaseModel): 2492 type: Literal["PredicateValidator"] 2493 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2494 ..., 2495 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2496 examples=[ 2497 "test-value", 2498 "{{ config['api_version'] }}", 2499 "{{ config['tenant_id'] }}", 2500 123, 2501 ], 2502 title="Value", 2503 ) 2504 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2505 ..., 2506 description="The validation strategy to apply to the value.", 2507 title="Validation Strategy", 2508 ) 2509 2510 2511class ConfigAddFields(BaseModel): 2512 type: Literal["ConfigAddFields"] 2513 fields: List[AddedFieldDefinition] = Field( 2514 ..., 2515 description="A list of transformations (path and corresponding value) that will be added to the config.", 2516 title="Fields", 2517 ) 2518 condition: Optional[str] = Field( 2519 "", 2520 description="Fields will be added if expression is evaluated to True.", 2521 examples=[ 2522 "{{ config['environemnt'] == 'sandbox' }}", 2523 "{{ property is integer }}", 2524 "{{ property|length > 5 }}", 2525 "{{ property == 'some_string_to_match' }}", 2526 ], 2527 ) 2528 2529 2530class RateLimitedMultipleTokenAuthenticator(BaseModel): 2531 type: Literal["RateLimitedMultipleTokenAuthenticator"] 2532 tokens: Union[str, List[str]] = Field( 2533 ..., 2534 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 2535 examples=[ 2536 "{{ config['credentials']['personal_access_token'] }}", 2537 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 2538 ], 2539 title="Tokens", 2540 ) 2541 token_delimiter: Optional[str] = Field( 2542 ",", 2543 description="Delimiter used to split a single token string into multiple tokens.", 2544 title="Token Delimiter", 2545 ) 2546 auth_method: Optional[str] = Field( 2547 "Bearer", 2548 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 2549 examples=["Bearer", "token"], 2550 title="Auth Method", 2551 ) 2552 header: Optional[str] = Field( 2553 "Authorization", 2554 description="The name of the HTTP header in which to inject the token.", 2555 title="Header Name", 2556 ) 2557 quota_status_source: QuotaStatusSource = Field( 2558 ..., 2559 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 2560 title="Quota Status Source", 2561 ) 2562 quotas: List[TokenQuota] = Field( 2563 ..., 2564 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 2565 min_items=1, 2566 title="Quota Pools", 2567 ) 2568 max_wait_time: Optional[str] = Field( 2569 "PT2H", 2570 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 2571 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 2572 title="Maximum Wait Time", 2573 ) 2574 budget_reserve_fraction: Optional[float] = Field( 2575 0.1, 2576 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 2577 title="Budget Reserve Fraction", 2578 ) 2579 budget_min_reserve: Optional[int] = Field( 2580 50, 2581 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 2582 title="Budget Minimum Reserve", 2583 ) 2584 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2585 2586 2587class CompositeErrorHandler(BaseModel): 2588 type: Literal["CompositeErrorHandler"] 2589 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2590 Field( 2591 ..., 2592 description="List of error handlers to iterate on to determine how to handle a failed response.", 2593 title="Error Handlers", 2594 ) 2595 ) 2596 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2597 2598 2599class HTTPAPIBudget(BaseModel): 2600 class Config: 2601 extra = Extra.allow 2602 2603 type: Literal["HTTPAPIBudget"] 2604 policies: List[ 2605 Union[ 2606 FixedWindowCallRatePolicy, 2607 MovingWindowCallRatePolicy, 2608 UnlimitedCallRatePolicy, 2609 ] 2610 ] = Field( 2611 ..., 2612 description="List of call rate policies that define how many calls are allowed.", 2613 title="Policies", 2614 ) 2615 ratelimit_reset_header: Optional[str] = Field( 2616 "ratelimit-reset", 2617 description="The HTTP response header name that indicates when the rate limit resets.", 2618 title="Rate Limit Reset Header", 2619 ) 2620 ratelimit_remaining_header: Optional[str] = Field( 2621 "ratelimit-remaining", 2622 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2623 title="Rate Limit Remaining Header", 2624 ) 2625 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2626 [429], 2627 description="List of HTTP status codes that indicate a rate limit has been hit.", 2628 title="Status Codes for Rate Limit Hit", 2629 ) 2630 2631 2632class DpathExtractor(BaseModel): 2633 type: Literal["DpathExtractor"] 2634 field_path: List[str] = Field( 2635 ..., 2636 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2637 examples=[ 2638 ["data"], 2639 ["data", "records"], 2640 ["data", "{{ parameters.name }}"], 2641 ["data", "*", "record"], 2642 ], 2643 title="Field Path", 2644 ) 2645 record_expander: Optional[RecordExpander] = Field( 2646 None, 2647 description="Optional component to expand records by extracting items from nested array fields.", 2648 title="Record Expander", 2649 ) 2650 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2651 2652 2653class CombinedExtractor(BaseModel): 2654 type: Literal["CombinedExtractor"] 2655 extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]] = Field( 2656 ..., 2657 description="The record extractors to combine. At least one is required. Each sub-extractor is given the same HTTP response and decodes it independently, so the response is parsed once per sub-extractor. Streaming decoders (CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder, IterableDecoder) can only read the response body once and are rejected.", 2658 title="Extractors", 2659 ) 2660 mode: Optional[CombineMode] = Field( 2661 CombineMode.union, 2662 description='How the records of the sub-extractors are combined. "union" (default) yields every record of every sub-extractor, in the order the extractors are declared. "first_match" yields the records of the first sub-extractor that produces at least one record and skips the remaining ones; nothing is yielded if none of them produces a record. Note that a paginator which counts the records of a page counts the combined records: under "union" that is the sum over all sub-extractors, which overshoots the API page size, so "union" is rejected with an OffsetIncrement paginator because the offset would skip records. Under "first_match" the count is the count of the winning sub-extractor, which is usually the number of records the API returned for the page.', 2663 title="Combine Mode", 2664 ) 2665 skip_empty_records: Optional[bool] = Field( 2666 False, 2667 description='Whether to drop empty records - null, {}, [], "" - yielded by a sub-extractor before the records are combined. Off by default, so the output of a sub-extractor is passed through as it is. Turn it on when the API can return nulls in the middle of a record list, which a GraphQL API does when it answers a partial response and reports the failure in a sibling error field. Under "first_match" this also changes which sub-extractor wins: a sub-extractor whose records are all empty no longer counts as a match, so the next one is tried. The dropped records still count toward the page size of an OffsetIncrement or PageIncrement paginator, so a full page holding a null does not end pagination early.', 2668 title="Skip Empty Records", 2669 ) 2670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2671 2672 2673class ZipfileDecoder(BaseModel): 2674 class Config: 2675 extra = Extra.allow 2676 2677 type: Literal["ZipfileDecoder"] 2678 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2679 ..., 2680 description="Parser to parse the decompressed data from the zipfile(s).", 2681 title="Parser", 2682 ) 2683 2684 2685class RecordSelector(BaseModel): 2686 type: Literal["RecordSelector"] 2687 extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 2688 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2689 None, 2690 description="Responsible for filtering records to be emitted by the Source.", 2691 title="Record Filter", 2692 ) 2693 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2694 None, 2695 description="Responsible for normalization according to the schema.", 2696 title="Schema Normalization", 2697 ) 2698 transform_before_filtering: Optional[bool] = Field( 2699 None, 2700 description="If true, transformation will be applied before record filtering.", 2701 title="Transform Before Filtering", 2702 ) 2703 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2704 2705 2706class ConfigMigration(BaseModel): 2707 type: Literal["ConfigMigration"] 2708 description: Optional[str] = Field( 2709 None, description="The description/purpose of the config migration." 2710 ) 2711 transformations: List[ 2712 Union[ 2713 ConfigRemapField, 2714 ConfigAddFields, 2715 ConfigRemoveFields, 2716 CustomConfigTransformation, 2717 ] 2718 ] = Field( 2719 ..., 2720 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2721 title="Transformations", 2722 ) 2723 2724 2725class ConfigNormalizationRules(BaseModel): 2726 class Config: 2727 extra = Extra.forbid 2728 2729 type: Literal["ConfigNormalizationRules"] 2730 config_migrations: Optional[List[ConfigMigration]] = Field( 2731 [], 2732 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2733 title="Config Migrations", 2734 ) 2735 transformations: Optional[ 2736 List[ 2737 Union[ 2738 ConfigRemapField, 2739 ConfigAddFields, 2740 ConfigRemoveFields, 2741 CustomConfigTransformation, 2742 ] 2743 ] 2744 ] = Field( 2745 [], 2746 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2747 title="Transformations", 2748 ) 2749 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2750 [], 2751 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2752 title="Validations", 2753 ) 2754 2755 2756class Spec(BaseModel): 2757 type: Literal["Spec"] 2758 connection_specification: Dict[str, Any] = Field( 2759 ..., 2760 description="A connection specification describing how a the connector can be configured.", 2761 title="Connection Specification", 2762 ) 2763 documentation_url: Optional[str] = Field( 2764 None, 2765 description="URL of the connector's documentation page.", 2766 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2767 title="Documentation URL", 2768 ) 2769 advanced_auth: Optional[AuthFlow] = Field( 2770 None, 2771 description="Advanced specification for configuring the authentication flow.", 2772 title="Advanced Auth", 2773 ) 2774 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2775 None, title="Config Normalization Rules" 2776 ) 2777 2778 2779class DeclarativeSource1(BaseModel): 2780 class Config: 2781 extra = Extra.forbid 2782 2783 type: Literal["DeclarativeSource"] 2784 check: Union[CheckStream, CheckDynamicStream] 2785 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2786 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2787 version: str = Field( 2788 ..., 2789 description="The version of the Airbyte CDK used to build and test the source.", 2790 ) 2791 schemas: Optional[Schemas] = None 2792 definitions: Optional[Dict[str, Any]] = None 2793 spec: Optional[Spec] = None 2794 concurrency_level: Optional[ConcurrencyLevel] = None 2795 api_budget: Optional[HTTPAPIBudget] = None 2796 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2797 None, 2798 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2799 title="Stream Groups", 2800 ) 2801 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2802 None, 2803 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2804 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2805 title="Maximum Concurrent Asynchronous Jobs", 2806 ) 2807 metadata: Optional[Dict[str, Any]] = Field( 2808 None, 2809 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2810 ) 2811 description: Optional[str] = Field( 2812 None, 2813 description="A description of the connector. It will be presented on the Source documentation page.", 2814 ) 2815 2816 2817class DeclarativeSource2(BaseModel): 2818 class Config: 2819 extra = Extra.forbid 2820 2821 type: Literal["DeclarativeSource"] 2822 check: Union[CheckStream, CheckDynamicStream] 2823 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2824 None 2825 ) 2826 dynamic_streams: List[DynamicDeclarativeStream] 2827 version: str = Field( 2828 ..., 2829 description="The version of the Airbyte CDK used to build and test the source.", 2830 ) 2831 schemas: Optional[Schemas] = None 2832 definitions: Optional[Dict[str, Any]] = None 2833 spec: Optional[Spec] = None 2834 concurrency_level: Optional[ConcurrencyLevel] = None 2835 api_budget: Optional[HTTPAPIBudget] = None 2836 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2837 None, 2838 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2839 title="Stream Groups", 2840 ) 2841 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2842 None, 2843 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2844 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2845 title="Maximum Concurrent Asynchronous Jobs", 2846 ) 2847 metadata: Optional[Dict[str, Any]] = Field( 2848 None, 2849 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2850 ) 2851 description: Optional[str] = Field( 2852 None, 2853 description="A description of the connector. It will be presented on the Source documentation page.", 2854 ) 2855 2856 2857class DeclarativeSource(BaseModel): 2858 class Config: 2859 extra = Extra.forbid 2860 2861 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2862 ..., 2863 description="An API source that extracts data according to its declarative components.", 2864 title="DeclarativeSource", 2865 ) 2866 2867 2868class SelectiveAuthenticator(BaseModel): 2869 class Config: 2870 extra = Extra.allow 2871 2872 type: Literal["SelectiveAuthenticator"] 2873 authenticator_selection_path: List[str] = Field( 2874 ..., 2875 description="Path of the field in config with selected authenticator name", 2876 examples=[["auth"], ["auth", "type"]], 2877 title="Authenticator Selection Path", 2878 ) 2879 authenticators: Dict[ 2880 str, 2881 Union[ 2882 ApiKeyAuthenticator, 2883 BasicHttpAuthenticator, 2884 BearerAuthenticator, 2885 OAuthAuthenticator, 2886 JwtAuthenticator, 2887 SessionTokenAuthenticator, 2888 LegacySessionTokenAuthenticator, 2889 CustomAuthenticator, 2890 NoAuth, 2891 RateLimitedMultipleTokenAuthenticator, 2892 ], 2893 ] = Field( 2894 ..., 2895 description="Authenticators to select from.", 2896 examples=[ 2897 { 2898 "authenticators": { 2899 "token": "#/definitions/ApiKeyAuthenticator", 2900 "oauth": "#/definitions/OAuthAuthenticator", 2901 "jwt": "#/definitions/JwtAuthenticator", 2902 } 2903 } 2904 ], 2905 title="Authenticators", 2906 ) 2907 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2908 2909 2910class ConditionalStreams(BaseModel): 2911 type: Literal["ConditionalStreams"] 2912 condition: str = Field( 2913 ..., 2914 description="Condition that will be evaluated to determine if a set of streams should be available.", 2915 examples=["{{ config['is_sandbox'] }}"], 2916 title="Condition", 2917 ) 2918 streams: List[DeclarativeStream] = Field( 2919 ..., 2920 description="Streams that will be used during an operation based on the condition.", 2921 title="Streams", 2922 ) 2923 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2924 2925 2926class FileUploader(BaseModel): 2927 type: Literal["FileUploader"] 2928 requester: Union[HttpRequester, CustomRequester] = Field( 2929 ..., 2930 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2931 ) 2932 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = ( 2933 Field( 2934 ..., 2935 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2936 ) 2937 ) 2938 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2939 None, 2940 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2941 ) 2942 filename_extractor: Optional[str] = Field( 2943 None, 2944 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2945 examples=[ 2946 "{{ record.id }}/{{ record.file_name }}/", 2947 "{{ record.id }}_{{ record.file_name }}/", 2948 ], 2949 ) 2950 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2951 2952 2953class DeclarativeStream(BaseModel): 2954 class Config: 2955 extra = Extra.allow 2956 2957 type: Literal["DeclarativeStream"] 2958 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2959 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2960 ..., 2961 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2962 title="Retriever", 2963 ) 2964 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2965 None, 2966 description="Component used to fetch data incrementally based on a time field in the data.", 2967 title="Incremental Sync", 2968 ) 2969 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2970 schema_loader: Optional[ 2971 Union[ 2972 InlineSchemaLoader, 2973 DynamicSchemaLoader, 2974 JsonFileSchemaLoader, 2975 List[ 2976 Union[ 2977 InlineSchemaLoader, 2978 DynamicSchemaLoader, 2979 JsonFileSchemaLoader, 2980 CustomSchemaLoader, 2981 ] 2982 ], 2983 CustomSchemaLoader, 2984 ] 2985 ] = Field( 2986 None, 2987 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2988 title="Schema Loader", 2989 ) 2990 transformations: Optional[ 2991 List[ 2992 Union[ 2993 AddFields, 2994 RemoveFields, 2995 KeysToLower, 2996 KeysToSnakeCase, 2997 FlattenFields, 2998 DpathFlattenFields, 2999 KeysReplace, 3000 CustomTransformation, 3001 ] 3002 ] 3003 ] = Field( 3004 None, 3005 description="A list of transformations to be applied to each output record.", 3006 title="Transformations", 3007 ) 3008 state_migrations: Optional[ 3009 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 3010 ] = Field( 3011 [], 3012 description="Array of state migrations to be applied on the input state", 3013 title="State Migrations", 3014 ) 3015 file_uploader: Optional[FileUploader] = Field( 3016 None, 3017 description="(experimental) Describes how to fetch a file", 3018 title="File Uploader", 3019 ) 3020 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3021 3022 3023class SessionTokenAuthenticator(BaseModel): 3024 type: Literal["SessionTokenAuthenticator"] 3025 login_requester: HttpRequester = Field( 3026 ..., 3027 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 3028 examples=[ 3029 { 3030 "type": "HttpRequester", 3031 "url_base": "https://my_api.com", 3032 "path": "/login", 3033 "authenticator": { 3034 "type": "BasicHttpAuthenticator", 3035 "username": "{{ config.username }}", 3036 "password": "{{ config.password }}", 3037 }, 3038 } 3039 ], 3040 title="Login Requester", 3041 ) 3042 session_token_path: List[str] = Field( 3043 ..., 3044 description="The path in the response body returned from the login requester to the session token.", 3045 examples=[["access_token"], ["result", "token"]], 3046 title="Session Token Path", 3047 ) 3048 expiration_duration: Optional[str] = Field( 3049 None, 3050 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 3051 examples=["PT1H", "P1D"], 3052 title="Expiration Duration", 3053 ) 3054 request_authentication: Union[ 3055 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 3056 ] = Field( 3057 ..., 3058 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 3059 title="Data Request Authentication", 3060 ) 3061 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 3062 None, description="Component used to decode the response.", title="Decoder" 3063 ) 3064 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3065 3066 3067class HttpRequester(BaseModelWithDeprecations): 3068 type: Literal["HttpRequester"] 3069 url_base: Optional[str] = Field( 3070 None, 3071 deprecated=True, 3072 deprecation_message="Use `url` field instead.", 3073 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3074 examples=[ 3075 "https://connect.squareup.com/v2", 3076 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 3077 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3078 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3079 ], 3080 title="API Base URL", 3081 ) 3082 url: Optional[str] = Field( 3083 None, 3084 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3085 examples=[ 3086 "https://connect.squareup.com/v2", 3087 "{{ config['url'] or 'https://app.posthog.com'}}/api", 3088 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3089 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3090 ], 3091 title="API Endpoint URL", 3092 ) 3093 path: Optional[str] = Field( 3094 None, 3095 deprecated=True, 3096 deprecation_message="Use `url` field instead.", 3097 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3098 examples=[ 3099 "/products", 3100 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3101 "/trades/{{ config['symbol_id'] }}/history", 3102 ], 3103 title="URL Path", 3104 ) 3105 http_method: Optional[HttpMethod] = Field( 3106 HttpMethod.GET, 3107 description="The HTTP method used to fetch data from the source (can be GET or POST).", 3108 examples=["GET", "POST"], 3109 title="HTTP Method", 3110 ) 3111 authenticator: Optional[ 3112 Union[ 3113 ApiKeyAuthenticator, 3114 BasicHttpAuthenticator, 3115 BearerAuthenticator, 3116 OAuthAuthenticator, 3117 JwtAuthenticator, 3118 SessionTokenAuthenticator, 3119 SelectiveAuthenticator, 3120 CustomAuthenticator, 3121 NoAuth, 3122 LegacySessionTokenAuthenticator, 3123 RateLimitedMultipleTokenAuthenticator, 3124 ] 3125 ] = Field( 3126 None, 3127 description="Authentication method to use for requests sent to the API.", 3128 title="Authenticator", 3129 ) 3130 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 3131 None, 3132 deprecated=True, 3133 deprecation_message="Use `query_properties` field instead.", 3134 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 3135 title="Fetch Properties from Endpoint", 3136 ) 3137 query_properties: Optional[QueryProperties] = Field( 3138 None, 3139 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 3140 title="Query Properties", 3141 ) 3142 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 3143 None, 3144 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3145 examples=[ 3146 {"unit": "day"}, 3147 { 3148 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3149 }, 3150 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3151 {"sort_by[asc]": "updated_at"}, 3152 ], 3153 title="Query Parameters", 3154 ) 3155 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3156 None, 3157 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3158 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3159 title="Request Headers", 3160 ) 3161 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3162 None, 3163 deprecated=True, 3164 deprecation_message="Use `request_body` field instead.", 3165 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3166 examples=[ 3167 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3168 ], 3169 title="Request Body Payload (Non-JSON)", 3170 ) 3171 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3172 None, 3173 deprecated=True, 3174 deprecation_message="Use `request_body` field instead.", 3175 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3176 examples=[ 3177 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3178 {"key": "{{ config['value'] }}"}, 3179 {"sort": {"field": "updated_at", "order": "ascending"}}, 3180 ], 3181 title="Request Body JSON Payload", 3182 ) 3183 request_body: Optional[ 3184 Union[ 3185 RequestBodyPlainText, 3186 RequestBodyUrlEncodedForm, 3187 RequestBodyJsonObject, 3188 RequestBodyGraphQL, 3189 ] 3190 ] = Field( 3191 None, 3192 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3193 title="Request Body", 3194 ) 3195 error_handler: Optional[ 3196 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3197 ] = Field( 3198 None, 3199 description="Error handler component that defines how to handle errors.", 3200 title="Error Handler", 3201 ) 3202 use_cache: Optional[bool] = Field( 3203 False, 3204 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3205 title="Use Cache", 3206 ) 3207 connect_timeout_in_seconds: Optional[float] = Field( 3208 None, 3209 description="Maximum time to establish the connection (TCP + TLS handshake). When unset, no connect timeout is applied.", 3210 examples=[30], 3211 gt=0, 3212 title="Connect Timeout (seconds)", 3213 ) 3214 read_timeout_in_seconds: Optional[float] = Field( 3215 None, 3216 description="The maximum time to wait for the server to send any data (the first byte of the response, or the next chunk while the body is read). It is NOT a limit on the total request duration. When unset, no read timeout is applied. Only set it when every endpoint served by this requester responds well within the value; synchronous report or export endpoints can stay silent for minutes. Each timed-out attempt counts against the error handler's retry window (600 s for manifest connectors), so a value near 300 allows only a few retries.", 3217 examples=[300], 3218 gt=0, 3219 title="Read Timeout (seconds)", 3220 ) 3221 use_tcp_keepalive: Optional[bool] = Field( 3222 False, 3223 description="Enables TCP keepalive (60 s idle, 10 s probe interval, 6 probes) on this requester's connections, so dead or NAT-dropped connections are detected and fail instead of hanging forever. Does not fail slow but alive servers; use read_timeout_in_seconds to bound those.", 3224 examples=[True], 3225 title="Use TCP Keepalive", 3226 ) 3227 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3228 3229 3230class DynamicSchemaLoader(BaseModel): 3231 type: Literal["DynamicSchemaLoader"] 3232 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3233 ..., 3234 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3235 title="Retriever", 3236 ) 3237 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3238 None, 3239 description="Responsible for filtering fields to be added to json schema.", 3240 title="Schema Filter", 3241 ) 3242 schema_transformations: Optional[ 3243 List[ 3244 Union[ 3245 AddFields, 3246 RemoveFields, 3247 KeysToLower, 3248 KeysToSnakeCase, 3249 FlattenFields, 3250 DpathFlattenFields, 3251 KeysReplace, 3252 CustomTransformation, 3253 ] 3254 ] 3255 ] = Field( 3256 None, 3257 description="A list of transformations to be applied to the schema.", 3258 title="Schema Transformations", 3259 ) 3260 schema_type_identifier: SchemaTypeIdentifier 3261 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3262 3263 3264class ParentStreamConfig(BaseModel): 3265 type: Literal["ParentStreamConfig"] 3266 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3267 ..., description="Reference to the parent stream.", title="Parent Stream" 3268 ) 3269 parent_key: str = Field( 3270 ..., 3271 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3272 examples=["id", "{{ config['parent_record_id'] }}"], 3273 title="Parent Key", 3274 ) 3275 partition_field: str = Field( 3276 ..., 3277 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3278 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3279 title="Current Parent Key Value Identifier", 3280 ) 3281 request_option: Optional[RequestOption] = Field( 3282 None, 3283 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3284 title="Request Option", 3285 ) 3286 incremental_dependency: Optional[bool] = Field( 3287 False, 3288 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3289 title="Incremental Dependency", 3290 ) 3291 lazy_read_pointer: Optional[List[str]] = Field( 3292 [], 3293 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3294 title="Lazy Read Pointer", 3295 ) 3296 extra_fields: Optional[List[List[str]]] = Field( 3297 None, 3298 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3299 title="Extra Fields", 3300 ) 3301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3302 3303 3304class PropertiesFromEndpoint(BaseModel): 3305 type: Literal["PropertiesFromEndpoint"] 3306 property_field_path: List[str] = Field( 3307 ..., 3308 description="Describes the path to the field that should be extracted", 3309 examples=[["name"]], 3310 ) 3311 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3312 ..., 3313 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3314 ) 3315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3316 3317 3318class QueryProperties(BaseModel): 3319 type: Literal["QueryProperties"] 3320 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3321 ..., 3322 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3323 title="Property List", 3324 ) 3325 always_include_properties: Optional[List[str]] = Field( 3326 None, 3327 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3328 title="Always Include Properties", 3329 ) 3330 property_chunking: Optional[PropertyChunking] = Field( 3331 None, 3332 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3333 title="Property Chunking", 3334 ) 3335 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3336 None, 3337 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3338 title="Property Selector", 3339 ) 3340 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3341 3342 3343class StateDelegatingStream(BaseModel): 3344 type: Literal["StateDelegatingStream"] 3345 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3346 full_refresh_stream: DeclarativeStream = Field( 3347 ..., 3348 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3349 title="Full Refresh Stream", 3350 ) 3351 incremental_stream: DeclarativeStream = Field( 3352 ..., 3353 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3354 title="Incremental Stream", 3355 ) 3356 api_retention_period: Optional[str] = Field( 3357 None, 3358 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", 3359 examples=["P30D", "P90D", "P1Y"], 3360 title="API Retention Period", 3361 ) 3362 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3363 3364 3365class SimpleRetriever(BaseModel): 3366 type: Literal["SimpleRetriever"] 3367 requester: Union[HttpRequester, CustomRequester] = Field( 3368 ..., 3369 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3370 ) 3371 decoder: Optional[ 3372 Union[ 3373 JsonDecoder, 3374 JsonItemsDecoder, 3375 XmlDecoder, 3376 CsvDecoder, 3377 JsonlDecoder, 3378 GzipDecoder, 3379 IterableDecoder, 3380 ZipfileDecoder, 3381 CustomDecoder, 3382 ] 3383 ] = Field( 3384 None, 3385 description="Component decoding the response so records can be extracted.", 3386 title="HTTP Response Format", 3387 ) 3388 record_selector: RecordSelector = Field( 3389 ..., 3390 description="Component that describes how to extract records from a HTTP response.", 3391 ) 3392 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3393 None, 3394 description="Paginator component that describes how to navigate through the API's pages.", 3395 ) 3396 pagination_reset: Optional[PaginationReset] = Field( 3397 None, 3398 description="Describes what triggers pagination reset and how to handle it.", 3399 ) 3400 page_size_reduction: Optional[PageSizeReduction] = Field( 3401 None, 3402 description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", 3403 ) 3404 request_window_splitting: Optional[RequestWindowSplitting] = Field( 3405 None, 3406 description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page size, and applies even when the API rejects the request before returning a single page.", 3407 ) 3408 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3409 False, 3410 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3411 ) 3412 partition_router: Optional[ 3413 Union[ 3414 SubstreamPartitionRouter, 3415 ListPartitionRouter, 3416 GroupingPartitionRouter, 3417 UnionPartitionRouter, 3418 CustomPartitionRouter, 3419 List[ 3420 Union[ 3421 SubstreamPartitionRouter, 3422 ListPartitionRouter, 3423 GroupingPartitionRouter, 3424 UnionPartitionRouter, 3425 CustomPartitionRouter, 3426 ] 3427 ], 3428 ] 3429 ] = Field( 3430 None, 3431 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3432 title="Partition Router", 3433 ) 3434 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3435 3436 3437class AsyncRetriever(BaseModel): 3438 type: Literal["AsyncRetriever"] 3439 record_selector: RecordSelector = Field( 3440 ..., 3441 description="Component that describes how to extract records from a HTTP response.", 3442 ) 3443 status_mapping: AsyncJobStatusMap = Field( 3444 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3445 ) 3446 status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = Field( 3447 ..., description="Responsible for fetching the actual status of the async job." 3448 ) 3449 download_target_extractor: Optional[ 3450 Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 3451 ] = Field( 3452 None, 3453 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3454 ) 3455 download_extractor: Optional[ 3456 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor] 3457 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3458 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3459 ..., 3460 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3461 ) 3462 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3463 ..., 3464 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3465 ) 3466 polling_job_timeout: Optional[Union[int, str]] = Field( 3467 None, 3468 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3469 ) 3470 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3471 None, 3472 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3473 ge=1, 3474 ) 3475 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3476 None, 3477 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3478 ) 3479 download_requester: Union[HttpRequester, CustomRequester] = Field( 3480 ..., 3481 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3482 ) 3483 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3484 None, 3485 description="Paginator component that describes how to navigate through the API's pages during download.", 3486 ) 3487 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3488 None, 3489 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3490 ) 3491 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3492 None, 3493 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3494 ) 3495 partition_router: Optional[ 3496 Union[ 3497 ListPartitionRouter, 3498 SubstreamPartitionRouter, 3499 GroupingPartitionRouter, 3500 UnionPartitionRouter, 3501 CustomPartitionRouter, 3502 List[ 3503 Union[ 3504 ListPartitionRouter, 3505 SubstreamPartitionRouter, 3506 GroupingPartitionRouter, 3507 UnionPartitionRouter, 3508 CustomPartitionRouter, 3509 ] 3510 ], 3511 ] 3512 ] = Field( 3513 [], 3514 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3515 title="Partition Router", 3516 ) 3517 decoder: Optional[ 3518 Union[ 3519 CsvDecoder, 3520 GzipDecoder, 3521 JsonDecoder, 3522 JsonItemsDecoder, 3523 JsonlDecoder, 3524 IterableDecoder, 3525 XmlDecoder, 3526 ZipfileDecoder, 3527 CustomDecoder, 3528 ] 3529 ] = Field( 3530 None, 3531 description="Component decoding the response so records can be extracted.", 3532 title="HTTP Response Format", 3533 ) 3534 download_decoder: Optional[ 3535 Union[ 3536 CsvDecoder, 3537 GzipDecoder, 3538 JsonDecoder, 3539 JsonItemsDecoder, 3540 JsonlDecoder, 3541 IterableDecoder, 3542 XmlDecoder, 3543 ZipfileDecoder, 3544 CustomDecoder, 3545 ] 3546 ] = Field( 3547 None, 3548 description="Component decoding the download response so records can be extracted.", 3549 title="Download HTTP Response Format", 3550 ) 3551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3552 3553 3554class StreamGroup(BaseModel): 3555 streams: List[DeclarativeStream] = Field( 3556 ..., 3557 description="List of references to streams that belong to this group.\n", 3558 title="Streams", 3559 ) 3560 action: BlockSimultaneousSyncsAction = Field( 3561 ..., description="The action to apply to streams in this group.", title="Action" 3562 ) 3563 3564 3565class SubstreamPartitionRouter(BaseModel): 3566 type: Literal["SubstreamPartitionRouter"] 3567 parent_stream_configs: List[ParentStreamConfig] = Field( 3568 ..., 3569 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3570 title="Parent Stream Configs", 3571 ) 3572 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3573 3574 3575class GroupingPartitionRouter(BaseModel): 3576 type: Literal["GroupingPartitionRouter"] 3577 group_size: int = Field( 3578 ..., 3579 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3580 examples=[10, 50], 3581 title="Group Size", 3582 ) 3583 underlying_partition_router: Union[ 3584 ListPartitionRouter, 3585 SubstreamPartitionRouter, 3586 UnionPartitionRouter, 3587 CustomPartitionRouter, 3588 ] = Field( 3589 ..., 3590 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3591 title="Underlying Partition Router", 3592 ) 3593 deduplicate: Optional[bool] = Field( 3594 True, 3595 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3596 title="Deduplicate Partitions", 3597 ) 3598 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3599 3600 3601class UnionPartitionRouter(BaseModel): 3602 type: Literal["UnionPartitionRouter"] 3603 partition_field: str = Field( 3604 ..., 3605 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3606 examples=["repository", "{{ config['partition_field'] }}"], 3607 title="Partition Field", 3608 ) 3609 partition_routers: List[ 3610 Union[ 3611 ListPartitionRouter, 3612 SubstreamPartitionRouter, 3613 UnionPartitionRouter, 3614 CustomPartitionRouter, 3615 ] 3616 ] = Field( 3617 ..., 3618 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3619 title="Partition Routers", 3620 ) 3621 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3622 3623 3624class HttpComponentsResolver(BaseModel): 3625 type: Literal["HttpComponentsResolver"] 3626 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3627 ..., 3628 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3629 title="Retriever", 3630 ) 3631 components_mapping: List[ComponentMappingDefinition] 3632 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3633 3634 3635class DynamicDeclarativeStream(BaseModel): 3636 type: Literal["DynamicDeclarativeStream"] 3637 name: Optional[str] = Field( 3638 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3639 ) 3640 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3641 ..., description="Reference to the stream template.", title="Stream Template" 3642 ) 3643 components_resolver: Union[ 3644 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3645 ] = Field( 3646 ..., 3647 description="Component resolve and populates stream templates with components values.", 3648 title="Components Resolver", 3649 ) 3650 use_parent_parameters: Optional[bool] = Field( 3651 True, 3652 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3653 title="Use Parent Parameters", 3654 ) 3655 3656 3657ComplexFieldType.update_forward_refs() 3658GzipDecoder.update_forward_refs() 3659CompositeErrorHandler.update_forward_refs() 3660CombinedExtractor.update_forward_refs() 3661DeclarativeSource1.update_forward_refs() 3662DeclarativeSource2.update_forward_refs() 3663SelectiveAuthenticator.update_forward_refs() 3664ConditionalStreams.update_forward_refs() 3665FileUploader.update_forward_refs() 3666DeclarativeStream.update_forward_refs() 3667DpathExtractor.update_forward_refs() 3668RecordExpander.update_forward_refs() 3669SessionTokenAuthenticator.update_forward_refs() 3670HttpRequester.update_forward_refs() 3671DynamicSchemaLoader.update_forward_refs() 3672ParentStreamConfig.update_forward_refs() 3673PropertiesFromEndpoint.update_forward_refs() 3674SimpleRetriever.update_forward_refs() 3675AsyncRetriever.update_forward_refs() 3676GroupingPartitionRouter.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
QuotaStatusSource(pydantic.v1.main.BaseModel):
55class QuotaStatusSource(BaseModel): 56 type: Literal["QuotaStatusSource"] 57 url: str = Field( 58 ..., 59 description="The full URL of the quota status endpoint.", 60 examples=[ 61 "https://api.github.com/rate_limit", 62 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 63 ], 64 title="URL", 65 ) 66 http_method: Optional[HttpMethod] = Field( 67 HttpMethod.GET, 68 description="The HTTP method used to fetch the quota status.", 69 title="HTTP Method", 70 ) 71 request_headers: Optional[Dict[str, str]] = Field( 72 None, 73 description="Additional headers to send with the quota status request.", 74 title="Request Headers", 75 ) 76 unavailable_status_codes: Optional[List[int]] = Field( 77 None, 78 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 79 examples=[[404]], 80 title="Unavailable Status Codes", 81 unique_items=True, 82 ) 83 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
http_method: Optional[HttpMethod]
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
86class DynamicStreamCheckConfig(BaseModel): 87 type: Literal["DynamicStreamCheckConfig"] 88 dynamic_stream_name: str = Field( 89 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 90 ) 91 stream_count: Optional[int] = Field( 92 None, 93 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 94 ge=1, 95 title="Stream Count", 96 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
99class CheckDynamicStream(BaseModel): 100 type: Literal["CheckDynamicStream"] 101 stream_count: int = Field( 102 ..., 103 description="Numbers of the streams to try reading from when running a check operation.", 104 title="Stream Count", 105 ) 106 use_check_availability: Optional[bool] = Field( 107 True, 108 description="Enables stream check availability. This field is automatically set by the CDK.", 109 title="Use Check Availability", 110 ) 111 config_overrides: Optional[Dict[str, Any]] = Field( 112 None, 113 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 114 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 115 title="Config Overrides", 116 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
119class ConcurrencyLevel(BaseModel): 120 type: Optional[Literal["ConcurrencyLevel"]] = None 121 default_concurrency: Union[int, str] = Field( 122 ..., 123 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 124 examples=[10, "{{ config['num_workers'] or 10 }}"], 125 title="Default Concurrency", 126 ) 127 max_concurrency: Optional[int] = Field( 128 None, 129 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 130 examples=[20, 100], 131 title="Max Concurrency", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
136class ConstantBackoffStrategy(BaseModel): 137 type: Literal["ConstantBackoffStrategy"] 138 backoff_time_in_seconds: Union[float, str] = Field( 139 ..., 140 description="Backoff time in seconds.", 141 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 142 title="Backoff Time", 143 ) 144 jitter_range_in_seconds: Optional[float] = Field( 145 None, 146 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 147 examples=[15], 148 ge=0, 149 title="Jitter Range", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
154class CursorPagination(BaseModel): 155 type: Literal["CursorPagination"] 156 cursor_value: str = Field( 157 ..., 158 description="Value of the cursor defining the next page to fetch.", 159 examples=[ 160 "{{ headers.link.next.cursor }}", 161 "{{ last_record['key'] }}", 162 "{{ response['nextPage'] }}", 163 ], 164 title="Cursor Value", 165 ) 166 page_size: Optional[Union[int, str]] = Field( 167 None, 168 description="The number of records to include in each pages.", 169 examples=[100, "{{ config['page_size'] }}"], 170 title="Page Size", 171 ) 172 stop_condition: Optional[str] = Field( 173 None, 174 description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", 175 examples=[ 176 "{{ response.data.has_more is false }}", 177 "{{ 'next' not in headers['link'] }}", 178 "{{ last_page_size < page_size }}", 179 ], 180 title="Stop Condition", 181 ) 182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
185class CustomAuthenticator(BaseModel): 186 class Config: 187 extra = Extra.allow 188 189 type: Literal["CustomAuthenticator"] 190 class_name: str = Field( 191 ..., 192 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 193 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 194 title="Class Name", 195 ) 196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
199class CustomBackoffStrategy(BaseModel): 200 class Config: 201 extra = Extra.allow 202 203 type: Literal["CustomBackoffStrategy"] 204 class_name: str = Field( 205 ..., 206 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 207 examples=["source_railz.components.MyCustomBackoffStrategy"], 208 title="Class Name", 209 ) 210 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
213class CustomErrorHandler(BaseModel): 214 class Config: 215 extra = Extra.allow 216 217 type: Literal["CustomErrorHandler"] 218 class_name: str = Field( 219 ..., 220 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 221 examples=["source_railz.components.MyCustomErrorHandler"], 222 title="Class Name", 223 ) 224 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
227class CustomPaginationStrategy(BaseModel): 228 class Config: 229 extra = Extra.allow 230 231 type: Literal["CustomPaginationStrategy"] 232 class_name: str = Field( 233 ..., 234 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 235 examples=["source_railz.components.MyCustomPaginationStrategy"], 236 title="Class Name", 237 ) 238 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
241class CustomRecordExtractor(BaseModel): 242 class Config: 243 extra = Extra.allow 244 245 type: Literal["CustomRecordExtractor"] 246 class_name: str = Field( 247 ..., 248 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 249 examples=["source_railz.components.MyCustomRecordExtractor"], 250 title="Class Name", 251 ) 252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
255class CustomRecordFilter(BaseModel): 256 class Config: 257 extra = Extra.allow 258 259 type: Literal["CustomRecordFilter"] 260 class_name: str = Field( 261 ..., 262 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 263 examples=["source_railz.components.MyCustomCustomRecordFilter"], 264 title="Class Name", 265 ) 266 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
269class CustomRequester(BaseModel): 270 class Config: 271 extra = Extra.allow 272 273 type: Literal["CustomRequester"] 274 class_name: str = Field( 275 ..., 276 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 277 examples=["source_railz.components.MyCustomRecordExtractor"], 278 title="Class Name", 279 ) 280 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
283class CustomRetriever(BaseModel): 284 class Config: 285 extra = Extra.allow 286 287 type: Literal["CustomRetriever"] 288 class_name: str = Field( 289 ..., 290 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 291 examples=["source_railz.components.MyCustomRetriever"], 292 title="Class Name", 293 ) 294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
297class CustomPartitionRouter(BaseModel): 298 class Config: 299 extra = Extra.allow 300 301 type: Literal["CustomPartitionRouter"] 302 class_name: str = Field( 303 ..., 304 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 305 examples=["source_railz.components.MyCustomPartitionRouter"], 306 title="Class Name", 307 ) 308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
311class CustomSchemaLoader(BaseModel): 312 class Config: 313 extra = Extra.allow 314 315 type: Literal["CustomSchemaLoader"] 316 class_name: str = Field( 317 ..., 318 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 319 examples=["source_railz.components.MyCustomSchemaLoader"], 320 title="Class Name", 321 ) 322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
325class CustomSchemaNormalization(BaseModel): 326 class Config: 327 extra = Extra.allow 328 329 type: Literal["CustomSchemaNormalization"] 330 class_name: str = Field( 331 ..., 332 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 333 examples=[ 334 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 335 ], 336 title="Class Name", 337 ) 338 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
341class CustomStateMigration(BaseModel): 342 class Config: 343 extra = Extra.allow 344 345 type: Literal["CustomStateMigration"] 346 class_name: str = Field( 347 ..., 348 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 349 examples=["source_railz.components.MyCustomStateMigration"], 350 title="Class Name", 351 ) 352 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
355class CustomTransformation(BaseModel): 356 class Config: 357 extra = Extra.allow 358 359 type: Literal["CustomTransformation"] 360 class_name: str = Field( 361 ..., 362 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 363 examples=["source_railz.components.MyCustomTransformation"], 364 title="Class Name", 365 ) 366 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
386class Algorithm(Enum): 387 HS256 = "HS256" 388 HS384 = "HS384" 389 HS512 = "HS512" 390 ES256 = "ES256" 391 ES256K = "ES256K" 392 ES384 = "ES384" 393 ES512 = "ES512" 394 RS256 = "RS256" 395 RS384 = "RS384" 396 RS512 = "RS512" 397 PS256 = "PS256" 398 PS384 = "PS384" 399 PS512 = "PS512" 400 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
403class JwtHeaders(BaseModel): 404 class Config: 405 extra = Extra.forbid 406 407 kid: Optional[str] = Field( 408 None, 409 description="Private key ID for user account.", 410 examples=["{{ config['kid'] }}"], 411 title="Key Identifier", 412 ) 413 typ: Optional[str] = Field( 414 "JWT", 415 description="The media type of the complete JWT.", 416 examples=["JWT"], 417 title="Type", 418 ) 419 cty: Optional[str] = Field( 420 None, 421 description="Content type of JWT header.", 422 examples=["JWT"], 423 title="Content Type", 424 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
427class JwtPayload(BaseModel): 428 class Config: 429 extra = Extra.forbid 430 431 iss: Optional[str] = Field( 432 None, 433 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 434 examples=["{{ config['iss'] }}"], 435 title="Issuer", 436 ) 437 sub: Optional[str] = Field( 438 None, 439 description="The subject of the JWT. Commonly defined by the API.", 440 title="Subject", 441 ) 442 aud: Optional[str] = Field( 443 None, 444 description="The recipient that the JWT is intended for. Commonly defined by the API.", 445 examples=["appstoreconnect-v1"], 446 title="Audience", 447 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
450class RefreshTokenUpdater(BaseModel): 451 refresh_token_name: Optional[str] = Field( 452 "refresh_token", 453 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 454 examples=["refresh_token"], 455 title="Refresh Token Property Name", 456 ) 457 access_token_config_path: Optional[List[str]] = Field( 458 ["credentials", "access_token"], 459 description="Config path to the access token. Make sure the field actually exists in the config.", 460 examples=[["credentials", "access_token"], ["access_token"]], 461 title="Config Path To Access Token", 462 ) 463 refresh_token_config_path: Optional[List[str]] = Field( 464 ["credentials", "refresh_token"], 465 description="Config path to the access token. Make sure the field actually exists in the config.", 466 examples=[["credentials", "refresh_token"], ["refresh_token"]], 467 title="Config Path To Refresh Token", 468 ) 469 token_expiry_date_config_path: Optional[List[str]] = Field( 470 ["credentials", "token_expiry_date"], 471 description="Config path to the expiry date. Make sure actually exists in the config.", 472 examples=[["credentials", "token_expiry_date"]], 473 title="Config Path To Expiry Date", 474 ) 475 refresh_token_error_status_codes: Optional[List[int]] = Field( 476 [], 477 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 478 examples=[[400, 500]], 479 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 480 ) 481 refresh_token_error_key: Optional[str] = Field( 482 "", 483 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 484 examples=["error"], 485 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 486 ) 487 refresh_token_error_values: Optional[List[str]] = Field( 488 [], 489 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 490 examples=[["invalid_grant", "invalid_permissions"]], 491 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 492 )
class
Rate(pydantic.v1.main.BaseModel):
495class Rate(BaseModel): 496 class Config: 497 extra = Extra.allow 498 499 limit: Union[int, str] = Field( 500 ..., 501 description="The maximum number of calls allowed within the interval.", 502 title="Limit", 503 ) 504 interval: str = Field( 505 ..., 506 description="The time interval for the rate limit.", 507 examples=["PT1H", "P1D"], 508 title="Interval", 509 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
512class HttpRequestRegexMatcher(BaseModel): 513 class Config: 514 extra = Extra.allow 515 516 method: Optional[str] = Field( 517 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 518 ) 519 url_base: Optional[str] = Field( 520 None, 521 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 522 title="URL Base", 523 ) 524 url_path_pattern: Optional[str] = Field( 525 None, 526 description="A regular expression pattern to match the URL path.", 527 title="URL Path Pattern", 528 ) 529 params: Optional[Dict[str, Any]] = Field( 530 None, description="The query parameters to match.", title="Parameters" 531 ) 532 headers: Optional[Dict[str, Any]] = Field( 533 None, description="The headers to match.", title="Headers" 534 ) 535 weight: Optional[Union[int, str]] = Field( 536 None, 537 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", 538 title="Weight", 539 )
class
HttpRequestRegexMatcher.Config:
class
CombineMode(enum.Enum):
union =
<CombineMode.union: 'union'>
first_match =
<CombineMode.first_match: 'first_match'>
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
547class ResponseToFileExtractor(BaseModel): 548 type: Literal["ResponseToFileExtractor"] 549 preserve_na_values: Optional[bool] = Field( 550 False, 551 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 552 title="Preserve NA Values", 553 ) 554 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ParentFieldPath(pydantic.v1.main.BaseModel):
562class ParentFieldPath(BaseModel): 563 type: Literal["ParentFieldPath"] 564 parent_path: List[str] = Field( 565 ..., 566 description="Path to the value on the record being expanded.", 567 examples=[["url"], ["id"], ["repository", "name"]], 568 title="Parent Path", 569 ) 570 record_path: List[str] = Field( 571 ..., 572 description="Path on the expanded item to write the value to. An existing value is overwritten. Intermediate objects are created as needed, so a path may not pass through a value the item already holds as a scalar, and a numeric path segment creates an array rather than an object - the same behavior as `AddFields`.", 573 examples=[["pull_request_url"], ["comment_id"], ["parent", "id"]], 574 title="Record Path", 575 ) 576 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
579class ExponentialBackoffStrategy(BaseModel): 580 type: Literal["ExponentialBackoffStrategy"] 581 factor: Optional[Union[float, str]] = Field( 582 5, 583 description="Multiplicative constant applied on each retry.", 584 examples=[5, 5.5, "10"], 585 title="Factor", 586 ) 587 jitter_range_in_seconds: Optional[float] = Field( 588 None, 589 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 590 examples=[2], 591 ge=0, 592 title="Jitter Range", 593 ) 594 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
597class GroupByKeyMergeStrategy(BaseModel): 598 type: Literal["GroupByKeyMergeStrategy"] 599 key: Union[str, List[str]] = Field( 600 ..., 601 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 602 examples=["id", ["parent_id", "end_date"]], 603 title="Key", 604 ) 605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
Action(enum.Enum):
612class Action(Enum): 613 SUCCESS = "SUCCESS" 614 FAIL = "FAIL" 615 RETRY = "RETRY" 616 IGNORE = "IGNORE" 617 RESET_PAGINATION = "RESET_PAGINATION" 618 RATE_LIMITED = "RATE_LIMITED" 619 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 620 REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" 621 SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
REDUCE_PAGE_SIZE =
<Action.REDUCE_PAGE_SIZE: 'REDUCE_PAGE_SIZE'>
SPLIT_REQUEST_WINDOW =
<Action.SPLIT_REQUEST_WINDOW: 'SPLIT_REQUEST_WINDOW'>
class
FailureType(enum.Enum):
624class FailureType(Enum): 625 system_error = "system_error" 626 config_error = "config_error" 627 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
630class HttpResponseFilter(BaseModel): 631 type: Literal["HttpResponseFilter"] 632 action: Optional[Action] = Field( 633 None, 634 description="Action to execute if a response matches the filter.", 635 examples=[ 636 "SUCCESS", 637 "FAIL", 638 "RETRY", 639 "IGNORE", 640 "RESET_PAGINATION", 641 "RATE_LIMITED", 642 "REFRESH_TOKEN_THEN_RETRY", 643 "REDUCE_PAGE_SIZE", 644 "SPLIT_REQUEST_WINDOW", 645 ], 646 title="Action", 647 ) 648 failure_type: Optional[FailureType] = Field( 649 None, 650 description="Failure type of traced exception if a response matches the filter.", 651 examples=["system_error", "config_error", "transient_error"], 652 title="Failure Type", 653 ) 654 error_message: Optional[str] = Field( 655 None, 656 description="Error Message to display if the response matches the filter.", 657 title="Error Message", 658 ) 659 error_message_contains: Optional[str] = Field( 660 None, 661 description="Match the response if its error message contains the substring.", 662 example=["This API operation is not enabled for this site"], 663 title="Error Message Substring", 664 ) 665 http_codes: Optional[List[int]] = Field( 666 None, 667 description="Match the response if its HTTP code is included in this list.", 668 examples=[[420, 429], [500]], 669 title="HTTP Codes", 670 unique_items=True, 671 ) 672 predicate: Optional[str] = Field( 673 None, 674 description="Match the response if the predicate evaluates to true.", 675 examples=[ 676 "{{ 'Too much requests' in response }}", 677 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 678 ], 679 title="Predicate", 680 ) 681 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
684class ComplexFieldType(BaseModel): 685 field_type: str 686 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
689class TypesMap(BaseModel): 690 target_type: Union[str, List[str], ComplexFieldType] 691 current_type: Union[str, List[str]] 692 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
695class SchemaTypeIdentifier(BaseModel): 696 type: Optional[Literal["SchemaTypeIdentifier"]] = None 697 schema_pointer: Optional[List[str]] = Field( 698 [], 699 description="List of nested fields defining the schema field path to extract. Defaults to [].", 700 title="Schema Path", 701 ) 702 key_pointer: List[str] = Field( 703 ..., 704 description="List of potentially nested fields describing the full path of the field key to extract.", 705 title="Key Path", 706 ) 707 type_pointer: Optional[List[str]] = Field( 708 None, 709 description="List of potentially nested fields describing the full path of the field type to extract.", 710 title="Type Path", 711 ) 712 types_mapping: Optional[List[TypesMap]] = None 713 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
716class InlineSchemaLoader(BaseModel): 717 type: Literal["InlineSchemaLoader"] 718 schema_: Optional[Dict[str, Any]] = Field( 719 None, 720 alias="schema", 721 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 722 title="Schema", 723 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
726class JsonFileSchemaLoader(BaseModel): 727 type: Literal["JsonFileSchemaLoader"] 728 file_path: Optional[str] = Field( 729 None, 730 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 731 example=["./schemas/users.json"], 732 title="File Path", 733 ) 734 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
741class JsonItemsDecoder(BaseModel): 742 type: Literal["JsonItemsDecoder"] 743 items_path: str = Field( 744 ..., 745 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", 746 examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], 747 title="Items Path", 748 ) 749 encoding: Optional[str] = Field( 750 "utf-8", 751 description="Text encoding used to decode the streamed bytes before JSON parsing.", 752 title="Encoding", 753 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
770class FlattenFields(BaseModel): 771 type: Literal["FlattenFields"] 772 flatten_lists: Optional[bool] = Field( 773 True, 774 description="Whether to flatten lists or leave it as is. Default is True.", 775 title="Flatten Lists", 776 ) 777 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
780class KeyTransformation(BaseModel): 781 type: Literal["KeyTransformation"] 782 prefix: Optional[str] = Field( 783 None, 784 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 785 examples=["flattened_"], 786 title="Key Prefix", 787 ) 788 suffix: Optional[str] = Field( 789 None, 790 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 791 examples=["_flattened"], 792 title="Key Suffix", 793 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
796class DpathFlattenFields(BaseModel): 797 type: Literal["DpathFlattenFields"] 798 field_path: List[str] = Field( 799 ..., 800 description="A path to field that needs to be flattened.", 801 examples=[["data"], ["data", "*", "field"]], 802 title="Field Path", 803 ) 804 delete_origin_value: Optional[bool] = Field( 805 None, 806 description="Whether to delete the origin value or keep it. Default is False.", 807 title="Delete Origin Value", 808 ) 809 replace_record: Optional[bool] = Field( 810 None, 811 description="Whether to replace the origin record or not. Default is False.", 812 title="Replace Origin Record", 813 ) 814 key_transformation: Optional[KeyTransformation] = Field( 815 None, 816 description="Transformation for object keys. If not provided, original key will be used.", 817 title="Key transformation", 818 ) 819 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
822class KeysReplace(BaseModel): 823 type: Literal["KeysReplace"] 824 old: str = Field( 825 ..., 826 description="Old value to replace.", 827 examples=[ 828 " ", 829 "{{ record.id }}", 830 "{{ config['id'] }}", 831 "{{ stream_slice['id'] }}", 832 ], 833 title="Old value", 834 ) 835 new: str = Field( 836 ..., 837 description="New value to set.", 838 examples=[ 839 "_", 840 "{{ record.id }}", 841 "{{ config['id'] }}", 842 "{{ stream_slice['id'] }}", 843 ], 844 title="New value", 845 ) 846 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
857class CustomDecoder(BaseModel): 858 class Config: 859 extra = Extra.allow 860 861 type: Literal["CustomDecoder"] 862 class_name: str = Field( 863 ..., 864 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 865 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 866 title="Class Name", 867 ) 868 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
871class MinMaxDatetime(BaseModel): 872 type: Literal["MinMaxDatetime"] 873 datetime: str = Field( 874 ..., 875 description="Datetime value.", 876 examples=[ 877 "2021-01-01", 878 "2021-01-01T00:00:00Z", 879 "{{ config['start_time'] }}", 880 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 881 ], 882 title="Datetime", 883 ) 884 datetime_format: Optional[str] = Field( 885 "", 886 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 887 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 888 title="Datetime Format", 889 ) 890 max_datetime: Optional[str] = Field( 891 None, 892 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 893 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 894 title="Max Datetime", 895 ) 896 min_datetime: Optional[str] = Field( 897 None, 898 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 899 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 900 title="Min Datetime", 901 ) 902 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
Scope(pydantic.v1.main.BaseModel):
class
Scope.Config:
class
OptionalScope(pydantic.v1.main.BaseModel):
class
OptionalScope.Config:
class
ScopesJoinStrategy(enum.Enum):
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
942class OauthConnectorInputSpecification(BaseModel): 943 class Config: 944 extra = Extra.allow 945 946 consent_url: str = Field( 947 ..., 948 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 949 examples=[ 950 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 951 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 952 ], 953 title="Consent URL", 954 ) 955 scope: Optional[str] = Field( 956 None, 957 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 958 examples=["user:read user:read_orders workspaces:read"], 959 title="Scopes", 960 ) 961 scopes: Optional[List[Scope]] = Field( 962 None, 963 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 964 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 965 title="Scopes", 966 ) 967 optional_scopes: Optional[List[OptionalScope]] = Field( 968 None, 969 description="Optional OAuth scope objects that may or may not be granted.", 970 examples=[[{"scope": "admin:read"}]], 971 title="Optional Scopes", 972 ) 973 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 974 ScopesJoinStrategy.space, 975 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 976 title="Scopes Join Strategy", 977 ) 978 access_token_url: str = Field( 979 ..., 980 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 981 examples=[ 982 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 983 ], 984 title="Access Token URL", 985 ) 986 access_token_headers: Optional[Dict[str, Any]] = Field( 987 None, 988 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 989 examples=[ 990 { 991 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 992 } 993 ], 994 title="Access Token Headers", 995 ) 996 access_token_params: Optional[Dict[str, Any]] = Field( 997 None, 998 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 999 examples=[ 1000 { 1001 "{{ auth_code_key }}": "{{ auth_code_value }}", 1002 "{{ client_id_key }}": "{{ client_id_value }}", 1003 "{{ client_secret_key }}": "{{ client_secret_value }}", 1004 } 1005 ], 1006 title="Access Token Query Params (Json Encoded)", 1007 ) 1008 extract_output: Optional[List[str]] = Field( 1009 None, 1010 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1011 examples=[["access_token", "refresh_token", "other_field"]], 1012 title="Extract Output", 1013 ) 1014 state: Optional[State] = Field( 1015 None, 1016 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1017 examples=[{"min": 7, "max": 128}], 1018 title="Configurable State Query Param", 1019 ) 1020 client_id_key: Optional[str] = Field( 1021 None, 1022 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1023 examples=["my_custom_client_id_key_name"], 1024 title="Client ID Key Override", 1025 ) 1026 client_secret_key: Optional[str] = Field( 1027 None, 1028 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1029 examples=["my_custom_client_secret_key_name"], 1030 title="Client Secret Key Override", 1031 ) 1032 scope_key: Optional[str] = Field( 1033 None, 1034 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1035 examples=["my_custom_scope_key_key_name"], 1036 title="Scopes Key Override", 1037 ) 1038 state_key: Optional[str] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1041 examples=["my_custom_state_key_key_name"], 1042 title="State Key Override", 1043 ) 1044 auth_code_key: Optional[str] = Field( 1045 None, 1046 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1047 examples=["my_custom_auth_code_key_name"], 1048 title="Auth Code Key Override", 1049 ) 1050 redirect_uri_key: Optional[str] = Field( 1051 None, 1052 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1053 examples=["my_custom_redirect_uri_key_name"], 1054 title="Redirect URI Key Override", 1055 )
scopes: Optional[List[Scope]]
optional_scopes: Optional[List[OptionalScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1058class OAuthConfigSpecification(BaseModel): 1059 class Config: 1060 extra = Extra.allow 1061 1062 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1063 None, 1064 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1065 examples=[ 1066 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1067 { 1068 "app_id": { 1069 "type": "string", 1070 "path_in_connector_config": ["info", "app_id"], 1071 } 1072 }, 1073 ], 1074 title="OAuth user input", 1075 ) 1076 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1077 None, 1078 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1079 title="DeclarativeOAuth Connector Specification", 1080 ) 1081 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1082 None, 1083 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1084 examples=[ 1085 { 1086 "refresh_token": { 1087 "type": "string,", 1088 "path_in_connector_config": ["credentials", "refresh_token"], 1089 } 1090 } 1091 ], 1092 title="OAuth output specification", 1093 ) 1094 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1095 None, 1096 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1097 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1098 title="OAuth input specification", 1099 ) 1100 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1101 None, 1102 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1103 examples=[ 1104 { 1105 "client_id": { 1106 "type": "string,", 1107 "path_in_connector_config": ["credentials", "client_id"], 1108 }, 1109 "client_secret": { 1110 "type": "string,", 1111 "path_in_connector_config": ["credentials", "client_secret"], 1112 }, 1113 } 1114 ], 1115 title="OAuth server output specification", 1116 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1119class OffsetIncrement(BaseModel): 1120 type: Literal["OffsetIncrement"] 1121 page_size: Optional[Union[int, str]] = Field( 1122 None, 1123 description="The number of records to include in each pages.", 1124 examples=[100, "{{ config['page_size'] }}"], 1125 title="Limit", 1126 ) 1127 inject_on_first_request: Optional[bool] = Field( 1128 False, 1129 description="Using the `offset` with value `0` during the first request", 1130 title="Inject Offset on First Request", 1131 ) 1132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1135class PageIncrement(BaseModel): 1136 type: Literal["PageIncrement"] 1137 page_size: Optional[Union[int, str]] = Field( 1138 None, 1139 description="The number of records to include in each pages.", 1140 examples=[100, "100", "{{ config['page_size'] }}"], 1141 title="Page Size", 1142 ) 1143 start_from_page: Optional[int] = Field( 1144 0, 1145 description="Index of the first page to request.", 1146 examples=[0, 1], 1147 title="Start From Page", 1148 ) 1149 inject_on_first_request: Optional[bool] = Field( 1150 False, 1151 description="Using the `page number` with value defined by `start_from_page` during the first request", 1152 title="Inject Page Number on First Request", 1153 ) 1154 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1157class PrimaryKey(BaseModel): 1158 __root__: Union[str, List[str], List[List[str]]] = Field( 1159 ..., 1160 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1161 examples=["id", ["code", "type"]], 1162 title="Primary Key", 1163 )
class
PropertyLimitType(enum.Enum):
1166class PropertyLimitType(Enum): 1167 characters = "characters" 1168 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1171class PropertyChunking(BaseModel): 1172 type: Literal["PropertyChunking"] 1173 property_limit_type: PropertyLimitType = Field( 1174 ..., 1175 description="The type used to determine the maximum number of properties per chunk", 1176 title="Property Limit Type", 1177 ) 1178 property_limit: Optional[int] = Field( 1179 None, 1180 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1181 title="Property Limit", 1182 ) 1183 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1184 None, 1185 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1186 title="Record Merge Strategy", 1187 ) 1188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1191class RecordFilter(BaseModel): 1192 type: Literal["RecordFilter"] 1193 condition: Optional[str] = Field( 1194 "", 1195 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1196 examples=[ 1197 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1198 "{{ record.status in ['active', 'expired'] }}", 1199 ], 1200 title="Condition", 1201 ) 1202 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1210class RemoveFields(BaseModel): 1211 type: Literal["RemoveFields"] 1212 condition: Optional[str] = Field( 1213 "", 1214 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1215 examples=[ 1216 "{{ property|string == '' }}", 1217 "{{ property is integer }}", 1218 "{{ property|length > 5 }}", 1219 "{{ property == 'some_string_to_match' }}", 1220 ], 1221 ) 1222 field_pointers: List[List[str]] = Field( 1223 ..., 1224 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1225 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1226 title="Field Paths", 1227 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1234class InjectInto(Enum): 1235 request_parameter = "request_parameter" 1236 header = "header" 1237 body_data = "body_data" 1238 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1241class RequestOption(BaseModel): 1242 type: Literal["RequestOption"] 1243 inject_into: InjectInto = Field( 1244 ..., 1245 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1246 examples=["request_parameter", "header", "body_data", "body_json"], 1247 title="Inject Into", 1248 ) 1249 field_name: Optional[str] = Field( 1250 None, 1251 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1252 examples=["segment_id"], 1253 title="Field Name", 1254 ) 1255 field_path: Optional[List[str]] = Field( 1256 None, 1257 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1258 examples=[["data", "viewer", "id"]], 1259 title="Field Path", 1260 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1270class LegacySessionTokenAuthenticator(BaseModel): 1271 type: Literal["LegacySessionTokenAuthenticator"] 1272 header: str = Field( 1273 ..., 1274 description="The name of the session token header that will be injected in the request", 1275 examples=["X-Session"], 1276 title="Session Request Header", 1277 ) 1278 login_url: str = Field( 1279 ..., 1280 description="Path of the login URL (do not include the base URL)", 1281 examples=["session"], 1282 title="Login Path", 1283 ) 1284 session_token: Optional[str] = Field( 1285 None, 1286 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1287 example=["{{ config['session_token'] }}"], 1288 title="Session Token", 1289 ) 1290 session_token_response_key: str = Field( 1291 ..., 1292 description="Name of the key of the session token to be extracted from the response", 1293 examples=["id"], 1294 title="Response Token Response Key", 1295 ) 1296 username: Optional[str] = Field( 1297 None, 1298 description="Username used to authenticate and obtain a session token", 1299 examples=[" {{ config['username'] }}"], 1300 title="Username", 1301 ) 1302 password: Optional[str] = Field( 1303 "", 1304 description="Password used to authenticate and obtain a session token", 1305 examples=["{{ config['password'] }}", ""], 1306 title="Password", 1307 ) 1308 validate_session_url: str = Field( 1309 ..., 1310 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1311 examples=["user/current"], 1312 title="Validate Session Path", 1313 ) 1314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
ResetPolicy(enum.Enum):
1327class ResetPolicy(Enum): 1328 NEVER = "NEVER" 1329 AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE"
NEVER =
<ResetPolicy.NEVER: 'NEVER'>
AFTER_SUCCESSFUL_PAGE =
<ResetPolicy.AFTER_SUCCESSFUL_PAGE: 'AFTER_SUCCESSFUL_PAGE'>
class
PageSizeReduction(pydantic.v1.main.BaseModel):
1332class PageSizeReduction(BaseModel): 1333 type: Literal["PageSizeReduction"] 1334 reduction_factor: Optional[float] = Field( 1335 2, 1336 description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", 1337 examples=[2, 4], 1338 gt=1, 1339 title="Reduction Factor", 1340 ) 1341 minimum_page_size: Optional[int] = Field( 1342 1, 1343 description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", 1344 examples=[1, 10], 1345 ge=1, 1346 title="Minimum Page Size", 1347 ) 1348 max_attempts: Optional[int] = Field( 1349 5, 1350 description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", 1351 examples=[5, 10], 1352 ge=1, 1353 title="Maximum Reduction Attempts", 1354 ) 1355 backoff_seconds: Optional[float] = Field( 1356 0.5, 1357 description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', 1358 examples=[0.5, 5], 1359 ge=0, 1360 title="Backoff Seconds", 1361 ) 1362 retries_at_minimum_page_size: Optional[int] = Field( 1363 0, 1364 description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", 1365 examples=[0, 3], 1366 ge=0, 1367 title="Retries At Minimum Page Size", 1368 ) 1369 failure_message: Optional[str] = Field( 1370 None, 1371 description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", 1372 examples=[ 1373 "Narrow the sync down by selecting fewer fields on this stream.", 1374 "Set a more recent start date so that each page covers less data.", 1375 ], 1376 title="Failure Message", 1377 ) 1378 reset_policy: Optional[ResetPolicy] = Field( 1379 ResetPolicy.NEVER, 1380 description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", 1381 title="Reset Policy", 1382 )
reset_policy: Optional[ResetPolicy]
class
RequestWindowSplitting(pydantic.v1.main.BaseModel):
1385class RequestWindowSplitting(BaseModel): 1386 type: Literal["RequestWindowSplitting"] 1387 failure_message: Optional[str] = Field( 1388 None, 1389 description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", 1390 examples=[ 1391 "Narrow the sync down by selecting fewer fields on this stream.", 1392 "Lower time_window so that each request covers less data.", 1393 ], 1394 title="Failure Message", 1395 ) 1396 min_split_window: Optional[str] = Field( 1397 None, 1398 description="Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or below this size is not split further, though the result can end up a bit smaller than this, not larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", 1399 examples=["P1D", "PT1H"], 1400 title="Minimum Split Window", 1401 )
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
1411class AsyncJobStatusMap(BaseModel): 1412 type: Optional[Literal["AsyncJobStatusMap"]] = None 1413 running: List[str] 1414 completed: List[str] 1415 failed: List[str] 1416 timeout: List[str] 1417 skipped: Optional[List[str]] = Field( 1418 None, 1419 description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", 1420 )
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1427class ValueType(Enum): 1428 string = "string" 1429 number = "number" 1430 integer = "integer" 1431 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1434class WaitTimeFromHeader(BaseModel): 1435 type: Literal["WaitTimeFromHeader"] 1436 header: str = Field( 1437 ..., 1438 description="The name of the response header defining how long to wait before retrying.", 1439 examples=["Retry-After"], 1440 title="Response Header Name", 1441 ) 1442 regex: Optional[str] = Field( 1443 None, 1444 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1445 examples=["([-+]?\\d+)"], 1446 title="Extraction Regex", 1447 ) 1448 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1449 None, 1450 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1451 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1452 title="Max Waiting Time in Seconds", 1453 ) 1454 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1457class WaitUntilTimeFromHeader(BaseModel): 1458 type: Literal["WaitUntilTimeFromHeader"] 1459 header: str = Field( 1460 ..., 1461 description="The name of the response header defining how long to wait before retrying.", 1462 examples=["wait_time"], 1463 title="Response Header", 1464 ) 1465 min_wait: Optional[Union[float, str]] = Field( 1466 None, 1467 description="Minimum time to wait before retrying.", 1468 examples=[10, "60"], 1469 title="Minimum Wait Time", 1470 ) 1471 regex: Optional[str] = Field( 1472 None, 1473 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1474 examples=["([-+]?\\d+)"], 1475 title="Extraction Regex", 1476 ) 1477 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1478 None, 1479 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1480 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1481 title="Max Waiting Time in Seconds", 1482 ) 1483 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1486class ComponentMappingDefinition(BaseModel): 1487 type: Literal["ComponentMappingDefinition"] 1488 field_path: List[str] = Field( 1489 ..., 1490 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1491 examples=[ 1492 ["name"], 1493 ["retriever", "requester", "url"], 1494 ["retriever", "requester", "{{ components_values.field }}"], 1495 ["*", "**", "name"], 1496 ], 1497 title="Field Path", 1498 ) 1499 value: str = Field( 1500 ..., 1501 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1502 examples=[ 1503 "{{ components_values['updates'] }}", 1504 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1505 "{{ config['segment_id'] }}", 1506 "{{ stream_slice['parent_id'] }}", 1507 "{{ stream_slice['extra_fields']['name'] }}", 1508 ], 1509 title="Value", 1510 ) 1511 value_type: Optional[ValueType] = Field( 1512 None, 1513 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1514 title="Value Type", 1515 ) 1516 create_or_update: Optional[bool] = Field( 1517 False, 1518 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1519 title="Create or Update", 1520 ) 1521 condition: Optional[str] = Field( 1522 None, 1523 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1524 examples=[ 1525 "{{ components_values.get('cursor_field', None) }}", 1526 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1527 ], 1528 title="Condition", 1529 ) 1530 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1533class StreamConfig(BaseModel): 1534 type: Literal["StreamConfig"] 1535 configs_pointer: List[str] = Field( 1536 ..., 1537 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1538 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1539 title="Configs Pointer", 1540 ) 1541 default_values: Optional[List[Dict[str, Any]]] = Field( 1542 None, 1543 description="A list of default values, each matching the structure expected from the parsed component value.", 1544 title="Default Values", 1545 ) 1546 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1549class ConfigComponentsResolver(BaseModel): 1550 type: Literal["ConfigComponentsResolver"] 1551 stream_config: Union[List[StreamConfig], StreamConfig] 1552 components_mapping: List[ComponentMappingDefinition] 1553 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1556class StreamParametersDefinition(BaseModel): 1557 type: Literal["StreamParametersDefinition"] 1558 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1559 ..., 1560 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1561 examples=[ 1562 [ 1563 { 1564 "name": "test stream", 1565 "$parameters": {"entity": "test entity"}, 1566 "primary_key": "test key", 1567 } 1568 ] 1569 ], 1570 title="Stream Parameters", 1571 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1574class ParametrizedComponentsResolver(BaseModel): 1575 type: Literal["ParametrizedComponentsResolver"] 1576 stream_parameters: StreamParametersDefinition 1577 components_mapping: List[ComponentMappingDefinition] 1578 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1603class ValidateAdheresToSchema(BaseModel): 1604 type: Literal["ValidateAdheresToSchema"] 1605 base_schema: Union[str, Dict[str, Any]] = Field( 1606 ..., 1607 description="The base JSON schema against which the user-provided schema will be validated.", 1608 examples=[ 1609 "{{ config['report_validation_schema'] }}", 1610 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1611 { 1612 "$schema": "http://json-schema.org/draft-07/schema#", 1613 "title": "Person", 1614 "type": "object", 1615 "properties": { 1616 "name": {"type": "string", "description": "The person's name"}, 1617 "age": { 1618 "type": "integer", 1619 "minimum": 0, 1620 "description": "The person's age", 1621 }, 1622 }, 1623 "required": ["name", "age"], 1624 }, 1625 ], 1626 title="Base JSON Schema", 1627 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1630class CustomValidationStrategy(BaseModel): 1631 class Config: 1632 extra = Extra.allow 1633 1634 type: Literal["CustomValidationStrategy"] 1635 class_name: str = Field( 1636 ..., 1637 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1638 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1639 title="Class Name", 1640 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1643class ConfigRemapField(BaseModel): 1644 type: Literal["ConfigRemapField"] 1645 map: Union[Dict[str, Any], str] = Field( 1646 ..., 1647 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1648 examples=[ 1649 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1650 "{{ config['status_mapping'] }}", 1651 ], 1652 title="Value Mapping", 1653 ) 1654 field_path: List[str] = Field( 1655 ..., 1656 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1657 examples=[ 1658 ["status"], 1659 ["data", "status"], 1660 ["data", "{{ config.name }}", "status"], 1661 ["data", "*", "status"], 1662 ], 1663 title="Field Path", 1664 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1667class ConfigRemoveFields(BaseModel): 1668 type: Literal["ConfigRemoveFields"] 1669 field_pointers: List[List[str]] = Field( 1670 ..., 1671 description="A list of field pointers to be removed from the config.", 1672 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1673 title="Field Pointers", 1674 ) 1675 condition: Optional[str] = Field( 1676 "", 1677 description="Fields will be removed if expression is evaluated to True.", 1678 examples=[ 1679 "{{ config['environemnt'] == 'sandbox' }}", 1680 "{{ property is integer }}", 1681 "{{ property|length > 5 }}", 1682 "{{ property == 'some_string_to_match' }}", 1683 ], 1684 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1687class CustomConfigTransformation(BaseModel): 1688 type: Literal["CustomConfigTransformation"] 1689 class_name: str = Field( 1690 ..., 1691 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1692 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1693 ) 1694 parameters: Optional[Dict[str, Any]] = Field( 1695 None, 1696 alias="$parameters", 1697 description="Additional parameters to be passed to the custom config transformation.", 1698 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1701class AddedFieldDefinition(BaseModel): 1702 type: Literal["AddedFieldDefinition"] 1703 path: List[str] = Field( 1704 ..., 1705 description="List of strings defining the path where to add the value on the record.", 1706 examples=[["segment_id"], ["metadata", "segment_id"]], 1707 title="Path", 1708 ) 1709 value: str = Field( 1710 ..., 1711 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1712 examples=[ 1713 "{{ record['updates'] }}", 1714 "{{ record['MetaData']['LastUpdatedTime'] }}", 1715 "{{ stream_partition['segment_id'] }}", 1716 ], 1717 title="Value", 1718 ) 1719 value_type: Optional[ValueType] = Field( 1720 None, 1721 description="Type of the value. If not specified, the type will be inferred from the value.", 1722 title="Value Type", 1723 ) 1724 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1727class AddFields(BaseModel): 1728 type: Literal["AddFields"] 1729 fields: List[AddedFieldDefinition] = Field( 1730 ..., 1731 description="List of transformations (path and corresponding value) that will be added to the record.", 1732 title="Fields", 1733 ) 1734 condition: Optional[str] = Field( 1735 "", 1736 description="Fields will be added if expression is evaluated to True.", 1737 examples=[ 1738 "{{ property|string == '' }}", 1739 "{{ property is integer }}", 1740 "{{ property|length > 5 }}", 1741 "{{ property == 'some_string_to_match' }}", 1742 ], 1743 ) 1744 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1747class ApiKeyAuthenticator(BaseModel): 1748 type: Literal["ApiKeyAuthenticator"] 1749 api_token: Optional[str] = Field( 1750 None, 1751 description="The API key to inject in the request. Fill it in the user inputs.", 1752 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1753 title="API Key", 1754 ) 1755 header: Optional[str] = Field( 1756 None, 1757 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1758 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1759 title="Header Name", 1760 ) 1761 inject_into: Optional[RequestOption] = Field( 1762 None, 1763 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1764 examples=[ 1765 {"inject_into": "header", "field_name": "Authorization"}, 1766 {"inject_into": "request_parameter", "field_name": "authKey"}, 1767 ], 1768 title="Inject API Key Into Outgoing HTTP Request", 1769 ) 1770 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1773class AuthFlow(BaseModel): 1774 auth_flow_type: Optional[AuthFlowType] = Field( 1775 None, description="The type of auth to use", title="Auth flow type" 1776 ) 1777 predicate_key: Optional[List[str]] = Field( 1778 None, 1779 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1780 examples=[["credentials", "auth_type"]], 1781 title="Predicate key", 1782 ) 1783 predicate_value: Optional[str] = Field( 1784 None, 1785 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1786 examples=["Oauth"], 1787 title="Predicate value", 1788 ) 1789 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
TokenQuota(pydantic.v1.main.BaseModel):
1792class TokenQuota(BaseModel): 1793 type: Literal["TokenQuota"] 1794 name: str = Field( 1795 ..., 1796 description="Name of the quota pool.", 1797 examples=["rest", "graphql"], 1798 title="Name", 1799 ) 1800 remaining_path: List[str] = Field( 1801 ..., 1802 description="Path to the remaining call count for this pool in the quota status response.", 1803 examples=[["resources", "core", "remaining"]], 1804 title="Remaining Path", 1805 ) 1806 reset_path: List[str] = Field( 1807 ..., 1808 description="Path to the quota reset timestamp for this pool in the quota status response.", 1809 examples=[["resources", "core", "reset"]], 1810 title="Reset Path", 1811 ) 1812 limit_path: Optional[List[str]] = Field( 1813 None, 1814 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 1815 examples=[["resources", "core", "limit"]], 1816 title="Limit Path", 1817 ) 1818 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 1819 None, 1820 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 1821 title="Matchers", 1822 ) 1823 remaining_header: Optional[str] = Field( 1824 None, 1825 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 1826 examples=["X-RateLimit-Remaining"], 1827 title="Remaining Header", 1828 ) 1829 reset_header: Optional[str] = Field( 1830 None, 1831 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 1832 examples=["X-RateLimit-Reset"], 1833 title="Reset Header", 1834 ) 1835 limit_header: Optional[str] = Field( 1836 None, 1837 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 1838 examples=["X-RateLimit-Limit"], 1839 title="Limit Header", 1840 ) 1841 exhaustion_status_codes: Optional[List[int]] = Field( 1842 None, 1843 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 1844 examples=[[429]], 1845 title="Exhaustion Status Codes", 1846 ) 1847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
matchers: Optional[List[HttpRequestRegexMatcher]]
class
CheckStream(pydantic.v1.main.BaseModel):
1850class CheckStream(BaseModel): 1851 type: Literal["CheckStream"] 1852 stream_names: Optional[List[str]] = Field( 1853 None, 1854 description="Names of the streams to try reading from when running a check operation.", 1855 examples=[["users"], ["users", "contacts"]], 1856 title="Stream Names", 1857 ) 1858 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1859 config_overrides: Optional[Dict[str, Any]] = Field( 1860 None, 1861 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1862 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1863 title="Config Overrides", 1864 )
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1867class IncrementingCountCursor(BaseModel): 1868 type: Literal["IncrementingCountCursor"] 1869 cursor_field: str = Field( 1870 ..., 1871 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1872 examples=["created_at", "{{ config['record_cursor'] }}"], 1873 title="Cursor Field", 1874 ) 1875 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1876 None, 1877 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1878 title="Allow Catalog Defined Cursor Field", 1879 ) 1880 start_value: Optional[Union[str, int]] = Field( 1881 None, 1882 description="The value that determines the earliest record that should be synced.", 1883 examples=[0, "{{ config['start_value'] }}"], 1884 title="Start Value", 1885 ) 1886 start_value_option: Optional[RequestOption] = Field( 1887 None, 1888 description="Optionally configures how the start value will be sent in requests to the source API.", 1889 title="Inject Start Value Into Outgoing HTTP Request", 1890 ) 1891 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1894class DatetimeBasedCursor(BaseModel): 1895 type: Literal["DatetimeBasedCursor"] 1896 clamping: Optional[Clamping] = Field( 1897 None, 1898 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1899 title="Date Range Clamping", 1900 ) 1901 cursor_field: str = Field( 1902 ..., 1903 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1904 examples=["created_at", "{{ config['record_cursor'] }}"], 1905 title="Cursor Field", 1906 ) 1907 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1908 None, 1909 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1910 title="Allow Catalog Defined Cursor Field", 1911 ) 1912 cursor_datetime_formats: Optional[List[str]] = Field( 1913 None, 1914 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1915 examples=[ 1916 "%Y-%m-%d", 1917 "%Y-%m-%d %H:%M:%S", 1918 "%Y-%m-%dT%H:%M:%S", 1919 "%Y-%m-%dT%H:%M:%SZ", 1920 "%Y-%m-%dT%H:%M:%S%z", 1921 "%Y-%m-%dT%H:%M:%S.%fZ", 1922 "%Y-%m-%dT%H:%M:%S.%f%z", 1923 "%Y-%m-%d %H:%M:%S.%f+00:00", 1924 "%s", 1925 "%ms", 1926 ], 1927 title="Cursor Datetime Formats", 1928 ) 1929 start_datetime: Union[MinMaxDatetime, str] = Field( 1930 ..., 1931 description="The datetime that determines the earliest record that should be synced.", 1932 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1933 title="Start Datetime", 1934 ) 1935 start_time_option: Optional[RequestOption] = Field( 1936 None, 1937 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1938 title="Inject Start Time Into Outgoing HTTP Request", 1939 ) 1940 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1941 None, 1942 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1943 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1944 title="End Datetime", 1945 ) 1946 end_time_option: Optional[RequestOption] = Field( 1947 None, 1948 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1949 title="Inject End Time Into Outgoing HTTP Request", 1950 ) 1951 datetime_format: str = Field( 1952 ..., 1953 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1954 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1955 title="Outgoing Datetime Format", 1956 ) 1957 cursor_granularity: Optional[str] = Field( 1958 None, 1959 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1960 examples=["PT1S"], 1961 title="Cursor Granularity", 1962 ) 1963 is_data_feed: Optional[bool] = Field( 1964 None, 1965 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1966 title="Data Feed API", 1967 ) 1968 is_client_side_incremental: Optional[bool] = Field( 1969 None, 1970 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1971 title="Client-side Incremental Filtering", 1972 ) 1973 is_compare_strictly: Optional[bool] = Field( 1974 False, 1975 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1976 title="Strict Start-End Time Comparison", 1977 ) 1978 global_substream_cursor: Optional[bool] = Field( 1979 False, 1980 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1981 title="Global Substream Cursor", 1982 ) 1983 lookback_window: Optional[str] = Field( 1984 None, 1985 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1986 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1987 title="Lookback Window", 1988 ) 1989 partition_field_end: Optional[str] = Field( 1990 None, 1991 description="Name of the partition start time field.", 1992 examples=["ending_time"], 1993 title="Partition Field End", 1994 ) 1995 partition_field_start: Optional[str] = Field( 1996 None, 1997 description="Name of the partition end time field.", 1998 examples=["starting_time"], 1999 title="Partition Field Start", 2000 ) 2001 step: Optional[str] = Field( 2002 None, 2003 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2004 examples=["P1W", "{{ config['step_increment'] }}"], 2005 title="Step", 2006 ) 2007 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
2010class JwtAuthenticator(BaseModel): 2011 type: Literal["JwtAuthenticator"] 2012 secret_key: str = Field( 2013 ..., 2014 description="Secret used to sign the JSON web token.", 2015 examples=["{{ config['secret_key'] }}"], 2016 title="Secret Key", 2017 ) 2018 base64_encode_secret_key: Optional[bool] = Field( 2019 False, 2020 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 2021 title="Base64-encode Secret Key", 2022 ) 2023 algorithm: Algorithm = Field( 2024 ..., 2025 description="Algorithm used to sign the JSON web token.", 2026 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 2027 title="Algorithm", 2028 ) 2029 token_duration: Optional[int] = Field( 2030 1200, 2031 description="The amount of time in seconds a JWT token can be valid after being issued.", 2032 examples=[1200, 3600], 2033 title="Token Duration", 2034 ) 2035 header_prefix: Optional[str] = Field( 2036 None, 2037 description="The prefix to be used within the Authentication header.", 2038 examples=["Bearer", "Basic"], 2039 title="Header Prefix", 2040 ) 2041 jwt_headers: Optional[JwtHeaders] = Field( 2042 None, 2043 description="JWT headers used when signing JSON web token.", 2044 title="JWT Headers", 2045 ) 2046 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 2047 None, 2048 description="Additional headers to be included with the JWT headers object.", 2049 title="Additional JWT Headers", 2050 ) 2051 jwt_payload: Optional[JwtPayload] = Field( 2052 None, 2053 description="JWT Payload used when signing JSON web token.", 2054 title="JWT Payload", 2055 ) 2056 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2057 None, 2058 description="Additional properties to be added to the JWT payload.", 2059 title="Additional JWT Payload Properties", 2060 ) 2061 passphrase: Optional[str] = Field( 2062 None, 2063 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2064 examples=["{{ config['passphrase'] }}"], 2065 title="Passphrase", 2066 ) 2067 request_option: Optional[RequestOption] = Field( 2068 None, 2069 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2070 title="Request Option", 2071 ) 2072 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
2075class OAuthAuthenticator(BaseModel): 2076 type: Literal["OAuthAuthenticator"] 2077 client_id_name: Optional[str] = Field( 2078 "client_id", 2079 description="The name of the property to use to refresh the `access_token`.", 2080 examples=["custom_app_id"], 2081 title="Client ID Property Name", 2082 ) 2083 client_id: Optional[str] = Field( 2084 None, 2085 description="The OAuth client ID. Fill it in the user inputs.", 2086 examples=[ 2087 "{{ config['client_id'] }}", 2088 "{{ config['credentials']['client_id }}", 2089 ], 2090 title="Client ID", 2091 ) 2092 client_secret_name: Optional[str] = Field( 2093 "client_secret", 2094 description="The name of the property to use to refresh the `access_token`.", 2095 examples=["custom_app_secret"], 2096 title="Client Secret Property Name", 2097 ) 2098 client_secret: Optional[str] = Field( 2099 None, 2100 description="The OAuth client secret. Fill it in the user inputs.", 2101 examples=[ 2102 "{{ config['client_secret'] }}", 2103 "{{ config['credentials']['client_secret }}", 2104 ], 2105 title="Client Secret", 2106 ) 2107 refresh_token_name: Optional[str] = Field( 2108 "refresh_token", 2109 description="The name of the property to use to refresh the `access_token`.", 2110 examples=["custom_app_refresh_value"], 2111 title="Refresh Token Property Name", 2112 ) 2113 refresh_token: Optional[str] = Field( 2114 None, 2115 description="Credential artifact used to get a new access token.", 2116 examples=[ 2117 "{{ config['refresh_token'] }}", 2118 "{{ config['credentials]['refresh_token'] }}", 2119 ], 2120 title="Refresh Token", 2121 ) 2122 token_refresh_endpoint: Optional[str] = Field( 2123 None, 2124 description="The full URL to call to obtain a new access token.", 2125 examples=["https://connect.squareup.com/oauth2/token"], 2126 title="Token Refresh Endpoint", 2127 ) 2128 access_token_name: Optional[str] = Field( 2129 "access_token", 2130 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2131 examples=["access_token"], 2132 title="Access Token Property Name", 2133 ) 2134 access_token_value: Optional[str] = Field( 2135 None, 2136 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2137 examples=["secret_access_token_value"], 2138 title="Access Token Value", 2139 ) 2140 expires_in_name: Optional[str] = Field( 2141 "expires_in", 2142 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2143 examples=["expires_in"], 2144 title="Token Expiry Property Name", 2145 ) 2146 grant_type_name: Optional[str] = Field( 2147 "grant_type", 2148 description="The name of the property to use to refresh the `access_token`.", 2149 examples=["custom_grant_type"], 2150 title="Grant Type Property Name", 2151 ) 2152 grant_type: Optional[str] = Field( 2153 "refresh_token", 2154 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2155 examples=["refresh_token", "client_credentials"], 2156 title="Grant Type", 2157 ) 2158 refresh_request_body: Optional[Dict[str, Any]] = Field( 2159 None, 2160 description="Body of the request sent to get a new access token.", 2161 examples=[ 2162 { 2163 "applicationId": "{{ config['application_id'] }}", 2164 "applicationSecret": "{{ config['application_secret'] }}", 2165 "token": "{{ config['token'] }}", 2166 } 2167 ], 2168 title="Refresh Request Body", 2169 ) 2170 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2171 None, 2172 description="Headers of the request sent to get a new access token.", 2173 examples=[ 2174 { 2175 "Authorization": "<AUTH_TOKEN>", 2176 "Content-Type": "application/x-www-form-urlencoded", 2177 } 2178 ], 2179 title="Refresh Request Headers", 2180 ) 2181 send_refresh_request_as_query_params: Optional[bool] = Field( 2182 False, 2183 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2184 examples=[True], 2185 title="Send Refresh Request As Query Params", 2186 ) 2187 scopes: Optional[List[str]] = Field( 2188 None, 2189 description="List of scopes that should be granted to the access token.", 2190 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2191 title="Scopes", 2192 ) 2193 token_expiry_date: Optional[str] = Field( 2194 None, 2195 description="The access token expiry date.", 2196 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2197 title="Token Expiry Date", 2198 ) 2199 token_expiry_date_format: Optional[str] = Field( 2200 None, 2201 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2202 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2203 title="Token Expiry Date Format", 2204 ) 2205 refresh_token_error_status_codes: Optional[List[int]] = Field( 2206 None, 2207 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2208 examples=[[400, 500]], 2209 title="Refresh Token Error Status Codes", 2210 ) 2211 refresh_token_error_key: Optional[str] = Field( 2212 None, 2213 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2214 examples=["error"], 2215 title="Refresh Token Error Key", 2216 ) 2217 refresh_token_error_values: Optional[List[str]] = Field( 2218 None, 2219 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2220 examples=[["invalid_grant", "invalid_permissions"]], 2221 title="Refresh Token Error Values", 2222 ) 2223 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2224 None, 2225 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2226 title="Refresh Token Updater", 2227 ) 2228 profile_assertion: Optional[JwtAuthenticator] = Field( 2229 None, 2230 description="The authenticator being used to authenticate the client authenticator.", 2231 title="Profile Assertion", 2232 ) 2233 use_profile_assertion: Optional[bool] = Field( 2234 False, 2235 description="Enable using profile assertion as a flow for OAuth authorization.", 2236 title="Use Profile Assertion", 2237 ) 2238 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2241class FixedWindowCallRatePolicy(BaseModel): 2242 class Config: 2243 extra = Extra.allow 2244 2245 type: Literal["FixedWindowCallRatePolicy"] 2246 period: str = Field( 2247 ..., description="The time interval for the rate limit window.", title="Period" 2248 ) 2249 call_limit: int = Field( 2250 ..., 2251 description="The maximum number of calls allowed within the period.", 2252 title="Call Limit", 2253 ) 2254 matchers: List[HttpRequestRegexMatcher] = Field( 2255 ..., 2256 description="List of matchers that define which requests this policy applies to.", 2257 title="Matchers", 2258 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2261class MovingWindowCallRatePolicy(BaseModel): 2262 class Config: 2263 extra = Extra.allow 2264 2265 type: Literal["MovingWindowCallRatePolicy"] 2266 rates: List[Rate] = Field( 2267 ..., 2268 description="List of rates that define the call limits for different time intervals.", 2269 title="Rates", 2270 ) 2271 matchers: List[HttpRequestRegexMatcher] = Field( 2272 ..., 2273 description="List of matchers that define which requests this policy applies to.", 2274 title="Matchers", 2275 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2278class UnlimitedCallRatePolicy(BaseModel): 2279 class Config: 2280 extra = Extra.allow 2281 2282 type: Literal["UnlimitedCallRatePolicy"] 2283 matchers: List[HttpRequestRegexMatcher] = Field( 2284 ..., 2285 description="List of matchers that define which requests this policy applies to.", 2286 title="Matchers", 2287 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2290class DefaultErrorHandler(BaseModel): 2291 type: Literal["DefaultErrorHandler"] 2292 backoff_strategies: Optional[ 2293 List[ 2294 Union[ 2295 ConstantBackoffStrategy, 2296 ExponentialBackoffStrategy, 2297 WaitTimeFromHeader, 2298 WaitUntilTimeFromHeader, 2299 CustomBackoffStrategy, 2300 ] 2301 ] 2302 ] = Field( 2303 None, 2304 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2305 title="Backoff Strategies", 2306 ) 2307 max_retries: Optional[Union[int, str]] = Field( 2308 5, 2309 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2310 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2311 title="Max Retry Count", 2312 ) 2313 response_filters: Optional[List[HttpResponseFilter]] = Field( 2314 None, 2315 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2316 title="Response Filters", 2317 ) 2318 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2321class DefaultPaginator(BaseModel): 2322 type: Literal["DefaultPaginator"] 2323 pagination_strategy: Union[ 2324 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2325 ] = Field( 2326 ..., 2327 description="Strategy defining how records are paginated.", 2328 title="Pagination Strategy", 2329 ) 2330 page_size_option: Optional[RequestOption] = Field( 2331 None, title="Inject Page Size Into Outgoing HTTP Request" 2332 ) 2333 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2334 None, 2335 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2336 title="Inject Page Token Into Outgoing HTTP Request", 2337 ) 2338 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2341class RecordExpander(BaseModel): 2342 type: Literal["RecordExpander"] 2343 expand_records_from_field: List[str] = Field( 2344 ..., 2345 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2346 examples=[ 2347 ["lines", "data"], 2348 ["items"], 2349 ["nested", "array"], 2350 ["sections", "*", "items"], 2351 ], 2352 title="Expand Records From Field", 2353 ) 2354 remain_original_record: Optional[bool] = Field( 2355 False, 2356 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2357 title="Remain Original Record", 2358 ) 2359 parent_fields: Optional[List[ParentFieldPath]] = Field( 2360 None, 2361 description="Named values to copy from the record being expanded onto each expanded item. Use this instead of `remain_original_record` when only a few parent fields are needed: it copies the named values rather than deep-copying the whole parent once per item, which matters when the parent record is large and the nested list is long. An existing value at `record_path` is overwritten, and a `parent_path` the parent does not have copies null. A copied object or array is deep-copied, so a transformation that writes inside it affects only that item. Independent of `remain_original_record`; both may be set. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2362 title="Parent Fields", 2363 ) 2364 merge_parent: Optional[bool] = Field( 2365 False, 2366 description="If true, each expanded item is the parent record shallow-merged with the item, the item's own keys winning on collision, and the expanded list removed from the parent's copy. Only the value at `expand_records_from_field` is removed, so for a multi-segment path the top-level key stays with its other fields. Each item receives its own deep copy of the merged parent fields, so a transformation that writes into a nested value affects only that item. The merge happens first, then `parent_fields` are copied, then `original_record` is embedded when `remain_original_record` is set; all three may be combined. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2367 title="Merge Parent", 2368 ) 2369 on_no_records: Optional[OnNoRecords] = Field( 2370 OnNoRecords.skip, 2371 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2372 title="On No Records", 2373 ) 2374 truncation_indicator_path: Optional[List[str]] = Field( 2375 None, 2376 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2377 examples=[["data", "object", "lines", "has_more"]], 2378 title="Truncation Indicator Path", 2379 ) 2380 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2381 None, 2382 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2383 title="Truncated List Retriever", 2384 ) 2385 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_fields: Optional[List[ParentFieldPath]]
on_no_records: Optional[OnNoRecords]
truncated_list_retriever: Union[SimpleRetriever, CustomRetriever, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2388class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2389 type: Literal["ApiKey"] 2390 inject_into: RequestOption = Field( 2391 ..., 2392 description="Configure how the API Key will be sent in requests to the source API.", 2393 examples=[ 2394 {"inject_into": "header", "field_name": "Authorization"}, 2395 {"inject_into": "request_parameter", "field_name": "authKey"}, 2396 ], 2397 title="Inject API Key Into Outgoing HTTP Request", 2398 ) 2399 api_token: Optional[str] = Field( 2400 "{{ session_token }}", 2401 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2402 examples=[ 2403 "{{ session_token }}", 2404 "Token {{ session_token }}", 2405 "Bearer {{ session_token }}", 2406 ], 2407 title="API Token Template", 2408 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2411class JsonSchemaPropertySelector(BaseModel): 2412 type: Literal["JsonSchemaPropertySelector"] 2413 transformations: Optional[ 2414 List[ 2415 Union[ 2416 AddFields, 2417 RemoveFields, 2418 KeysToLower, 2419 KeysToSnakeCase, 2420 FlattenFields, 2421 DpathFlattenFields, 2422 KeysReplace, 2423 CustomTransformation, 2424 ] 2425 ] 2426 ] = Field( 2427 None, 2428 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2429 title="Transformations", 2430 ) 2431 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2434class ListPartitionRouter(BaseModel): 2435 type: Literal["ListPartitionRouter"] 2436 cursor_field: str = Field( 2437 ..., 2438 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2439 examples=["section", "{{ config['section_key'] }}"], 2440 title="Current Partition Value Identifier", 2441 ) 2442 values: Union[str, List[str]] = Field( 2443 ..., 2444 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2445 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2446 title="Partition Values", 2447 ) 2448 request_option: Optional[RequestOption] = Field( 2449 None, 2450 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2451 title="Inject Partition Value Into Outgoing HTTP Request", 2452 ) 2453 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2456class PaginationReset(BaseModel): 2457 type: Literal["PaginationReset"] 2458 action: Action1 2459 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2462class GzipDecoder(BaseModel): 2463 type: Literal["GzipDecoder"] 2464 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2467class RequestBodyGraphQL(BaseModel): 2468 type: Literal["RequestBodyGraphQL"] 2469 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2472class DpathValidator(BaseModel): 2473 type: Literal["DpathValidator"] 2474 field_path: List[str] = Field( 2475 ..., 2476 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2477 examples=[ 2478 ["data"], 2479 ["data", "records"], 2480 ["data", "{{ parameters.name }}"], 2481 ["data", "*", "record"], 2482 ], 2483 title="Field Path", 2484 ) 2485 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2486 ..., 2487 description="The condition that the specified config value will be evaluated against", 2488 title="Validation Strategy", 2489 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2492class PredicateValidator(BaseModel): 2493 type: Literal["PredicateValidator"] 2494 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2495 ..., 2496 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2497 examples=[ 2498 "test-value", 2499 "{{ config['api_version'] }}", 2500 "{{ config['tenant_id'] }}", 2501 123, 2502 ], 2503 title="Value", 2504 ) 2505 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2506 ..., 2507 description="The validation strategy to apply to the value.", 2508 title="Validation Strategy", 2509 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2512class ConfigAddFields(BaseModel): 2513 type: Literal["ConfigAddFields"] 2514 fields: List[AddedFieldDefinition] = Field( 2515 ..., 2516 description="A list of transformations (path and corresponding value) that will be added to the config.", 2517 title="Fields", 2518 ) 2519 condition: Optional[str] = Field( 2520 "", 2521 description="Fields will be added if expression is evaluated to True.", 2522 examples=[ 2523 "{{ config['environemnt'] == 'sandbox' }}", 2524 "{{ property is integer }}", 2525 "{{ property|length > 5 }}", 2526 "{{ property == 'some_string_to_match' }}", 2527 ], 2528 )
fields: List[AddedFieldDefinition]
class
RateLimitedMultipleTokenAuthenticator(pydantic.v1.main.BaseModel):
2531class RateLimitedMultipleTokenAuthenticator(BaseModel): 2532 type: Literal["RateLimitedMultipleTokenAuthenticator"] 2533 tokens: Union[str, List[str]] = Field( 2534 ..., 2535 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 2536 examples=[ 2537 "{{ config['credentials']['personal_access_token'] }}", 2538 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 2539 ], 2540 title="Tokens", 2541 ) 2542 token_delimiter: Optional[str] = Field( 2543 ",", 2544 description="Delimiter used to split a single token string into multiple tokens.", 2545 title="Token Delimiter", 2546 ) 2547 auth_method: Optional[str] = Field( 2548 "Bearer", 2549 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 2550 examples=["Bearer", "token"], 2551 title="Auth Method", 2552 ) 2553 header: Optional[str] = Field( 2554 "Authorization", 2555 description="The name of the HTTP header in which to inject the token.", 2556 title="Header Name", 2557 ) 2558 quota_status_source: QuotaStatusSource = Field( 2559 ..., 2560 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 2561 title="Quota Status Source", 2562 ) 2563 quotas: List[TokenQuota] = Field( 2564 ..., 2565 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 2566 min_items=1, 2567 title="Quota Pools", 2568 ) 2569 max_wait_time: Optional[str] = Field( 2570 "PT2H", 2571 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 2572 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 2573 title="Maximum Wait Time", 2574 ) 2575 budget_reserve_fraction: Optional[float] = Field( 2576 0.1, 2577 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 2578 title="Budget Reserve Fraction", 2579 ) 2580 budget_min_reserve: Optional[int] = Field( 2581 50, 2582 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 2583 title="Budget Minimum Reserve", 2584 ) 2585 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
quota_status_source: QuotaStatusSource
quotas: List[TokenQuota]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2588class CompositeErrorHandler(BaseModel): 2589 type: Literal["CompositeErrorHandler"] 2590 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2591 Field( 2592 ..., 2593 description="List of error handlers to iterate on to determine how to handle a failed response.", 2594 title="Error Handlers", 2595 ) 2596 ) 2597 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2600class HTTPAPIBudget(BaseModel): 2601 class Config: 2602 extra = Extra.allow 2603 2604 type: Literal["HTTPAPIBudget"] 2605 policies: List[ 2606 Union[ 2607 FixedWindowCallRatePolicy, 2608 MovingWindowCallRatePolicy, 2609 UnlimitedCallRatePolicy, 2610 ] 2611 ] = Field( 2612 ..., 2613 description="List of call rate policies that define how many calls are allowed.", 2614 title="Policies", 2615 ) 2616 ratelimit_reset_header: Optional[str] = Field( 2617 "ratelimit-reset", 2618 description="The HTTP response header name that indicates when the rate limit resets.", 2619 title="Rate Limit Reset Header", 2620 ) 2621 ratelimit_remaining_header: Optional[str] = Field( 2622 "ratelimit-remaining", 2623 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2624 title="Rate Limit Remaining Header", 2625 ) 2626 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2627 [429], 2628 description="List of HTTP status codes that indicate a rate limit has been hit.", 2629 title="Status Codes for Rate Limit Hit", 2630 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2633class DpathExtractor(BaseModel): 2634 type: Literal["DpathExtractor"] 2635 field_path: List[str] = Field( 2636 ..., 2637 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2638 examples=[ 2639 ["data"], 2640 ["data", "records"], 2641 ["data", "{{ parameters.name }}"], 2642 ["data", "*", "record"], 2643 ], 2644 title="Field Path", 2645 ) 2646 record_expander: Optional[RecordExpander] = Field( 2647 None, 2648 description="Optional component to expand records by extracting items from nested array fields.", 2649 title="Record Expander", 2650 ) 2651 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
CombinedExtractor(pydantic.v1.main.BaseModel):
2654class CombinedExtractor(BaseModel): 2655 type: Literal["CombinedExtractor"] 2656 extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]] = Field( 2657 ..., 2658 description="The record extractors to combine. At least one is required. Each sub-extractor is given the same HTTP response and decodes it independently, so the response is parsed once per sub-extractor. Streaming decoders (CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder, IterableDecoder) can only read the response body once and are rejected.", 2659 title="Extractors", 2660 ) 2661 mode: Optional[CombineMode] = Field( 2662 CombineMode.union, 2663 description='How the records of the sub-extractors are combined. "union" (default) yields every record of every sub-extractor, in the order the extractors are declared. "first_match" yields the records of the first sub-extractor that produces at least one record and skips the remaining ones; nothing is yielded if none of them produces a record. Note that a paginator which counts the records of a page counts the combined records: under "union" that is the sum over all sub-extractors, which overshoots the API page size, so "union" is rejected with an OffsetIncrement paginator because the offset would skip records. Under "first_match" the count is the count of the winning sub-extractor, which is usually the number of records the API returned for the page.', 2664 title="Combine Mode", 2665 ) 2666 skip_empty_records: Optional[bool] = Field( 2667 False, 2668 description='Whether to drop empty records - null, {}, [], "" - yielded by a sub-extractor before the records are combined. Off by default, so the output of a sub-extractor is passed through as it is. Turn it on when the API can return nulls in the middle of a record list, which a GraphQL API does when it answers a partial response and reports the failure in a sibling error field. Under "first_match" this also changes which sub-extractor wins: a sub-extractor whose records are all empty no longer counts as a match, so the next one is tried. The dropped records still count toward the page size of an OffsetIncrement or PageIncrement paginator, so a full page holding a null does not end pagination early.', 2669 title="Skip Empty Records", 2670 ) 2671 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]]
mode: Optional[CombineMode]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2674class ZipfileDecoder(BaseModel): 2675 class Config: 2676 extra = Extra.allow 2677 2678 type: Literal["ZipfileDecoder"] 2679 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2680 ..., 2681 description="Parser to parse the decompressed data from the zipfile(s).", 2682 title="Parser", 2683 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2686class RecordSelector(BaseModel): 2687 type: Literal["RecordSelector"] 2688 extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 2689 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2690 None, 2691 description="Responsible for filtering records to be emitted by the Source.", 2692 title="Record Filter", 2693 ) 2694 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2695 None, 2696 description="Responsible for normalization according to the schema.", 2697 title="Schema Normalization", 2698 ) 2699 transform_before_filtering: Optional[bool] = Field( 2700 None, 2701 description="If true, transformation will be applied before record filtering.", 2702 title="Transform Before Filtering", 2703 ) 2704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2707class ConfigMigration(BaseModel): 2708 type: Literal["ConfigMigration"] 2709 description: Optional[str] = Field( 2710 None, description="The description/purpose of the config migration." 2711 ) 2712 transformations: List[ 2713 Union[ 2714 ConfigRemapField, 2715 ConfigAddFields, 2716 ConfigRemoveFields, 2717 CustomConfigTransformation, 2718 ] 2719 ] = Field( 2720 ..., 2721 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2722 title="Transformations", 2723 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2726class ConfigNormalizationRules(BaseModel): 2727 class Config: 2728 extra = Extra.forbid 2729 2730 type: Literal["ConfigNormalizationRules"] 2731 config_migrations: Optional[List[ConfigMigration]] = Field( 2732 [], 2733 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2734 title="Config Migrations", 2735 ) 2736 transformations: Optional[ 2737 List[ 2738 Union[ 2739 ConfigRemapField, 2740 ConfigAddFields, 2741 ConfigRemoveFields, 2742 CustomConfigTransformation, 2743 ] 2744 ] 2745 ] = Field( 2746 [], 2747 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2748 title="Transformations", 2749 ) 2750 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2751 [], 2752 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2753 title="Validations", 2754 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2757class Spec(BaseModel): 2758 type: Literal["Spec"] 2759 connection_specification: Dict[str, Any] = Field( 2760 ..., 2761 description="A connection specification describing how a the connector can be configured.", 2762 title="Connection Specification", 2763 ) 2764 documentation_url: Optional[str] = Field( 2765 None, 2766 description="URL of the connector's documentation page.", 2767 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2768 title="Documentation URL", 2769 ) 2770 advanced_auth: Optional[AuthFlow] = Field( 2771 None, 2772 description="Advanced specification for configuring the authentication flow.", 2773 title="Advanced Auth", 2774 ) 2775 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2776 None, title="Config Normalization Rules" 2777 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2780class DeclarativeSource1(BaseModel): 2781 class Config: 2782 extra = Extra.forbid 2783 2784 type: Literal["DeclarativeSource"] 2785 check: Union[CheckStream, CheckDynamicStream] 2786 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2787 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2788 version: str = Field( 2789 ..., 2790 description="The version of the Airbyte CDK used to build and test the source.", 2791 ) 2792 schemas: Optional[Schemas] = None 2793 definitions: Optional[Dict[str, Any]] = None 2794 spec: Optional[Spec] = None 2795 concurrency_level: Optional[ConcurrencyLevel] = None 2796 api_budget: Optional[HTTPAPIBudget] = None 2797 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2798 None, 2799 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2800 title="Stream Groups", 2801 ) 2802 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2803 None, 2804 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2805 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2806 title="Maximum Concurrent Asynchronous Jobs", 2807 ) 2808 metadata: Optional[Dict[str, Any]] = Field( 2809 None, 2810 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2811 ) 2812 description: Optional[str] = Field( 2813 None, 2814 description="A description of the connector. It will be presented on the Source documentation page.", 2815 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2818class DeclarativeSource2(BaseModel): 2819 class Config: 2820 extra = Extra.forbid 2821 2822 type: Literal["DeclarativeSource"] 2823 check: Union[CheckStream, CheckDynamicStream] 2824 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2825 None 2826 ) 2827 dynamic_streams: List[DynamicDeclarativeStream] 2828 version: str = Field( 2829 ..., 2830 description="The version of the Airbyte CDK used to build and test the source.", 2831 ) 2832 schemas: Optional[Schemas] = None 2833 definitions: Optional[Dict[str, Any]] = None 2834 spec: Optional[Spec] = None 2835 concurrency_level: Optional[ConcurrencyLevel] = None 2836 api_budget: Optional[HTTPAPIBudget] = None 2837 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2838 None, 2839 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2840 title="Stream Groups", 2841 ) 2842 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2843 None, 2844 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2845 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2846 title="Maximum Concurrent Asynchronous Jobs", 2847 ) 2848 metadata: Optional[Dict[str, Any]] = Field( 2849 None, 2850 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2851 ) 2852 description: Optional[str] = Field( 2853 None, 2854 description="A description of the connector. It will be presented on the Source documentation page.", 2855 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2858class DeclarativeSource(BaseModel): 2859 class Config: 2860 extra = Extra.forbid 2861 2862 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2863 ..., 2864 description="An API source that extracts data according to its declarative components.", 2865 title="DeclarativeSource", 2866 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2869class SelectiveAuthenticator(BaseModel): 2870 class Config: 2871 extra = Extra.allow 2872 2873 type: Literal["SelectiveAuthenticator"] 2874 authenticator_selection_path: List[str] = Field( 2875 ..., 2876 description="Path of the field in config with selected authenticator name", 2877 examples=[["auth"], ["auth", "type"]], 2878 title="Authenticator Selection Path", 2879 ) 2880 authenticators: Dict[ 2881 str, 2882 Union[ 2883 ApiKeyAuthenticator, 2884 BasicHttpAuthenticator, 2885 BearerAuthenticator, 2886 OAuthAuthenticator, 2887 JwtAuthenticator, 2888 SessionTokenAuthenticator, 2889 LegacySessionTokenAuthenticator, 2890 CustomAuthenticator, 2891 NoAuth, 2892 RateLimitedMultipleTokenAuthenticator, 2893 ], 2894 ] = Field( 2895 ..., 2896 description="Authenticators to select from.", 2897 examples=[ 2898 { 2899 "authenticators": { 2900 "token": "#/definitions/ApiKeyAuthenticator", 2901 "oauth": "#/definitions/OAuthAuthenticator", 2902 "jwt": "#/definitions/JwtAuthenticator", 2903 } 2904 } 2905 ], 2906 title="Authenticators", 2907 ) 2908 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2911class ConditionalStreams(BaseModel): 2912 type: Literal["ConditionalStreams"] 2913 condition: str = Field( 2914 ..., 2915 description="Condition that will be evaluated to determine if a set of streams should be available.", 2916 examples=["{{ config['is_sandbox'] }}"], 2917 title="Condition", 2918 ) 2919 streams: List[DeclarativeStream] = Field( 2920 ..., 2921 description="Streams that will be used during an operation based on the condition.", 2922 title="Streams", 2923 ) 2924 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2927class FileUploader(BaseModel): 2928 type: Literal["FileUploader"] 2929 requester: Union[HttpRequester, CustomRequester] = Field( 2930 ..., 2931 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2932 ) 2933 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = ( 2934 Field( 2935 ..., 2936 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2937 ) 2938 ) 2939 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2940 None, 2941 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2942 ) 2943 filename_extractor: Optional[str] = Field( 2944 None, 2945 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2946 examples=[ 2947 "{{ record.id }}/{{ record.file_name }}/", 2948 "{{ record.id }}_{{ record.file_name }}/", 2949 ], 2950 ) 2951 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2954class DeclarativeStream(BaseModel): 2955 class Config: 2956 extra = Extra.allow 2957 2958 type: Literal["DeclarativeStream"] 2959 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2960 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2961 ..., 2962 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2963 title="Retriever", 2964 ) 2965 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2966 None, 2967 description="Component used to fetch data incrementally based on a time field in the data.", 2968 title="Incremental Sync", 2969 ) 2970 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2971 schema_loader: Optional[ 2972 Union[ 2973 InlineSchemaLoader, 2974 DynamicSchemaLoader, 2975 JsonFileSchemaLoader, 2976 List[ 2977 Union[ 2978 InlineSchemaLoader, 2979 DynamicSchemaLoader, 2980 JsonFileSchemaLoader, 2981 CustomSchemaLoader, 2982 ] 2983 ], 2984 CustomSchemaLoader, 2985 ] 2986 ] = Field( 2987 None, 2988 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2989 title="Schema Loader", 2990 ) 2991 transformations: Optional[ 2992 List[ 2993 Union[ 2994 AddFields, 2995 RemoveFields, 2996 KeysToLower, 2997 KeysToSnakeCase, 2998 FlattenFields, 2999 DpathFlattenFields, 3000 KeysReplace, 3001 CustomTransformation, 3002 ] 3003 ] 3004 ] = Field( 3005 None, 3006 description="A list of transformations to be applied to each output record.", 3007 title="Transformations", 3008 ) 3009 state_migrations: Optional[ 3010 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 3011 ] = Field( 3012 [], 3013 description="Array of state migrations to be applied on the input state", 3014 title="State Migrations", 3015 ) 3016 file_uploader: Optional[FileUploader] = Field( 3017 None, 3018 description="(experimental) Describes how to fetch a file", 3019 title="File Uploader", 3020 ) 3021 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
3024class SessionTokenAuthenticator(BaseModel): 3025 type: Literal["SessionTokenAuthenticator"] 3026 login_requester: HttpRequester = Field( 3027 ..., 3028 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 3029 examples=[ 3030 { 3031 "type": "HttpRequester", 3032 "url_base": "https://my_api.com", 3033 "path": "/login", 3034 "authenticator": { 3035 "type": "BasicHttpAuthenticator", 3036 "username": "{{ config.username }}", 3037 "password": "{{ config.password }}", 3038 }, 3039 } 3040 ], 3041 title="Login Requester", 3042 ) 3043 session_token_path: List[str] = Field( 3044 ..., 3045 description="The path in the response body returned from the login requester to the session token.", 3046 examples=[["access_token"], ["result", "token"]], 3047 title="Session Token Path", 3048 ) 3049 expiration_duration: Optional[str] = Field( 3050 None, 3051 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 3052 examples=["PT1H", "P1D"], 3053 title="Expiration Duration", 3054 ) 3055 request_authentication: Union[ 3056 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 3057 ] = Field( 3058 ..., 3059 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 3060 title="Data Request Authentication", 3061 ) 3062 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 3063 None, description="Component used to decode the response.", title="Decoder" 3064 ) 3065 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
3068class HttpRequester(BaseModelWithDeprecations): 3069 type: Literal["HttpRequester"] 3070 url_base: Optional[str] = Field( 3071 None, 3072 deprecated=True, 3073 deprecation_message="Use `url` field instead.", 3074 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3075 examples=[ 3076 "https://connect.squareup.com/v2", 3077 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 3078 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3079 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3080 ], 3081 title="API Base URL", 3082 ) 3083 url: Optional[str] = Field( 3084 None, 3085 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3086 examples=[ 3087 "https://connect.squareup.com/v2", 3088 "{{ config['url'] or 'https://app.posthog.com'}}/api", 3089 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3090 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3091 ], 3092 title="API Endpoint URL", 3093 ) 3094 path: Optional[str] = Field( 3095 None, 3096 deprecated=True, 3097 deprecation_message="Use `url` field instead.", 3098 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3099 examples=[ 3100 "/products", 3101 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3102 "/trades/{{ config['symbol_id'] }}/history", 3103 ], 3104 title="URL Path", 3105 ) 3106 http_method: Optional[HttpMethod] = Field( 3107 HttpMethod.GET, 3108 description="The HTTP method used to fetch data from the source (can be GET or POST).", 3109 examples=["GET", "POST"], 3110 title="HTTP Method", 3111 ) 3112 authenticator: Optional[ 3113 Union[ 3114 ApiKeyAuthenticator, 3115 BasicHttpAuthenticator, 3116 BearerAuthenticator, 3117 OAuthAuthenticator, 3118 JwtAuthenticator, 3119 SessionTokenAuthenticator, 3120 SelectiveAuthenticator, 3121 CustomAuthenticator, 3122 NoAuth, 3123 LegacySessionTokenAuthenticator, 3124 RateLimitedMultipleTokenAuthenticator, 3125 ] 3126 ] = Field( 3127 None, 3128 description="Authentication method to use for requests sent to the API.", 3129 title="Authenticator", 3130 ) 3131 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 3132 None, 3133 deprecated=True, 3134 deprecation_message="Use `query_properties` field instead.", 3135 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 3136 title="Fetch Properties from Endpoint", 3137 ) 3138 query_properties: Optional[QueryProperties] = Field( 3139 None, 3140 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 3141 title="Query Properties", 3142 ) 3143 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 3144 None, 3145 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3146 examples=[ 3147 {"unit": "day"}, 3148 { 3149 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3150 }, 3151 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3152 {"sort_by[asc]": "updated_at"}, 3153 ], 3154 title="Query Parameters", 3155 ) 3156 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3157 None, 3158 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3159 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3160 title="Request Headers", 3161 ) 3162 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3163 None, 3164 deprecated=True, 3165 deprecation_message="Use `request_body` field instead.", 3166 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3167 examples=[ 3168 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3169 ], 3170 title="Request Body Payload (Non-JSON)", 3171 ) 3172 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3173 None, 3174 deprecated=True, 3175 deprecation_message="Use `request_body` field instead.", 3176 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3177 examples=[ 3178 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3179 {"key": "{{ config['value'] }}"}, 3180 {"sort": {"field": "updated_at", "order": "ascending"}}, 3181 ], 3182 title="Request Body JSON Payload", 3183 ) 3184 request_body: Optional[ 3185 Union[ 3186 RequestBodyPlainText, 3187 RequestBodyUrlEncodedForm, 3188 RequestBodyJsonObject, 3189 RequestBodyGraphQL, 3190 ] 3191 ] = Field( 3192 None, 3193 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3194 title="Request Body", 3195 ) 3196 error_handler: Optional[ 3197 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3198 ] = Field( 3199 None, 3200 description="Error handler component that defines how to handle errors.", 3201 title="Error Handler", 3202 ) 3203 use_cache: Optional[bool] = Field( 3204 False, 3205 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3206 title="Use Cache", 3207 ) 3208 connect_timeout_in_seconds: Optional[float] = Field( 3209 None, 3210 description="Maximum time to establish the connection (TCP + TLS handshake). When unset, no connect timeout is applied.", 3211 examples=[30], 3212 gt=0, 3213 title="Connect Timeout (seconds)", 3214 ) 3215 read_timeout_in_seconds: Optional[float] = Field( 3216 None, 3217 description="The maximum time to wait for the server to send any data (the first byte of the response, or the next chunk while the body is read). It is NOT a limit on the total request duration. When unset, no read timeout is applied. Only set it when every endpoint served by this requester responds well within the value; synchronous report or export endpoints can stay silent for minutes. Each timed-out attempt counts against the error handler's retry window (600 s for manifest connectors), so a value near 300 allows only a few retries.", 3218 examples=[300], 3219 gt=0, 3220 title="Read Timeout (seconds)", 3221 ) 3222 use_tcp_keepalive: Optional[bool] = Field( 3223 False, 3224 description="Enables TCP keepalive (60 s idle, 10 s probe interval, 6 probes) on this requester's connections, so dead or NAT-dropped connections are detected and fail instead of hanging forever. Does not fail slow but alive servers; use read_timeout_in_seconds to bound those.", 3225 examples=[True], 3226 title="Use TCP Keepalive", 3227 ) 3228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, RateLimitedMultipleTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
3231class DynamicSchemaLoader(BaseModel): 3232 type: Literal["DynamicSchemaLoader"] 3233 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3234 ..., 3235 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3236 title="Retriever", 3237 ) 3238 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3239 None, 3240 description="Responsible for filtering fields to be added to json schema.", 3241 title="Schema Filter", 3242 ) 3243 schema_transformations: Optional[ 3244 List[ 3245 Union[ 3246 AddFields, 3247 RemoveFields, 3248 KeysToLower, 3249 KeysToSnakeCase, 3250 FlattenFields, 3251 DpathFlattenFields, 3252 KeysReplace, 3253 CustomTransformation, 3254 ] 3255 ] 3256 ] = Field( 3257 None, 3258 description="A list of transformations to be applied to the schema.", 3259 title="Schema Transformations", 3260 ) 3261 schema_type_identifier: SchemaTypeIdentifier 3262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
3265class ParentStreamConfig(BaseModel): 3266 type: Literal["ParentStreamConfig"] 3267 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3268 ..., description="Reference to the parent stream.", title="Parent Stream" 3269 ) 3270 parent_key: str = Field( 3271 ..., 3272 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3273 examples=["id", "{{ config['parent_record_id'] }}"], 3274 title="Parent Key", 3275 ) 3276 partition_field: str = Field( 3277 ..., 3278 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3279 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3280 title="Current Parent Key Value Identifier", 3281 ) 3282 request_option: Optional[RequestOption] = Field( 3283 None, 3284 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3285 title="Request Option", 3286 ) 3287 incremental_dependency: Optional[bool] = Field( 3288 False, 3289 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3290 title="Incremental Dependency", 3291 ) 3292 lazy_read_pointer: Optional[List[str]] = Field( 3293 [], 3294 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3295 title="Lazy Read Pointer", 3296 ) 3297 extra_fields: Optional[List[List[str]]] = Field( 3298 None, 3299 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3300 title="Extra Fields", 3301 ) 3302 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
3305class PropertiesFromEndpoint(BaseModel): 3306 type: Literal["PropertiesFromEndpoint"] 3307 property_field_path: List[str] = Field( 3308 ..., 3309 description="Describes the path to the field that should be extracted", 3310 examples=[["name"]], 3311 ) 3312 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3313 ..., 3314 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3315 ) 3316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
3319class QueryProperties(BaseModel): 3320 type: Literal["QueryProperties"] 3321 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3322 ..., 3323 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3324 title="Property List", 3325 ) 3326 always_include_properties: Optional[List[str]] = Field( 3327 None, 3328 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3329 title="Always Include Properties", 3330 ) 3331 property_chunking: Optional[PropertyChunking] = Field( 3332 None, 3333 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3334 title="Property Chunking", 3335 ) 3336 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3337 None, 3338 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3339 title="Property Selector", 3340 ) 3341 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3344class StateDelegatingStream(BaseModel): 3345 type: Literal["StateDelegatingStream"] 3346 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3347 full_refresh_stream: DeclarativeStream = Field( 3348 ..., 3349 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3350 title="Full Refresh Stream", 3351 ) 3352 incremental_stream: DeclarativeStream = Field( 3353 ..., 3354 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3355 title="Incremental Stream", 3356 ) 3357 api_retention_period: Optional[str] = Field( 3358 None, 3359 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", 3360 examples=["P30D", "P90D", "P1Y"], 3361 title="API Retention Period", 3362 ) 3363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3366class SimpleRetriever(BaseModel): 3367 type: Literal["SimpleRetriever"] 3368 requester: Union[HttpRequester, CustomRequester] = Field( 3369 ..., 3370 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3371 ) 3372 decoder: Optional[ 3373 Union[ 3374 JsonDecoder, 3375 JsonItemsDecoder, 3376 XmlDecoder, 3377 CsvDecoder, 3378 JsonlDecoder, 3379 GzipDecoder, 3380 IterableDecoder, 3381 ZipfileDecoder, 3382 CustomDecoder, 3383 ] 3384 ] = Field( 3385 None, 3386 description="Component decoding the response so records can be extracted.", 3387 title="HTTP Response Format", 3388 ) 3389 record_selector: RecordSelector = Field( 3390 ..., 3391 description="Component that describes how to extract records from a HTTP response.", 3392 ) 3393 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3394 None, 3395 description="Paginator component that describes how to navigate through the API's pages.", 3396 ) 3397 pagination_reset: Optional[PaginationReset] = Field( 3398 None, 3399 description="Describes what triggers pagination reset and how to handle it.", 3400 ) 3401 page_size_reduction: Optional[PageSizeReduction] = Field( 3402 None, 3403 description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", 3404 ) 3405 request_window_splitting: Optional[RequestWindowSplitting] = Field( 3406 None, 3407 description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page size, and applies even when the API rejects the request before returning a single page.", 3408 ) 3409 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3410 False, 3411 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3412 ) 3413 partition_router: Optional[ 3414 Union[ 3415 SubstreamPartitionRouter, 3416 ListPartitionRouter, 3417 GroupingPartitionRouter, 3418 UnionPartitionRouter, 3419 CustomPartitionRouter, 3420 List[ 3421 Union[ 3422 SubstreamPartitionRouter, 3423 ListPartitionRouter, 3424 GroupingPartitionRouter, 3425 UnionPartitionRouter, 3426 CustomPartitionRouter, 3427 ] 3428 ], 3429 ] 3430 ] = Field( 3431 None, 3432 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3433 title="Partition Router", 3434 ) 3435 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
page_size_reduction: Optional[PageSizeReduction]
request_window_splitting: Optional[RequestWindowSplitting]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3438class AsyncRetriever(BaseModel): 3439 type: Literal["AsyncRetriever"] 3440 record_selector: RecordSelector = Field( 3441 ..., 3442 description="Component that describes how to extract records from a HTTP response.", 3443 ) 3444 status_mapping: AsyncJobStatusMap = Field( 3445 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3446 ) 3447 status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = Field( 3448 ..., description="Responsible for fetching the actual status of the async job." 3449 ) 3450 download_target_extractor: Optional[ 3451 Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 3452 ] = Field( 3453 None, 3454 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3455 ) 3456 download_extractor: Optional[ 3457 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor] 3458 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3459 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3460 ..., 3461 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3462 ) 3463 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3464 ..., 3465 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3466 ) 3467 polling_job_timeout: Optional[Union[int, str]] = Field( 3468 None, 3469 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3470 ) 3471 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3472 None, 3473 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3474 ge=1, 3475 ) 3476 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3477 None, 3478 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3479 ) 3480 download_requester: Union[HttpRequester, CustomRequester] = Field( 3481 ..., 3482 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3483 ) 3484 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3485 None, 3486 description="Paginator component that describes how to navigate through the API's pages during download.", 3487 ) 3488 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3489 None, 3490 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3491 ) 3492 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3493 None, 3494 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3495 ) 3496 partition_router: Optional[ 3497 Union[ 3498 ListPartitionRouter, 3499 SubstreamPartitionRouter, 3500 GroupingPartitionRouter, 3501 UnionPartitionRouter, 3502 CustomPartitionRouter, 3503 List[ 3504 Union[ 3505 ListPartitionRouter, 3506 SubstreamPartitionRouter, 3507 GroupingPartitionRouter, 3508 UnionPartitionRouter, 3509 CustomPartitionRouter, 3510 ] 3511 ], 3512 ] 3513 ] = Field( 3514 [], 3515 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3516 title="Partition Router", 3517 ) 3518 decoder: Optional[ 3519 Union[ 3520 CsvDecoder, 3521 GzipDecoder, 3522 JsonDecoder, 3523 JsonItemsDecoder, 3524 JsonlDecoder, 3525 IterableDecoder, 3526 XmlDecoder, 3527 ZipfileDecoder, 3528 CustomDecoder, 3529 ] 3530 ] = Field( 3531 None, 3532 description="Component decoding the response so records can be extracted.", 3533 title="HTTP Response Format", 3534 ) 3535 download_decoder: Optional[ 3536 Union[ 3537 CsvDecoder, 3538 GzipDecoder, 3539 JsonDecoder, 3540 JsonItemsDecoder, 3541 JsonlDecoder, 3542 IterableDecoder, 3543 XmlDecoder, 3544 ZipfileDecoder, 3545 CustomDecoder, 3546 ] 3547 ] = Field( 3548 None, 3549 description="Component decoding the download response so records can be extracted.", 3550 title="Download HTTP Response Format", 3551 ) 3552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
StreamGroup(pydantic.v1.main.BaseModel):
3555class StreamGroup(BaseModel): 3556 streams: List[DeclarativeStream] = Field( 3557 ..., 3558 description="List of references to streams that belong to this group.\n", 3559 title="Streams", 3560 ) 3561 action: BlockSimultaneousSyncsAction = Field( 3562 ..., description="The action to apply to streams in this group.", title="Action" 3563 )
streams: List[DeclarativeStream]
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3566class SubstreamPartitionRouter(BaseModel): 3567 type: Literal["SubstreamPartitionRouter"] 3568 parent_stream_configs: List[ParentStreamConfig] = Field( 3569 ..., 3570 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3571 title="Parent Stream Configs", 3572 ) 3573 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3576class GroupingPartitionRouter(BaseModel): 3577 type: Literal["GroupingPartitionRouter"] 3578 group_size: int = Field( 3579 ..., 3580 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3581 examples=[10, 50], 3582 title="Group Size", 3583 ) 3584 underlying_partition_router: Union[ 3585 ListPartitionRouter, 3586 SubstreamPartitionRouter, 3587 UnionPartitionRouter, 3588 CustomPartitionRouter, 3589 ] = Field( 3590 ..., 3591 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3592 title="Underlying Partition Router", 3593 ) 3594 deduplicate: Optional[bool] = Field( 3595 True, 3596 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3597 title="Deduplicate Partitions", 3598 ) 3599 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]
class
UnionPartitionRouter(pydantic.v1.main.BaseModel):
3602class UnionPartitionRouter(BaseModel): 3603 type: Literal["UnionPartitionRouter"] 3604 partition_field: str = Field( 3605 ..., 3606 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3607 examples=["repository", "{{ config['partition_field'] }}"], 3608 title="Partition Field", 3609 ) 3610 partition_routers: List[ 3611 Union[ 3612 ListPartitionRouter, 3613 SubstreamPartitionRouter, 3614 UnionPartitionRouter, 3615 CustomPartitionRouter, 3616 ] 3617 ] = Field( 3618 ..., 3619 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3620 title="Partition Routers", 3621 ) 3622 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
partition_routers: List[Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3625class HttpComponentsResolver(BaseModel): 3626 type: Literal["HttpComponentsResolver"] 3627 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3628 ..., 3629 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3630 title="Retriever", 3631 ) 3632 components_mapping: List[ComponentMappingDefinition] 3633 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3636class DynamicDeclarativeStream(BaseModel): 3637 type: Literal["DynamicDeclarativeStream"] 3638 name: Optional[str] = Field( 3639 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3640 ) 3641 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3642 ..., description="Reference to the stream template.", title="Stream Template" 3643 ) 3644 components_resolver: Union[ 3645 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3646 ] = Field( 3647 ..., 3648 description="Component resolve and populates stream templates with components values.", 3649 title="Components Resolver", 3650 ) 3651 use_parent_parameters: Optional[bool] = Field( 3652 True, 3653 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3654 title="Use Parent Parameters", 3655 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]