airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class BasicHttpAuthenticator(BaseModel): 22 type: Literal["BasicHttpAuthenticator"] 23 username: str = Field( 24 ..., 25 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 26 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 27 title="Username", 28 ) 29 password: Optional[str] = Field( 30 "", 31 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['password'] }}", ""], 33 title="Password", 34 ) 35 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 36 37 38class BearerAuthenticator(BaseModel): 39 type: Literal["BearerAuthenticator"] 40 api_token: str = Field( 41 ..., 42 description="Token to inject as request header for authenticating with the API.", 43 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 44 title="Bearer Token", 45 ) 46 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 47 48 49class HttpMethod(Enum): 50 GET = "GET" 51 POST = "POST" 52 53 54class QuotaStatusSource(BaseModel): 55 type: Literal["QuotaStatusSource"] 56 url: str = Field( 57 ..., 58 description="The full URL of the quota status endpoint.", 59 examples=[ 60 "https://api.github.com/rate_limit", 61 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 62 ], 63 title="URL", 64 ) 65 http_method: Optional[HttpMethod] = Field( 66 HttpMethod.GET, 67 description="The HTTP method used to fetch the quota status.", 68 title="HTTP Method", 69 ) 70 request_headers: Optional[Dict[str, str]] = Field( 71 None, 72 description="Additional headers to send with the quota status request.", 73 title="Request Headers", 74 ) 75 unavailable_status_codes: Optional[List[int]] = Field( 76 None, 77 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 78 examples=[[404]], 79 title="Unavailable Status Codes", 80 unique_items=True, 81 ) 82 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 83 84 85class DynamicStreamCheckConfig(BaseModel): 86 type: Literal["DynamicStreamCheckConfig"] 87 dynamic_stream_name: str = Field( 88 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 89 ) 90 stream_count: Optional[int] = Field( 91 None, 92 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 93 ge=1, 94 title="Stream Count", 95 ) 96 97 98class CheckDynamicStream(BaseModel): 99 type: Literal["CheckDynamicStream"] 100 stream_count: int = Field( 101 ..., 102 description="Numbers of the streams to try reading from when running a check operation.", 103 title="Stream Count", 104 ) 105 use_check_availability: Optional[bool] = Field( 106 True, 107 description="Enables stream check availability. This field is automatically set by the CDK.", 108 title="Use Check Availability", 109 ) 110 config_overrides: Optional[Dict[str, Any]] = Field( 111 None, 112 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 113 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 114 title="Config Overrides", 115 ) 116 117 118class ConcurrencyLevel(BaseModel): 119 type: Optional[Literal["ConcurrencyLevel"]] = None 120 default_concurrency: Union[int, str] = Field( 121 ..., 122 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 123 examples=[10, "{{ config['num_workers'] or 10 }}"], 124 title="Default Concurrency", 125 ) 126 max_concurrency: Optional[int] = Field( 127 None, 128 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 129 examples=[20, 100], 130 title="Max Concurrency", 131 ) 132 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 133 134 135class ConstantBackoffStrategy(BaseModel): 136 type: Literal["ConstantBackoffStrategy"] 137 backoff_time_in_seconds: Union[float, str] = Field( 138 ..., 139 description="Backoff time in seconds.", 140 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 141 title="Backoff Time", 142 ) 143 jitter_range_in_seconds: Optional[float] = Field( 144 None, 145 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 146 examples=[15], 147 ge=0, 148 title="Jitter Range", 149 ) 150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 151 152 153class CursorPagination(BaseModel): 154 type: Literal["CursorPagination"] 155 cursor_value: str = Field( 156 ..., 157 description="Value of the cursor defining the next page to fetch.", 158 examples=[ 159 "{{ headers.link.next.cursor }}", 160 "{{ last_record['key'] }}", 161 "{{ response['nextPage'] }}", 162 ], 163 title="Cursor Value", 164 ) 165 page_size: Optional[Union[int, str]] = Field( 166 None, 167 description="The number of records to include in each pages.", 168 examples=[100, "{{ config['page_size'] }}"], 169 title="Page Size", 170 ) 171 stop_condition: Optional[str] = Field( 172 None, 173 description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", 174 examples=[ 175 "{{ response.data.has_more is false }}", 176 "{{ 'next' not in headers['link'] }}", 177 "{{ last_page_size < page_size }}", 178 ], 179 title="Stop Condition", 180 ) 181 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 182 183 184class CustomAuthenticator(BaseModel): 185 class Config: 186 extra = Extra.allow 187 188 type: Literal["CustomAuthenticator"] 189 class_name: str = Field( 190 ..., 191 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 192 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 193 title="Class Name", 194 ) 195 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 196 197 198class CustomBackoffStrategy(BaseModel): 199 class Config: 200 extra = Extra.allow 201 202 type: Literal["CustomBackoffStrategy"] 203 class_name: str = Field( 204 ..., 205 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 206 examples=["source_railz.components.MyCustomBackoffStrategy"], 207 title="Class Name", 208 ) 209 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 210 211 212class CustomErrorHandler(BaseModel): 213 class Config: 214 extra = Extra.allow 215 216 type: Literal["CustomErrorHandler"] 217 class_name: str = Field( 218 ..., 219 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 220 examples=["source_railz.components.MyCustomErrorHandler"], 221 title="Class Name", 222 ) 223 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 224 225 226class CustomPaginationStrategy(BaseModel): 227 class Config: 228 extra = Extra.allow 229 230 type: Literal["CustomPaginationStrategy"] 231 class_name: str = Field( 232 ..., 233 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 234 examples=["source_railz.components.MyCustomPaginationStrategy"], 235 title="Class Name", 236 ) 237 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 238 239 240class CustomRecordExtractor(BaseModel): 241 class Config: 242 extra = Extra.allow 243 244 type: Literal["CustomRecordExtractor"] 245 class_name: str = Field( 246 ..., 247 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 248 examples=["source_railz.components.MyCustomRecordExtractor"], 249 title="Class Name", 250 ) 251 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 252 253 254class CustomRecordFilter(BaseModel): 255 class Config: 256 extra = Extra.allow 257 258 type: Literal["CustomRecordFilter"] 259 class_name: str = Field( 260 ..., 261 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 262 examples=["source_railz.components.MyCustomCustomRecordFilter"], 263 title="Class Name", 264 ) 265 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 266 267 268class CustomRequester(BaseModel): 269 class Config: 270 extra = Extra.allow 271 272 type: Literal["CustomRequester"] 273 class_name: str = Field( 274 ..., 275 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 276 examples=["source_railz.components.MyCustomRecordExtractor"], 277 title="Class Name", 278 ) 279 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 280 281 282class CustomRetriever(BaseModel): 283 class Config: 284 extra = Extra.allow 285 286 type: Literal["CustomRetriever"] 287 class_name: str = Field( 288 ..., 289 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 290 examples=["source_railz.components.MyCustomRetriever"], 291 title="Class Name", 292 ) 293 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 294 295 296class CustomPartitionRouter(BaseModel): 297 class Config: 298 extra = Extra.allow 299 300 type: Literal["CustomPartitionRouter"] 301 class_name: str = Field( 302 ..., 303 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 304 examples=["source_railz.components.MyCustomPartitionRouter"], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 308 309 310class CustomSchemaLoader(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomSchemaLoader"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomSchemaLoader"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 322 323 324class CustomSchemaNormalization(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomSchemaNormalization"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 332 examples=[ 333 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 334 ], 335 title="Class Name", 336 ) 337 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 338 339 340class CustomStateMigration(BaseModel): 341 class Config: 342 extra = Extra.allow 343 344 type: Literal["CustomStateMigration"] 345 class_name: str = Field( 346 ..., 347 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 348 examples=["source_railz.components.MyCustomStateMigration"], 349 title="Class Name", 350 ) 351 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 352 353 354class CustomTransformation(BaseModel): 355 class Config: 356 extra = Extra.allow 357 358 type: Literal["CustomTransformation"] 359 class_name: str = Field( 360 ..., 361 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 362 examples=["source_railz.components.MyCustomTransformation"], 363 title="Class Name", 364 ) 365 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 366 367 368class LegacyToPerPartitionStateMigration(BaseModel): 369 class Config: 370 extra = Extra.allow 371 372 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 373 374 375class Clamping(BaseModel): 376 target: str = Field( 377 ..., 378 description="The period of time that datetime windows will be clamped by", 379 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 380 title="Target", 381 ) 382 target_details: Optional[Dict[str, Any]] = None 383 384 385class Algorithm(Enum): 386 HS256 = "HS256" 387 HS384 = "HS384" 388 HS512 = "HS512" 389 ES256 = "ES256" 390 ES256K = "ES256K" 391 ES384 = "ES384" 392 ES512 = "ES512" 393 RS256 = "RS256" 394 RS384 = "RS384" 395 RS512 = "RS512" 396 PS256 = "PS256" 397 PS384 = "PS384" 398 PS512 = "PS512" 399 EdDSA = "EdDSA" 400 401 402class JwtHeaders(BaseModel): 403 class Config: 404 extra = Extra.forbid 405 406 kid: Optional[str] = Field( 407 None, 408 description="Private key ID for user account.", 409 examples=["{{ config['kid'] }}"], 410 title="Key Identifier", 411 ) 412 typ: Optional[str] = Field( 413 "JWT", 414 description="The media type of the complete JWT.", 415 examples=["JWT"], 416 title="Type", 417 ) 418 cty: Optional[str] = Field( 419 None, 420 description="Content type of JWT header.", 421 examples=["JWT"], 422 title="Content Type", 423 ) 424 425 426class JwtPayload(BaseModel): 427 class Config: 428 extra = Extra.forbid 429 430 iss: Optional[str] = Field( 431 None, 432 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 433 examples=["{{ config['iss'] }}"], 434 title="Issuer", 435 ) 436 sub: Optional[str] = Field( 437 None, 438 description="The subject of the JWT. Commonly defined by the API.", 439 title="Subject", 440 ) 441 aud: Optional[str] = Field( 442 None, 443 description="The recipient that the JWT is intended for. Commonly defined by the API.", 444 examples=["appstoreconnect-v1"], 445 title="Audience", 446 ) 447 448 449class RefreshTokenUpdater(BaseModel): 450 refresh_token_name: Optional[str] = Field( 451 "refresh_token", 452 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 453 examples=["refresh_token"], 454 title="Refresh Token Property Name", 455 ) 456 access_token_config_path: Optional[List[str]] = Field( 457 ["credentials", "access_token"], 458 description="Config path to the access token. Make sure the field actually exists in the config.", 459 examples=[["credentials", "access_token"], ["access_token"]], 460 title="Config Path To Access Token", 461 ) 462 refresh_token_config_path: Optional[List[str]] = Field( 463 ["credentials", "refresh_token"], 464 description="Config path to the access token. Make sure the field actually exists in the config.", 465 examples=[["credentials", "refresh_token"], ["refresh_token"]], 466 title="Config Path To Refresh Token", 467 ) 468 token_expiry_date_config_path: Optional[List[str]] = Field( 469 ["credentials", "token_expiry_date"], 470 description="Config path to the expiry date. Make sure actually exists in the config.", 471 examples=[["credentials", "token_expiry_date"]], 472 title="Config Path To Expiry Date", 473 ) 474 refresh_token_error_status_codes: Optional[List[int]] = Field( 475 [], 476 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 477 examples=[[400, 500]], 478 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 479 ) 480 refresh_token_error_key: Optional[str] = Field( 481 "", 482 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 483 examples=["error"], 484 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 485 ) 486 refresh_token_error_values: Optional[List[str]] = Field( 487 [], 488 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 489 examples=[["invalid_grant", "invalid_permissions"]], 490 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 491 ) 492 493 494class Rate(BaseModel): 495 class Config: 496 extra = Extra.allow 497 498 limit: Union[int, str] = Field( 499 ..., 500 description="The maximum number of calls allowed within the interval.", 501 title="Limit", 502 ) 503 interval: str = Field( 504 ..., 505 description="The time interval for the rate limit.", 506 examples=["PT1H", "P1D"], 507 title="Interval", 508 ) 509 510 511class HttpRequestRegexMatcher(BaseModel): 512 class Config: 513 extra = Extra.allow 514 515 method: Optional[str] = Field( 516 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 517 ) 518 url_base: Optional[str] = Field( 519 None, 520 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 521 title="URL Base", 522 ) 523 url_path_pattern: Optional[str] = Field( 524 None, 525 description="A regular expression pattern to match the URL path.", 526 title="URL Path Pattern", 527 ) 528 params: Optional[Dict[str, Any]] = Field( 529 None, description="The query parameters to match.", title="Parameters" 530 ) 531 headers: Optional[Dict[str, Any]] = Field( 532 None, description="The headers to match.", title="Headers" 533 ) 534 weight: Optional[Union[int, str]] = Field( 535 None, 536 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", 537 title="Weight", 538 ) 539 540 541class CombineMode(Enum): 542 union = "union" 543 first_match = "first_match" 544 545 546class ResponseToFileExtractor(BaseModel): 547 type: Literal["ResponseToFileExtractor"] 548 preserve_na_values: Optional[bool] = Field( 549 False, 550 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 551 title="Preserve NA Values", 552 ) 553 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 554 555 556class OnNoRecords(Enum): 557 skip = "skip" 558 emit_parent = "emit_parent" 559 560 561class ParentFieldPath(BaseModel): 562 type: Literal["ParentFieldPath"] 563 parent_path: List[str] = Field( 564 ..., 565 description="Path to the value on the record being expanded.", 566 examples=[["url"], ["id"], ["repository", "name"]], 567 title="Parent Path", 568 ) 569 record_path: List[str] = Field( 570 ..., 571 description="Path on the expanded item to write the value to. An existing value is overwritten. Intermediate objects are created as needed, so a path may not pass through a value the item already holds as a scalar, and a numeric path segment creates an array rather than an object - the same behavior as `AddFields`.", 572 examples=[["pull_request_url"], ["comment_id"], ["parent", "id"]], 573 title="Record Path", 574 ) 575 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 576 577 578class ExponentialBackoffStrategy(BaseModel): 579 type: Literal["ExponentialBackoffStrategy"] 580 factor: Optional[Union[float, str]] = Field( 581 5, 582 description="Multiplicative constant applied on each retry.", 583 examples=[5, 5.5, "10"], 584 title="Factor", 585 ) 586 jitter_range_in_seconds: Optional[float] = Field( 587 None, 588 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 589 examples=[2], 590 ge=0, 591 title="Jitter Range", 592 ) 593 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 594 595 596class GroupByKeyMergeStrategy(BaseModel): 597 type: Literal["GroupByKeyMergeStrategy"] 598 key: Union[str, List[str]] = Field( 599 ..., 600 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 601 examples=["id", ["parent_id", "end_date"]], 602 title="Key", 603 ) 604 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 605 606 607class SessionTokenRequestBearerAuthenticator(BaseModel): 608 type: Literal["Bearer"] 609 610 611class Action(Enum): 612 SUCCESS = "SUCCESS" 613 FAIL = "FAIL" 614 RETRY = "RETRY" 615 IGNORE = "IGNORE" 616 RESET_PAGINATION = "RESET_PAGINATION" 617 RATE_LIMITED = "RATE_LIMITED" 618 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 619 REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" 620 SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW" 621 622 623class FailureType(Enum): 624 system_error = "system_error" 625 config_error = "config_error" 626 transient_error = "transient_error" 627 628 629class HttpResponseFilter(BaseModel): 630 type: Literal["HttpResponseFilter"] 631 action: Optional[Action] = Field( 632 None, 633 description="Action to execute if a response matches the filter.", 634 examples=[ 635 "SUCCESS", 636 "FAIL", 637 "RETRY", 638 "IGNORE", 639 "RESET_PAGINATION", 640 "RATE_LIMITED", 641 "REFRESH_TOKEN_THEN_RETRY", 642 "REDUCE_PAGE_SIZE", 643 "SPLIT_REQUEST_WINDOW", 644 ], 645 title="Action", 646 ) 647 failure_type: Optional[FailureType] = Field( 648 None, 649 description="Failure type of traced exception if a response matches the filter.", 650 examples=["system_error", "config_error", "transient_error"], 651 title="Failure Type", 652 ) 653 error_message: Optional[str] = Field( 654 None, 655 description="Error Message to display if the response matches the filter.", 656 examples=["Skipping {{ request_url }} (HTTP 404)."], 657 title="Error Message", 658 ) 659 error_message_contains: Optional[str] = Field( 660 None, 661 description="Match the response if its error message contains the substring.", 662 example=["This API operation is not enabled for this site"], 663 title="Error Message Substring", 664 ) 665 http_codes: Optional[List[int]] = Field( 666 None, 667 description="Match the response if its HTTP code is included in this list.", 668 examples=[[420, 429], [500]], 669 title="HTTP Codes", 670 unique_items=True, 671 ) 672 predicate: Optional[str] = Field( 673 None, 674 description="Match the response if the predicate evaluates to true.", 675 examples=[ 676 "{{ 'Too much requests' in response }}", 677 "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", 678 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 679 ], 680 title="Predicate", 681 ) 682 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 683 684 685class ComplexFieldType(BaseModel): 686 field_type: str 687 items: Optional[Union[str, ComplexFieldType]] = None 688 689 690class TypesMap(BaseModel): 691 target_type: Union[str, List[str], ComplexFieldType] 692 current_type: Union[str, List[str]] 693 condition: Optional[str] = None 694 695 696class SchemaTypeIdentifier(BaseModel): 697 type: Optional[Literal["SchemaTypeIdentifier"]] = None 698 schema_pointer: Optional[List[str]] = Field( 699 [], 700 description="List of nested fields defining the schema field path to extract. Defaults to [].", 701 title="Schema Path", 702 ) 703 key_pointer: List[str] = Field( 704 ..., 705 description="List of potentially nested fields describing the full path of the field key to extract.", 706 title="Key Path", 707 ) 708 type_pointer: Optional[List[str]] = Field( 709 None, 710 description="List of potentially nested fields describing the full path of the field type to extract.", 711 title="Type Path", 712 ) 713 types_mapping: Optional[List[TypesMap]] = None 714 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 715 716 717class InlineSchemaLoader(BaseModel): 718 type: Literal["InlineSchemaLoader"] 719 schema_: Optional[Dict[str, Any]] = Field( 720 None, 721 alias="schema", 722 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 723 title="Schema", 724 ) 725 726 727class JsonFileSchemaLoader(BaseModel): 728 type: Literal["JsonFileSchemaLoader"] 729 file_path: Optional[str] = Field( 730 None, 731 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 732 example=["./schemas/users.json"], 733 title="File Path", 734 ) 735 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 736 737 738class JsonDecoder(BaseModel): 739 type: Literal["JsonDecoder"] 740 741 742class JsonItemsDecoder(BaseModel): 743 type: Literal["JsonItemsDecoder"] 744 items_path: str = Field( 745 ..., 746 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", 747 examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], 748 title="Items Path", 749 ) 750 encoding: Optional[str] = Field( 751 "utf-8", 752 description="Text encoding used to decode the streamed bytes before JSON parsing.", 753 title="Encoding", 754 ) 755 756 757class JsonlDecoder(BaseModel): 758 type: Literal["JsonlDecoder"] 759 760 761class KeysToLower(BaseModel): 762 type: Literal["KeysToLower"] 763 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 764 765 766class KeysToSnakeCase(BaseModel): 767 type: Literal["KeysToSnakeCase"] 768 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 769 770 771class FlattenFields(BaseModel): 772 type: Literal["FlattenFields"] 773 flatten_lists: Optional[bool] = Field( 774 True, 775 description="Whether to flatten lists or leave it as is. Default is True.", 776 title="Flatten Lists", 777 ) 778 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 779 780 781class KeyTransformation(BaseModel): 782 type: Literal["KeyTransformation"] 783 prefix: Optional[str] = Field( 784 None, 785 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 786 examples=["flattened_"], 787 title="Key Prefix", 788 ) 789 suffix: Optional[str] = Field( 790 None, 791 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 792 examples=["_flattened"], 793 title="Key Suffix", 794 ) 795 796 797class DpathFlattenFields(BaseModel): 798 type: Literal["DpathFlattenFields"] 799 field_path: List[str] = Field( 800 ..., 801 description="A path to field that needs to be flattened.", 802 examples=[["data"], ["data", "*", "field"]], 803 title="Field Path", 804 ) 805 delete_origin_value: Optional[bool] = Field( 806 None, 807 description="Whether to delete the origin value or keep it. Default is False.", 808 title="Delete Origin Value", 809 ) 810 replace_record: Optional[bool] = Field( 811 None, 812 description="Whether to replace the origin record or not. Default is False.", 813 title="Replace Origin Record", 814 ) 815 key_transformation: Optional[KeyTransformation] = Field( 816 None, 817 description="Transformation for object keys. If not provided, original key will be used.", 818 title="Key transformation", 819 ) 820 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 821 822 823class KeysReplace(BaseModel): 824 type: Literal["KeysReplace"] 825 old: str = Field( 826 ..., 827 description="Old value to replace.", 828 examples=[ 829 " ", 830 "{{ record.id }}", 831 "{{ config['id'] }}", 832 "{{ stream_slice['id'] }}", 833 ], 834 title="Old value", 835 ) 836 new: str = Field( 837 ..., 838 description="New value to set.", 839 examples=[ 840 "_", 841 "{{ record.id }}", 842 "{{ config['id'] }}", 843 "{{ stream_slice['id'] }}", 844 ], 845 title="New value", 846 ) 847 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 848 849 850class IterableDecoder(BaseModel): 851 type: Literal["IterableDecoder"] 852 853 854class XmlDecoder(BaseModel): 855 type: Literal["XmlDecoder"] 856 857 858class CustomDecoder(BaseModel): 859 class Config: 860 extra = Extra.allow 861 862 type: Literal["CustomDecoder"] 863 class_name: str = Field( 864 ..., 865 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 866 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 867 title="Class Name", 868 ) 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class MinMaxDatetime(BaseModel): 873 type: Literal["MinMaxDatetime"] 874 datetime: str = Field( 875 ..., 876 description="Datetime value.", 877 examples=[ 878 "2021-01-01", 879 "2021-01-01T00:00:00Z", 880 "{{ config['start_time'] }}", 881 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 882 ], 883 title="Datetime", 884 ) 885 datetime_format: Optional[str] = Field( 886 "", 887 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 888 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 889 title="Datetime Format", 890 ) 891 max_datetime: Optional[str] = Field( 892 None, 893 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 894 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 895 title="Max Datetime", 896 ) 897 min_datetime: Optional[str] = Field( 898 None, 899 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 900 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 901 title="Min Datetime", 902 ) 903 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 904 905 906class NoAuth(BaseModel): 907 type: Literal["NoAuth"] 908 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 909 910 911class NoPagination(BaseModel): 912 type: Literal["NoPagination"] 913 914 915class Scope(BaseModel): 916 class Config: 917 extra = Extra.allow 918 919 scope: str = Field(..., description="The OAuth scope string to request from the provider.") 920 921 922class OptionalScope(BaseModel): 923 class Config: 924 extra = Extra.allow 925 926 scope: str = Field(..., description="The OAuth scope string to request from the provider.") 927 928 929class ScopesJoinStrategy(Enum): 930 space = "space" 931 comma = "comma" 932 plus = "plus" 933 934 935class State(BaseModel): 936 class Config: 937 extra = Extra.allow 938 939 min: int 940 max: int 941 942 943class OauthConnectorInputSpecification(BaseModel): 944 class Config: 945 extra = Extra.allow 946 947 consent_url: str = Field( 948 ..., 949 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 950 examples=[ 951 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 952 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 953 ], 954 title="Consent URL", 955 ) 956 scope: Optional[str] = Field( 957 None, 958 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 959 examples=["user:read user:read_orders workspaces:read"], 960 title="Scopes", 961 ) 962 scopes: Optional[List[Scope]] = Field( 963 None, 964 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 965 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 966 title="Scopes", 967 ) 968 optional_scopes: Optional[List[OptionalScope]] = Field( 969 None, 970 description="Optional OAuth scope objects that may or may not be granted.", 971 examples=[[{"scope": "admin:read"}]], 972 title="Optional Scopes", 973 ) 974 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 975 ScopesJoinStrategy.space, 976 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 977 title="Scopes Join Strategy", 978 ) 979 access_token_url: str = Field( 980 ..., 981 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 982 examples=[ 983 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 984 ], 985 title="Access Token URL", 986 ) 987 access_token_headers: Optional[Dict[str, Any]] = Field( 988 None, 989 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 990 examples=[ 991 { 992 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 993 } 994 ], 995 title="Access Token Headers", 996 ) 997 access_token_params: Optional[Dict[str, Any]] = Field( 998 None, 999 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1000 examples=[ 1001 { 1002 "{{ auth_code_key }}": "{{ auth_code_value }}", 1003 "{{ client_id_key }}": "{{ client_id_value }}", 1004 "{{ client_secret_key }}": "{{ client_secret_value }}", 1005 } 1006 ], 1007 title="Access Token Query Params (Json Encoded)", 1008 ) 1009 extract_output: Optional[List[str]] = Field( 1010 None, 1011 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1012 examples=[["access_token", "refresh_token", "other_field"]], 1013 title="Extract Output", 1014 ) 1015 state: Optional[State] = Field( 1016 None, 1017 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1018 examples=[{"min": 7, "max": 128}], 1019 title="Configurable State Query Param", 1020 ) 1021 client_id_key: Optional[str] = Field( 1022 None, 1023 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1024 examples=["my_custom_client_id_key_name"], 1025 title="Client ID Key Override", 1026 ) 1027 client_secret_key: Optional[str] = Field( 1028 None, 1029 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1030 examples=["my_custom_client_secret_key_name"], 1031 title="Client Secret Key Override", 1032 ) 1033 scope_key: Optional[str] = Field( 1034 None, 1035 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1036 examples=["my_custom_scope_key_key_name"], 1037 title="Scopes Key Override", 1038 ) 1039 state_key: Optional[str] = Field( 1040 None, 1041 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1042 examples=["my_custom_state_key_key_name"], 1043 title="State Key Override", 1044 ) 1045 auth_code_key: Optional[str] = Field( 1046 None, 1047 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1048 examples=["my_custom_auth_code_key_name"], 1049 title="Auth Code Key Override", 1050 ) 1051 redirect_uri_key: Optional[str] = Field( 1052 None, 1053 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1054 examples=["my_custom_redirect_uri_key_name"], 1055 title="Redirect URI Key Override", 1056 ) 1057 1058 1059class OAuthConfigSpecification(BaseModel): 1060 class Config: 1061 extra = Extra.allow 1062 1063 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1064 None, 1065 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1066 examples=[ 1067 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1068 { 1069 "app_id": { 1070 "type": "string", 1071 "path_in_connector_config": ["info", "app_id"], 1072 } 1073 }, 1074 ], 1075 title="OAuth user input", 1076 ) 1077 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1078 None, 1079 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1080 title="DeclarativeOAuth Connector Specification", 1081 ) 1082 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1083 None, 1084 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1085 examples=[ 1086 { 1087 "refresh_token": { 1088 "type": "string,", 1089 "path_in_connector_config": ["credentials", "refresh_token"], 1090 } 1091 } 1092 ], 1093 title="OAuth output specification", 1094 ) 1095 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1096 None, 1097 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1098 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1099 title="OAuth input specification", 1100 ) 1101 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1102 None, 1103 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1104 examples=[ 1105 { 1106 "client_id": { 1107 "type": "string,", 1108 "path_in_connector_config": ["credentials", "client_id"], 1109 }, 1110 "client_secret": { 1111 "type": "string,", 1112 "path_in_connector_config": ["credentials", "client_secret"], 1113 }, 1114 } 1115 ], 1116 title="OAuth server output specification", 1117 ) 1118 1119 1120class OffsetIncrement(BaseModel): 1121 type: Literal["OffsetIncrement"] 1122 page_size: Optional[Union[int, str]] = Field( 1123 None, 1124 description="The number of records to include in each pages.", 1125 examples=[100, "{{ config['page_size'] }}"], 1126 title="Limit", 1127 ) 1128 inject_on_first_request: Optional[bool] = Field( 1129 False, 1130 description="Using the `offset` with value `0` during the first request", 1131 title="Inject Offset on First Request", 1132 ) 1133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1134 1135 1136class PageIncrement(BaseModel): 1137 type: Literal["PageIncrement"] 1138 page_size: Optional[Union[int, str]] = Field( 1139 None, 1140 description="The number of records to include in each pages.", 1141 examples=[100, "100", "{{ config['page_size'] }}"], 1142 title="Page Size", 1143 ) 1144 start_from_page: Optional[int] = Field( 1145 0, 1146 description="Index of the first page to request.", 1147 examples=[0, 1], 1148 title="Start From Page", 1149 ) 1150 inject_on_first_request: Optional[bool] = Field( 1151 False, 1152 description="Using the `page number` with value defined by `start_from_page` during the first request", 1153 title="Inject Page Number on First Request", 1154 ) 1155 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1156 1157 1158class PrimaryKey(BaseModel): 1159 __root__: Union[str, List[str], List[List[str]]] = Field( 1160 ..., 1161 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1162 examples=["id", ["code", "type"]], 1163 title="Primary Key", 1164 ) 1165 1166 1167class PropertyLimitType(Enum): 1168 characters = "characters" 1169 property_count = "property_count" 1170 1171 1172class PropertyChunking(BaseModel): 1173 type: Literal["PropertyChunking"] 1174 property_limit_type: PropertyLimitType = Field( 1175 ..., 1176 description="The type used to determine the maximum number of properties per chunk", 1177 title="Property Limit Type", 1178 ) 1179 property_limit: Optional[int] = Field( 1180 None, 1181 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1182 title="Property Limit", 1183 ) 1184 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1185 None, 1186 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1187 title="Record Merge Strategy", 1188 ) 1189 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1190 1191 1192class RecordFilter(BaseModel): 1193 type: Literal["RecordFilter"] 1194 condition: Optional[str] = Field( 1195 "", 1196 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1197 examples=[ 1198 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1199 "{{ record.status in ['active', 'expired'] }}", 1200 ], 1201 title="Condition", 1202 ) 1203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1204 1205 1206class SchemaNormalization(Enum): 1207 Default = "Default" 1208 None_ = "None" 1209 1210 1211class RemoveFields(BaseModel): 1212 type: Literal["RemoveFields"] 1213 condition: Optional[str] = Field( 1214 "", 1215 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1216 examples=[ 1217 "{{ property|string == '' }}", 1218 "{{ property is integer }}", 1219 "{{ property|length > 5 }}", 1220 "{{ property == 'some_string_to_match' }}", 1221 ], 1222 ) 1223 field_pointers: List[List[str]] = Field( 1224 ..., 1225 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1226 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1227 title="Field Paths", 1228 ) 1229 1230 1231class RequestPath(BaseModel): 1232 type: Literal["RequestPath"] 1233 1234 1235class InjectInto(Enum): 1236 request_parameter = "request_parameter" 1237 header = "header" 1238 body_data = "body_data" 1239 body_json = "body_json" 1240 1241 1242class RequestOption(BaseModel): 1243 type: Literal["RequestOption"] 1244 inject_into: InjectInto = Field( 1245 ..., 1246 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1247 examples=["request_parameter", "header", "body_data", "body_json"], 1248 title="Inject Into", 1249 ) 1250 field_name: Optional[str] = Field( 1251 None, 1252 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1253 examples=["segment_id"], 1254 title="Field Name", 1255 ) 1256 field_path: Optional[List[str]] = Field( 1257 None, 1258 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1259 examples=[["data", "viewer", "id"]], 1260 title="Field Path", 1261 ) 1262 1263 1264class Schemas(BaseModel): 1265 pass 1266 1267 class Config: 1268 extra = Extra.allow 1269 1270 1271class LegacySessionTokenAuthenticator(BaseModel): 1272 type: Literal["LegacySessionTokenAuthenticator"] 1273 header: str = Field( 1274 ..., 1275 description="The name of the session token header that will be injected in the request", 1276 examples=["X-Session"], 1277 title="Session Request Header", 1278 ) 1279 login_url: str = Field( 1280 ..., 1281 description="Path of the login URL (do not include the base URL)", 1282 examples=["session"], 1283 title="Login Path", 1284 ) 1285 session_token: Optional[str] = Field( 1286 None, 1287 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1288 example=["{{ config['session_token'] }}"], 1289 title="Session Token", 1290 ) 1291 session_token_response_key: str = Field( 1292 ..., 1293 description="Name of the key of the session token to be extracted from the response", 1294 examples=["id"], 1295 title="Response Token Response Key", 1296 ) 1297 username: Optional[str] = Field( 1298 None, 1299 description="Username used to authenticate and obtain a session token", 1300 examples=[" {{ config['username'] }}"], 1301 title="Username", 1302 ) 1303 password: Optional[str] = Field( 1304 "", 1305 description="Password used to authenticate and obtain a session token", 1306 examples=["{{ config['password'] }}", ""], 1307 title="Password", 1308 ) 1309 validate_session_url: str = Field( 1310 ..., 1311 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1312 examples=["user/current"], 1313 title="Validate Session Path", 1314 ) 1315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1316 1317 1318class Action1(Enum): 1319 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1320 RESET = "RESET" 1321 1322 1323class PaginationResetLimits(BaseModel): 1324 type: Literal["PaginationResetLimits"] 1325 number_of_records: Optional[int] = None 1326 1327 1328class ResetPolicy(Enum): 1329 NEVER = "NEVER" 1330 AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE" 1331 1332 1333class PageSizeReduction(BaseModel): 1334 type: Literal["PageSizeReduction"] 1335 reduction_factor: Optional[float] = Field( 1336 2, 1337 description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", 1338 examples=[2, 4], 1339 gt=1, 1340 title="Reduction Factor", 1341 ) 1342 minimum_page_size: Optional[int] = Field( 1343 1, 1344 description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", 1345 examples=[1, 10], 1346 ge=1, 1347 title="Minimum Page Size", 1348 ) 1349 max_attempts: Optional[int] = Field( 1350 5, 1351 description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", 1352 examples=[5, 10], 1353 ge=1, 1354 title="Maximum Reduction Attempts", 1355 ) 1356 backoff_seconds: Optional[float] = Field( 1357 0.5, 1358 description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', 1359 examples=[0.5, 5], 1360 ge=0, 1361 title="Backoff Seconds", 1362 ) 1363 retries_at_minimum_page_size: Optional[int] = Field( 1364 0, 1365 description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", 1366 examples=[0, 3], 1367 ge=0, 1368 title="Retries At Minimum Page Size", 1369 ) 1370 failure_message: Optional[str] = Field( 1371 None, 1372 description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", 1373 examples=[ 1374 "Narrow the sync down by selecting fewer fields on this stream.", 1375 "Set a more recent start date so that each page covers less data.", 1376 ], 1377 title="Failure Message", 1378 ) 1379 reset_policy: Optional[ResetPolicy] = Field( 1380 ResetPolicy.NEVER, 1381 description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", 1382 title="Reset Policy", 1383 ) 1384 1385 1386class RequestWindowSplitting(BaseModel): 1387 type: Literal["RequestWindowSplitting"] 1388 failure_message: Optional[str] = Field( 1389 None, 1390 description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", 1391 examples=[ 1392 "Narrow the sync down by selecting fewer fields on this stream.", 1393 "Lower time_window so that each request covers less data.", 1394 ], 1395 title="Failure Message", 1396 ) 1397 min_split_window: Optional[str] = Field( 1398 None, 1399 description="Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or below this size is not split further, though the result can end up a bit smaller than this, not larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", 1400 examples=["P1D", "PT1H"], 1401 title="Minimum Split Window", 1402 ) 1403 1404 1405class CsvDecoder(BaseModel): 1406 type: Literal["CsvDecoder"] 1407 encoding: Optional[str] = "utf-8" 1408 delimiter: Optional[str] = "," 1409 set_values_to_none: Optional[List[str]] = None 1410 1411 1412class AsyncJobStatusMap(BaseModel): 1413 type: Optional[Literal["AsyncJobStatusMap"]] = None 1414 running: List[str] 1415 completed: List[str] 1416 failed: List[str] 1417 timeout: List[str] 1418 skipped: Optional[List[str]] = Field( 1419 None, 1420 description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", 1421 ) 1422 1423 1424class BlockSimultaneousSyncsAction(BaseModel): 1425 type: Literal["BlockSimultaneousSyncsAction"] 1426 1427 1428class ValueType(Enum): 1429 string = "string" 1430 number = "number" 1431 integer = "integer" 1432 boolean = "boolean" 1433 1434 1435class WaitTimeFromHeader(BaseModel): 1436 type: Literal["WaitTimeFromHeader"] 1437 header: str = Field( 1438 ..., 1439 description="The name of the response header defining how long to wait before retrying.", 1440 examples=["Retry-After"], 1441 title="Response Header Name", 1442 ) 1443 regex: Optional[str] = Field( 1444 None, 1445 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1446 examples=["([-+]?\\d+)"], 1447 title="Extraction Regex", 1448 ) 1449 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1450 None, 1451 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1452 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1453 title="Max Waiting Time in Seconds", 1454 ) 1455 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1456 1457 1458class WaitTimeFromResponse(BaseModel): 1459 type: Literal["WaitTimeFromResponse"] 1460 wait_time_in_seconds: str = Field( 1461 ..., 1462 description="The expression defining how long to wait before retrying, in seconds relative to now (convert an absolute timestamp with `timestamp(...) - now_utc().timestamp()`). An empty, zero, negative or non-numeric result, or an expression that cannot be evaluated on the response, defers to the next backoff strategy. Nothing bounds the wait unless `max_waiting_time_in_seconds` is set, so setting it is strongly recommended.", 1463 examples=[ 1464 "{{ response.retry_after_seconds }}", 1465 "{{ max(1, timestamp(response.error.message | regex_search('Retry after (\\S+Z)')) - now_utc().timestamp()) if (response is mapping and response.get('error') is mapping and ((response.error.message or '') | string | regex_search('Retry after (\\S+Z)'))) else '' }}", 1466 ], 1467 title="Wait Time in Seconds", 1468 ) 1469 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1470 None, 1471 description="Stop the stream instead of waiting, when the computed wait is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1472 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1473 title="Max Waiting Time in Seconds", 1474 ) 1475 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1476 1477 1478class WaitUntilTimeFromHeader(BaseModel): 1479 type: Literal["WaitUntilTimeFromHeader"] 1480 header: str = Field( 1481 ..., 1482 description="The name of the response header defining how long to wait before retrying.", 1483 examples=["wait_time"], 1484 title="Response Header", 1485 ) 1486 min_wait: Optional[Union[float, str]] = Field( 1487 None, 1488 description="Minimum time to wait before retrying.", 1489 examples=[10, "60"], 1490 title="Minimum Wait Time", 1491 ) 1492 regex: Optional[str] = Field( 1493 None, 1494 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1495 examples=["([-+]?\\d+)"], 1496 title="Extraction Regex", 1497 ) 1498 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1499 None, 1500 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1501 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1502 title="Max Waiting Time in Seconds", 1503 ) 1504 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1505 1506 1507class ComponentMappingDefinition(BaseModel): 1508 type: Literal["ComponentMappingDefinition"] 1509 field_path: List[str] = Field( 1510 ..., 1511 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1512 examples=[ 1513 ["name"], 1514 ["retriever", "requester", "url"], 1515 ["retriever", "requester", "{{ components_values.field }}"], 1516 ["*", "**", "name"], 1517 ], 1518 title="Field Path", 1519 ) 1520 value: str = Field( 1521 ..., 1522 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1523 examples=[ 1524 "{{ components_values['updates'] }}", 1525 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1526 "{{ config['segment_id'] }}", 1527 "{{ stream_slice['parent_id'] }}", 1528 "{{ stream_slice['extra_fields']['name'] }}", 1529 ], 1530 title="Value", 1531 ) 1532 value_type: Optional[ValueType] = Field( 1533 None, 1534 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1535 title="Value Type", 1536 ) 1537 create_or_update: Optional[bool] = Field( 1538 False, 1539 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1540 title="Create or Update", 1541 ) 1542 condition: Optional[str] = Field( 1543 None, 1544 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1545 examples=[ 1546 "{{ components_values.get('cursor_field', None) }}", 1547 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1548 ], 1549 title="Condition", 1550 ) 1551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1552 1553 1554class StreamConfig(BaseModel): 1555 type: Literal["StreamConfig"] 1556 configs_pointer: List[str] = Field( 1557 ..., 1558 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1559 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1560 title="Configs Pointer", 1561 ) 1562 default_values: Optional[List[Dict[str, Any]]] = Field( 1563 None, 1564 description="A list of default values, each matching the structure expected from the parsed component value.", 1565 title="Default Values", 1566 ) 1567 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1568 1569 1570class ConfigComponentsResolver(BaseModel): 1571 type: Literal["ConfigComponentsResolver"] 1572 stream_config: Union[List[StreamConfig], StreamConfig] 1573 components_mapping: List[ComponentMappingDefinition] 1574 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1575 1576 1577class StreamParametersDefinition(BaseModel): 1578 type: Literal["StreamParametersDefinition"] 1579 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1580 ..., 1581 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1582 examples=[ 1583 [ 1584 { 1585 "name": "test stream", 1586 "$parameters": {"entity": "test entity"}, 1587 "primary_key": "test key", 1588 } 1589 ] 1590 ], 1591 title="Stream Parameters", 1592 ) 1593 1594 1595class ParametrizedComponentsResolver(BaseModel): 1596 type: Literal["ParametrizedComponentsResolver"] 1597 stream_parameters: StreamParametersDefinition 1598 components_mapping: List[ComponentMappingDefinition] 1599 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1600 1601 1602class RequestBodyPlainText(BaseModel): 1603 type: Literal["RequestBodyPlainText"] 1604 value: str 1605 1606 1607class RequestBodyUrlEncodedForm(BaseModel): 1608 type: Literal["RequestBodyUrlEncodedForm"] 1609 value: Dict[str, str] 1610 1611 1612class RequestBodyJsonObject(BaseModel): 1613 type: Literal["RequestBodyJsonObject"] 1614 value: Dict[str, Any] 1615 1616 1617class RequestBodyGraphQlQuery(BaseModel): 1618 class Config: 1619 extra = Extra.allow 1620 1621 query: str = Field(..., description="The GraphQL query to be executed") 1622 1623 1624class ValidateAdheresToSchema(BaseModel): 1625 type: Literal["ValidateAdheresToSchema"] 1626 base_schema: Union[str, Dict[str, Any]] = Field( 1627 ..., 1628 description="The base JSON schema against which the user-provided schema will be validated.", 1629 examples=[ 1630 "{{ config['report_validation_schema'] }}", 1631 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1632 { 1633 "$schema": "http://json-schema.org/draft-07/schema#", 1634 "title": "Person", 1635 "type": "object", 1636 "properties": { 1637 "name": {"type": "string", "description": "The person's name"}, 1638 "age": { 1639 "type": "integer", 1640 "minimum": 0, 1641 "description": "The person's age", 1642 }, 1643 }, 1644 "required": ["name", "age"], 1645 }, 1646 ], 1647 title="Base JSON Schema", 1648 ) 1649 1650 1651class CustomValidationStrategy(BaseModel): 1652 class Config: 1653 extra = Extra.allow 1654 1655 type: Literal["CustomValidationStrategy"] 1656 class_name: str = Field( 1657 ..., 1658 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1659 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1660 title="Class Name", 1661 ) 1662 1663 1664class ConfigRemapField(BaseModel): 1665 type: Literal["ConfigRemapField"] 1666 map: Union[Dict[str, Any], str] = Field( 1667 ..., 1668 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1669 examples=[ 1670 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1671 "{{ config['status_mapping'] }}", 1672 ], 1673 title="Value Mapping", 1674 ) 1675 field_path: List[str] = Field( 1676 ..., 1677 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1678 examples=[ 1679 ["status"], 1680 ["data", "status"], 1681 ["data", "{{ config.name }}", "status"], 1682 ["data", "*", "status"], 1683 ], 1684 title="Field Path", 1685 ) 1686 1687 1688class ConfigRemoveFields(BaseModel): 1689 type: Literal["ConfigRemoveFields"] 1690 field_pointers: List[List[str]] = Field( 1691 ..., 1692 description="A list of field pointers to be removed from the config.", 1693 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1694 title="Field Pointers", 1695 ) 1696 condition: Optional[str] = Field( 1697 "", 1698 description="Fields will be removed if expression is evaluated to True.", 1699 examples=[ 1700 "{{ config['environemnt'] == 'sandbox' }}", 1701 "{{ property is integer }}", 1702 "{{ property|length > 5 }}", 1703 "{{ property == 'some_string_to_match' }}", 1704 ], 1705 ) 1706 1707 1708class CustomConfigTransformation(BaseModel): 1709 type: Literal["CustomConfigTransformation"] 1710 class_name: str = Field( 1711 ..., 1712 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1713 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1714 ) 1715 parameters: Optional[Dict[str, Any]] = Field( 1716 None, 1717 alias="$parameters", 1718 description="Additional parameters to be passed to the custom config transformation.", 1719 ) 1720 1721 1722class AddedFieldDefinition(BaseModel): 1723 type: Literal["AddedFieldDefinition"] 1724 path: List[str] = Field( 1725 ..., 1726 description="List of strings defining the path where to add the value on the record.", 1727 examples=[["segment_id"], ["metadata", "segment_id"]], 1728 title="Path", 1729 ) 1730 value: str = Field( 1731 ..., 1732 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1733 examples=[ 1734 "{{ record['updates'] }}", 1735 "{{ record['MetaData']['LastUpdatedTime'] }}", 1736 "{{ stream_partition['segment_id'] }}", 1737 ], 1738 title="Value", 1739 ) 1740 value_type: Optional[ValueType] = Field( 1741 None, 1742 description="Type of the value. If not specified, the type will be inferred from the value.", 1743 title="Value Type", 1744 ) 1745 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1746 1747 1748class AddFields(BaseModel): 1749 type: Literal["AddFields"] 1750 fields: List[AddedFieldDefinition] = Field( 1751 ..., 1752 description="List of transformations (path and corresponding value) that will be added to the record.", 1753 title="Fields", 1754 ) 1755 condition: Optional[str] = Field( 1756 "", 1757 description="Fields will be added if expression is evaluated to True.", 1758 examples=[ 1759 "{{ property|string == '' }}", 1760 "{{ property is integer }}", 1761 "{{ property|length > 5 }}", 1762 "{{ property == 'some_string_to_match' }}", 1763 ], 1764 ) 1765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1766 1767 1768class ApiKeyAuthenticator(BaseModel): 1769 type: Literal["ApiKeyAuthenticator"] 1770 api_token: Optional[str] = Field( 1771 None, 1772 description="The API key to inject in the request. Fill it in the user inputs.", 1773 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1774 title="API Key", 1775 ) 1776 header: Optional[str] = Field( 1777 None, 1778 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1779 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1780 title="Header Name", 1781 ) 1782 inject_into: Optional[RequestOption] = Field( 1783 None, 1784 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1785 examples=[ 1786 {"inject_into": "header", "field_name": "Authorization"}, 1787 {"inject_into": "request_parameter", "field_name": "authKey"}, 1788 ], 1789 title="Inject API Key Into Outgoing HTTP Request", 1790 ) 1791 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1792 1793 1794class AuthFlow(BaseModel): 1795 auth_flow_type: Optional[AuthFlowType] = Field( 1796 None, description="The type of auth to use", title="Auth flow type" 1797 ) 1798 predicate_key: Optional[List[str]] = Field( 1799 None, 1800 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1801 examples=[["credentials", "auth_type"]], 1802 title="Predicate key", 1803 ) 1804 predicate_value: Optional[str] = Field( 1805 None, 1806 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1807 examples=["Oauth"], 1808 title="Predicate value", 1809 ) 1810 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1811 1812 1813class TokenQuota(BaseModel): 1814 type: Literal["TokenQuota"] 1815 name: str = Field( 1816 ..., 1817 description="Name of the quota pool.", 1818 examples=["rest", "graphql"], 1819 title="Name", 1820 ) 1821 remaining_path: List[str] = Field( 1822 ..., 1823 description="Path to the remaining call count for this pool in the quota status response.", 1824 examples=[["resources", "core", "remaining"]], 1825 title="Remaining Path", 1826 ) 1827 reset_path: List[str] = Field( 1828 ..., 1829 description="Path to the quota reset timestamp for this pool in the quota status response.", 1830 examples=[["resources", "core", "reset"]], 1831 title="Reset Path", 1832 ) 1833 limit_path: Optional[List[str]] = Field( 1834 None, 1835 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 1836 examples=[["resources", "core", "limit"]], 1837 title="Limit Path", 1838 ) 1839 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 1840 None, 1841 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 1842 title="Matchers", 1843 ) 1844 remaining_header: Optional[str] = Field( 1845 None, 1846 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 1847 examples=["X-RateLimit-Remaining"], 1848 title="Remaining Header", 1849 ) 1850 reset_header: Optional[str] = Field( 1851 None, 1852 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 1853 examples=["X-RateLimit-Reset"], 1854 title="Reset Header", 1855 ) 1856 limit_header: Optional[str] = Field( 1857 None, 1858 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 1859 examples=["X-RateLimit-Limit"], 1860 title="Limit Header", 1861 ) 1862 exhaustion_status_codes: Optional[List[int]] = Field( 1863 None, 1864 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 1865 examples=[[429]], 1866 title="Exhaustion Status Codes", 1867 ) 1868 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1869 1870 1871class CheckStream(BaseModel): 1872 type: Literal["CheckStream"] 1873 stream_names: Optional[List[str]] = Field( 1874 None, 1875 description="Names of the streams to try reading from when running a check operation.", 1876 examples=[["users"], ["users", "contacts"]], 1877 title="Stream Names", 1878 ) 1879 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1880 config_overrides: Optional[Dict[str, Any]] = Field( 1881 None, 1882 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1883 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1884 title="Config Overrides", 1885 ) 1886 1887 1888class IncrementingCountCursor(BaseModel): 1889 type: Literal["IncrementingCountCursor"] 1890 cursor_field: str = Field( 1891 ..., 1892 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1893 examples=["created_at", "{{ config['record_cursor'] }}"], 1894 title="Cursor Field", 1895 ) 1896 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1897 None, 1898 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1899 title="Allow Catalog Defined Cursor Field", 1900 ) 1901 start_value: Optional[Union[str, int]] = Field( 1902 None, 1903 description="The value that determines the earliest record that should be synced.", 1904 examples=[0, "{{ config['start_value'] }}"], 1905 title="Start Value", 1906 ) 1907 start_value_option: Optional[RequestOption] = Field( 1908 None, 1909 description="Optionally configures how the start value will be sent in requests to the source API.", 1910 title="Inject Start Value Into Outgoing HTTP Request", 1911 ) 1912 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1913 1914 1915class DatetimeBasedCursor(BaseModel): 1916 type: Literal["DatetimeBasedCursor"] 1917 clamping: Optional[Clamping] = Field( 1918 None, 1919 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1920 title="Date Range Clamping", 1921 ) 1922 cursor_field: str = Field( 1923 ..., 1924 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1925 examples=["created_at", "{{ config['record_cursor'] }}"], 1926 title="Cursor Field", 1927 ) 1928 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1929 None, 1930 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1931 title="Allow Catalog Defined Cursor Field", 1932 ) 1933 cursor_datetime_formats: Optional[List[str]] = Field( 1934 None, 1935 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1936 examples=[ 1937 "%Y-%m-%d", 1938 "%Y-%m-%d %H:%M:%S", 1939 "%Y-%m-%dT%H:%M:%S", 1940 "%Y-%m-%dT%H:%M:%SZ", 1941 "%Y-%m-%dT%H:%M:%S%z", 1942 "%Y-%m-%dT%H:%M:%S.%fZ", 1943 "%Y-%m-%dT%H:%M:%S.%f%z", 1944 "%Y-%m-%d %H:%M:%S.%f+00:00", 1945 "%s", 1946 "%ms", 1947 ], 1948 title="Cursor Datetime Formats", 1949 ) 1950 start_datetime: Union[MinMaxDatetime, str] = Field( 1951 ..., 1952 description="The datetime that determines the earliest record that should be synced.", 1953 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1954 title="Start Datetime", 1955 ) 1956 start_time_option: Optional[RequestOption] = Field( 1957 None, 1958 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1959 title="Inject Start Time Into Outgoing HTTP Request", 1960 ) 1961 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1962 None, 1963 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1964 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1965 title="End Datetime", 1966 ) 1967 end_time_option: Optional[RequestOption] = Field( 1968 None, 1969 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1970 title="Inject End Time Into Outgoing HTTP Request", 1971 ) 1972 datetime_format: str = Field( 1973 ..., 1974 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1975 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1976 title="Outgoing Datetime Format", 1977 ) 1978 cursor_granularity: Optional[str] = Field( 1979 None, 1980 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1981 examples=["PT1S"], 1982 title="Cursor Granularity", 1983 ) 1984 is_data_feed: Optional[bool] = Field( 1985 None, 1986 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1987 title="Data Feed API", 1988 ) 1989 is_client_side_incremental: Optional[bool] = Field( 1990 None, 1991 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1992 title="Client-side Incremental Filtering", 1993 ) 1994 is_compare_strictly: Optional[bool] = Field( 1995 False, 1996 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1997 title="Strict Start-End Time Comparison", 1998 ) 1999 global_substream_cursor: Optional[bool] = Field( 2000 False, 2001 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 2002 title="Global Substream Cursor", 2003 ) 2004 lookback_window: Optional[str] = Field( 2005 None, 2006 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2007 examples=["P1D", "P{{ config['lookback_days'] }}D"], 2008 title="Lookback Window", 2009 ) 2010 partition_field_end: Optional[str] = Field( 2011 None, 2012 description="Name of the partition start time field.", 2013 examples=["ending_time"], 2014 title="Partition Field End", 2015 ) 2016 partition_field_start: Optional[str] = Field( 2017 None, 2018 description="Name of the partition end time field.", 2019 examples=["starting_time"], 2020 title="Partition Field Start", 2021 ) 2022 step: Optional[str] = Field( 2023 None, 2024 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2025 examples=["P1W", "{{ config['step_increment'] }}"], 2026 title="Step", 2027 ) 2028 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2029 2030 2031class JwtAuthenticator(BaseModel): 2032 type: Literal["JwtAuthenticator"] 2033 secret_key: str = Field( 2034 ..., 2035 description="Secret used to sign the JSON web token.", 2036 examples=["{{ config['secret_key'] }}"], 2037 title="Secret Key", 2038 ) 2039 base64_encode_secret_key: Optional[bool] = Field( 2040 False, 2041 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 2042 title="Base64-encode Secret Key", 2043 ) 2044 algorithm: Algorithm = Field( 2045 ..., 2046 description="Algorithm used to sign the JSON web token.", 2047 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 2048 title="Algorithm", 2049 ) 2050 token_duration: Optional[int] = Field( 2051 1200, 2052 description="The amount of time in seconds a JWT token can be valid after being issued.", 2053 examples=[1200, 3600], 2054 title="Token Duration", 2055 ) 2056 header_prefix: Optional[str] = Field( 2057 None, 2058 description="The prefix to be used within the Authentication header.", 2059 examples=["Bearer", "Basic"], 2060 title="Header Prefix", 2061 ) 2062 jwt_headers: Optional[JwtHeaders] = Field( 2063 None, 2064 description="JWT headers used when signing JSON web token.", 2065 title="JWT Headers", 2066 ) 2067 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 2068 None, 2069 description="Additional headers to be included with the JWT headers object.", 2070 title="Additional JWT Headers", 2071 ) 2072 jwt_payload: Optional[JwtPayload] = Field( 2073 None, 2074 description="JWT Payload used when signing JSON web token.", 2075 title="JWT Payload", 2076 ) 2077 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2078 None, 2079 description="Additional properties to be added to the JWT payload.", 2080 title="Additional JWT Payload Properties", 2081 ) 2082 passphrase: Optional[str] = Field( 2083 None, 2084 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2085 examples=["{{ config['passphrase'] }}"], 2086 title="Passphrase", 2087 ) 2088 request_option: Optional[RequestOption] = Field( 2089 None, 2090 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2091 title="Request Option", 2092 ) 2093 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2094 2095 2096class OAuthAuthenticator(BaseModel): 2097 type: Literal["OAuthAuthenticator"] 2098 client_id_name: Optional[str] = Field( 2099 "client_id", 2100 description="The name of the property to use to refresh the `access_token`.", 2101 examples=["custom_app_id"], 2102 title="Client ID Property Name", 2103 ) 2104 client_id: Optional[str] = Field( 2105 None, 2106 description="The OAuth client ID. Fill it in the user inputs.", 2107 examples=[ 2108 "{{ config['client_id'] }}", 2109 "{{ config['credentials']['client_id }}", 2110 ], 2111 title="Client ID", 2112 ) 2113 client_secret_name: Optional[str] = Field( 2114 "client_secret", 2115 description="The name of the property to use to refresh the `access_token`.", 2116 examples=["custom_app_secret"], 2117 title="Client Secret Property Name", 2118 ) 2119 client_secret: Optional[str] = Field( 2120 None, 2121 description="The OAuth client secret. Fill it in the user inputs.", 2122 examples=[ 2123 "{{ config['client_secret'] }}", 2124 "{{ config['credentials']['client_secret }}", 2125 ], 2126 title="Client Secret", 2127 ) 2128 refresh_token_name: Optional[str] = Field( 2129 "refresh_token", 2130 description="The name of the property to use to refresh the `access_token`.", 2131 examples=["custom_app_refresh_value"], 2132 title="Refresh Token Property Name", 2133 ) 2134 refresh_token: Optional[str] = Field( 2135 None, 2136 description="Credential artifact used to get a new access token.", 2137 examples=[ 2138 "{{ config['refresh_token'] }}", 2139 "{{ config['credentials]['refresh_token'] }}", 2140 ], 2141 title="Refresh Token", 2142 ) 2143 token_refresh_endpoint: Optional[str] = Field( 2144 None, 2145 description="The full URL to call to obtain a new access token.", 2146 examples=["https://connect.squareup.com/oauth2/token"], 2147 title="Token Refresh Endpoint", 2148 ) 2149 access_token_name: Optional[str] = Field( 2150 "access_token", 2151 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2152 examples=["access_token"], 2153 title="Access Token Property Name", 2154 ) 2155 access_token_value: Optional[str] = Field( 2156 None, 2157 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2158 examples=["secret_access_token_value"], 2159 title="Access Token Value", 2160 ) 2161 expires_in_name: Optional[str] = Field( 2162 "expires_in", 2163 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2164 examples=["expires_in"], 2165 title="Token Expiry Property Name", 2166 ) 2167 grant_type_name: Optional[str] = Field( 2168 "grant_type", 2169 description="The name of the property to use to refresh the `access_token`.", 2170 examples=["custom_grant_type"], 2171 title="Grant Type Property Name", 2172 ) 2173 grant_type: Optional[str] = Field( 2174 "refresh_token", 2175 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2176 examples=["refresh_token", "client_credentials"], 2177 title="Grant Type", 2178 ) 2179 refresh_request_body: Optional[Dict[str, Any]] = Field( 2180 None, 2181 description="Body of the request sent to get a new access token.", 2182 examples=[ 2183 { 2184 "applicationId": "{{ config['application_id'] }}", 2185 "applicationSecret": "{{ config['application_secret'] }}", 2186 "token": "{{ config['token'] }}", 2187 } 2188 ], 2189 title="Refresh Request Body", 2190 ) 2191 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2192 None, 2193 description="Headers of the request sent to get a new access token.", 2194 examples=[ 2195 { 2196 "Authorization": "<AUTH_TOKEN>", 2197 "Content-Type": "application/x-www-form-urlencoded", 2198 } 2199 ], 2200 title="Refresh Request Headers", 2201 ) 2202 send_refresh_request_as_query_params: Optional[bool] = Field( 2203 False, 2204 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2205 examples=[True], 2206 title="Send Refresh Request As Query Params", 2207 ) 2208 scopes: Optional[List[str]] = Field( 2209 None, 2210 description="List of scopes that should be granted to the access token.", 2211 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2212 title="Scopes", 2213 ) 2214 token_expiry_date: Optional[str] = Field( 2215 None, 2216 description="The access token expiry date.", 2217 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2218 title="Token Expiry Date", 2219 ) 2220 token_expiry_date_format: Optional[str] = Field( 2221 None, 2222 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2223 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2224 title="Token Expiry Date Format", 2225 ) 2226 refresh_token_error_status_codes: Optional[List[int]] = Field( 2227 None, 2228 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2229 examples=[[400, 500]], 2230 title="Refresh Token Error Status Codes", 2231 ) 2232 refresh_token_error_key: Optional[str] = Field( 2233 None, 2234 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2235 examples=["error"], 2236 title="Refresh Token Error Key", 2237 ) 2238 refresh_token_error_values: Optional[List[str]] = Field( 2239 None, 2240 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2241 examples=[["invalid_grant", "invalid_permissions"]], 2242 title="Refresh Token Error Values", 2243 ) 2244 refresh_token_error_messages: Optional[Dict[str, str]] = Field( 2245 None, 2246 description="Fixed user-facing messages for refresh token errors. Keys are matched in order against the provider error code leading error_description (a token such as ABC12345 followed by a word boundary), the whitespace-normalized full error_description, then the OAuth error value; the first match replaces the default re-authentication guidance, and the provider error detail is still appended. Only applies to responses already flagged as config errors by Refresh Token Error Status Codes, Refresh Token Error Key and Refresh Token Error Values (or their defaults); it does not flag any additional responses. Values are used verbatim, without interpolation, so the message is identical on every attempt.", 2247 examples=[ 2248 { 2249 "invalid_grant": "The refresh token expired or was revoked. Re-authenticate this source to restore syncs." 2250 } 2251 ], 2252 title="Refresh Token Error Messages", 2253 ) 2254 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2255 None, 2256 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2257 title="Refresh Token Updater", 2258 ) 2259 profile_assertion: Optional[JwtAuthenticator] = Field( 2260 None, 2261 description="The authenticator being used to authenticate the client authenticator.", 2262 title="Profile Assertion", 2263 ) 2264 use_profile_assertion: Optional[bool] = Field( 2265 False, 2266 description="Enable using profile assertion as a flow for OAuth authorization.", 2267 title="Use Profile Assertion", 2268 ) 2269 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2270 2271 2272class FixedWindowCallRatePolicy(BaseModel): 2273 class Config: 2274 extra = Extra.allow 2275 2276 type: Literal["FixedWindowCallRatePolicy"] 2277 period: str = Field( 2278 ..., description="The time interval for the rate limit window.", title="Period" 2279 ) 2280 call_limit: int = Field( 2281 ..., 2282 description="The maximum number of calls allowed within the period.", 2283 title="Call Limit", 2284 ) 2285 matchers: List[HttpRequestRegexMatcher] = Field( 2286 ..., 2287 description="List of matchers that define which requests this policy applies to.", 2288 title="Matchers", 2289 ) 2290 2291 2292class MovingWindowCallRatePolicy(BaseModel): 2293 class Config: 2294 extra = Extra.allow 2295 2296 type: Literal["MovingWindowCallRatePolicy"] 2297 rates: List[Rate] = Field( 2298 ..., 2299 description="List of rates that define the call limits for different time intervals.", 2300 title="Rates", 2301 ) 2302 matchers: List[HttpRequestRegexMatcher] = Field( 2303 ..., 2304 description="List of matchers that define which requests this policy applies to.", 2305 title="Matchers", 2306 ) 2307 2308 2309class UnlimitedCallRatePolicy(BaseModel): 2310 class Config: 2311 extra = Extra.allow 2312 2313 type: Literal["UnlimitedCallRatePolicy"] 2314 matchers: List[HttpRequestRegexMatcher] = Field( 2315 ..., 2316 description="List of matchers that define which requests this policy applies to.", 2317 title="Matchers", 2318 ) 2319 2320 2321class DefaultErrorHandler(BaseModel): 2322 type: Literal["DefaultErrorHandler"] 2323 backoff_strategies: Optional[ 2324 List[ 2325 Union[ 2326 ConstantBackoffStrategy, 2327 ExponentialBackoffStrategy, 2328 WaitTimeFromHeader, 2329 WaitTimeFromResponse, 2330 WaitUntilTimeFromHeader, 2331 CustomBackoffStrategy, 2332 ] 2333 ] 2334 ] = Field( 2335 None, 2336 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2337 title="Backoff Strategies", 2338 ) 2339 max_retries: Optional[Union[int, str]] = Field( 2340 5, 2341 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2342 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2343 title="Max Retry Count", 2344 ) 2345 response_filters: Optional[List[HttpResponseFilter]] = Field( 2346 None, 2347 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2348 title="Response Filters", 2349 ) 2350 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2351 2352 2353class DefaultPaginator(BaseModel): 2354 type: Literal["DefaultPaginator"] 2355 pagination_strategy: Union[ 2356 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2357 ] = Field( 2358 ..., 2359 description="Strategy defining how records are paginated.", 2360 title="Pagination Strategy", 2361 ) 2362 page_size_option: Optional[RequestOption] = Field( 2363 None, title="Inject Page Size Into Outgoing HTTP Request" 2364 ) 2365 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2366 None, 2367 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2368 title="Inject Page Token Into Outgoing HTTP Request", 2369 ) 2370 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2371 2372 2373class RecordExpander(BaseModel): 2374 type: Literal["RecordExpander"] 2375 expand_records_from_field: List[str] = Field( 2376 ..., 2377 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2378 examples=[ 2379 ["lines", "data"], 2380 ["items"], 2381 ["nested", "array"], 2382 ["sections", "*", "items"], 2383 ], 2384 title="Expand Records From Field", 2385 ) 2386 remain_original_record: Optional[bool] = Field( 2387 False, 2388 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2389 title="Remain Original Record", 2390 ) 2391 parent_fields: Optional[List[ParentFieldPath]] = Field( 2392 None, 2393 description="Named values to copy from the record being expanded onto each expanded item. Use this instead of `remain_original_record` when only a few parent fields are needed: it copies the named values rather than deep-copying the whole parent once per item, which matters when the parent record is large and the nested list is long. An existing value at `record_path` is overwritten, and a `parent_path` the parent does not have copies null. A copied object or array is deep-copied, so a transformation that writes inside it affects only that item. Independent of `remain_original_record`; both may be set. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2394 title="Parent Fields", 2395 ) 2396 merge_parent: Optional[bool] = Field( 2397 False, 2398 description="If true, each expanded item is the parent record shallow-merged with the item, the item's own keys winning on collision, and the expanded list removed from the parent's copy. Only the value at `expand_records_from_field` is removed, so for a multi-segment path the top-level key stays with its other fields. Each item receives its own deep copy of the merged parent fields, so a transformation that writes into a nested value affects only that item. The merge happens first, then `parent_fields` are copied, then `original_record` is embedded when `remain_original_record` is set; all three may be combined. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2399 title="Merge Parent", 2400 ) 2401 on_no_records: Optional[OnNoRecords] = Field( 2402 OnNoRecords.skip, 2403 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2404 title="On No Records", 2405 ) 2406 truncation_indicator_path: Optional[List[str]] = Field( 2407 None, 2408 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2409 examples=[["data", "object", "lines", "has_more"]], 2410 title="Truncation Indicator Path", 2411 ) 2412 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2413 None, 2414 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2415 title="Truncated List Retriever", 2416 ) 2417 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2418 2419 2420class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2421 type: Literal["ApiKey"] 2422 inject_into: RequestOption = Field( 2423 ..., 2424 description="Configure how the API Key will be sent in requests to the source API.", 2425 examples=[ 2426 {"inject_into": "header", "field_name": "Authorization"}, 2427 {"inject_into": "request_parameter", "field_name": "authKey"}, 2428 ], 2429 title="Inject API Key Into Outgoing HTTP Request", 2430 ) 2431 api_token: Optional[str] = Field( 2432 "{{ session_token }}", 2433 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2434 examples=[ 2435 "{{ session_token }}", 2436 "Token {{ session_token }}", 2437 "Bearer {{ session_token }}", 2438 ], 2439 title="API Token Template", 2440 ) 2441 2442 2443class JsonSchemaPropertySelector(BaseModel): 2444 type: Literal["JsonSchemaPropertySelector"] 2445 transformations: Optional[ 2446 List[ 2447 Union[ 2448 AddFields, 2449 RemoveFields, 2450 KeysToLower, 2451 KeysToSnakeCase, 2452 FlattenFields, 2453 DpathFlattenFields, 2454 KeysReplace, 2455 CustomTransformation, 2456 ] 2457 ] 2458 ] = Field( 2459 None, 2460 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2461 title="Transformations", 2462 ) 2463 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2464 2465 2466class ListPartitionRouter(BaseModel): 2467 type: Literal["ListPartitionRouter"] 2468 cursor_field: str = Field( 2469 ..., 2470 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2471 examples=["section", "{{ config['section_key'] }}"], 2472 title="Current Partition Value Identifier", 2473 ) 2474 values: Union[str, List[str]] = Field( 2475 ..., 2476 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2477 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2478 title="Partition Values", 2479 ) 2480 request_option: Optional[RequestOption] = Field( 2481 None, 2482 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2483 title="Inject Partition Value Into Outgoing HTTP Request", 2484 ) 2485 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2486 2487 2488class PaginationReset(BaseModel): 2489 type: Literal["PaginationReset"] 2490 action: Action1 2491 limits: Optional[PaginationResetLimits] = None 2492 2493 2494class GzipDecoder(BaseModel): 2495 type: Literal["GzipDecoder"] 2496 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2497 2498 2499class RequestBodyGraphQL(BaseModel): 2500 type: Literal["RequestBodyGraphQL"] 2501 value: RequestBodyGraphQlQuery 2502 2503 2504class DpathValidator(BaseModel): 2505 type: Literal["DpathValidator"] 2506 field_path: List[str] = Field( 2507 ..., 2508 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2509 examples=[ 2510 ["data"], 2511 ["data", "records"], 2512 ["data", "{{ parameters.name }}"], 2513 ["data", "*", "record"], 2514 ], 2515 title="Field Path", 2516 ) 2517 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2518 ..., 2519 description="The condition that the specified config value will be evaluated against", 2520 title="Validation Strategy", 2521 ) 2522 2523 2524class PredicateValidator(BaseModel): 2525 type: Literal["PredicateValidator"] 2526 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2527 ..., 2528 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2529 examples=[ 2530 "test-value", 2531 "{{ config['api_version'] }}", 2532 "{{ config['tenant_id'] }}", 2533 123, 2534 ], 2535 title="Value", 2536 ) 2537 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2538 ..., 2539 description="The validation strategy to apply to the value.", 2540 title="Validation Strategy", 2541 ) 2542 2543 2544class ConfigAddFields(BaseModel): 2545 type: Literal["ConfigAddFields"] 2546 fields: List[AddedFieldDefinition] = Field( 2547 ..., 2548 description="A list of transformations (path and corresponding value) that will be added to the config.", 2549 title="Fields", 2550 ) 2551 condition: Optional[str] = Field( 2552 "", 2553 description="Fields will be added if expression is evaluated to True.", 2554 examples=[ 2555 "{{ config['environemnt'] == 'sandbox' }}", 2556 "{{ property is integer }}", 2557 "{{ property|length > 5 }}", 2558 "{{ property == 'some_string_to_match' }}", 2559 ], 2560 ) 2561 2562 2563class RateLimitedMultipleTokenAuthenticator(BaseModel): 2564 type: Literal["RateLimitedMultipleTokenAuthenticator"] 2565 tokens: Union[str, List[str]] = Field( 2566 ..., 2567 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 2568 examples=[ 2569 "{{ config['credentials']['personal_access_token'] }}", 2570 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 2571 ], 2572 title="Tokens", 2573 ) 2574 token_delimiter: Optional[str] = Field( 2575 ",", 2576 description="Delimiter used to split a single token string into multiple tokens.", 2577 title="Token Delimiter", 2578 ) 2579 auth_method: Optional[str] = Field( 2580 "Bearer", 2581 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 2582 examples=["Bearer", "token"], 2583 title="Auth Method", 2584 ) 2585 header: Optional[str] = Field( 2586 "Authorization", 2587 description="The name of the HTTP header in which to inject the token.", 2588 title="Header Name", 2589 ) 2590 quota_status_source: QuotaStatusSource = Field( 2591 ..., 2592 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 2593 title="Quota Status Source", 2594 ) 2595 quotas: List[TokenQuota] = Field( 2596 ..., 2597 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 2598 min_items=1, 2599 title="Quota Pools", 2600 ) 2601 max_wait_time: Optional[str] = Field( 2602 "PT2H", 2603 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 2604 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 2605 title="Maximum Wait Time", 2606 ) 2607 budget_reserve_fraction: Optional[float] = Field( 2608 0.1, 2609 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 2610 title="Budget Reserve Fraction", 2611 ) 2612 budget_min_reserve: Optional[int] = Field( 2613 50, 2614 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 2615 title="Budget Minimum Reserve", 2616 ) 2617 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2618 2619 2620class CompositeErrorHandler(BaseModel): 2621 type: Literal["CompositeErrorHandler"] 2622 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2623 Field( 2624 ..., 2625 description="List of error handlers to iterate on to determine how to handle a failed response.", 2626 title="Error Handlers", 2627 ) 2628 ) 2629 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2630 2631 2632class HTTPAPIBudget(BaseModel): 2633 class Config: 2634 extra = Extra.allow 2635 2636 type: Literal["HTTPAPIBudget"] 2637 policies: List[ 2638 Union[ 2639 FixedWindowCallRatePolicy, 2640 MovingWindowCallRatePolicy, 2641 UnlimitedCallRatePolicy, 2642 ] 2643 ] = Field( 2644 ..., 2645 description="List of call rate policies that define how many calls are allowed.", 2646 title="Policies", 2647 ) 2648 ratelimit_reset_header: Optional[str] = Field( 2649 "ratelimit-reset", 2650 description="The HTTP response header name that indicates when the rate limit resets.", 2651 title="Rate Limit Reset Header", 2652 ) 2653 ratelimit_remaining_header: Optional[str] = Field( 2654 "ratelimit-remaining", 2655 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2656 title="Rate Limit Remaining Header", 2657 ) 2658 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2659 [429], 2660 description="List of HTTP status codes that indicate a rate limit has been hit.", 2661 title="Status Codes for Rate Limit Hit", 2662 ) 2663 2664 2665class DpathExtractor(BaseModel): 2666 type: Literal["DpathExtractor"] 2667 field_path: List[str] = Field( 2668 ..., 2669 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2670 examples=[ 2671 ["data"], 2672 ["data", "records"], 2673 ["data", "{{ parameters.name }}"], 2674 ["data", "*", "record"], 2675 ], 2676 title="Field Path", 2677 ) 2678 record_expander: Optional[RecordExpander] = Field( 2679 None, 2680 description="Optional component to expand records by extracting items from nested array fields.", 2681 title="Record Expander", 2682 ) 2683 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2684 2685 2686class CombinedExtractor(BaseModel): 2687 type: Literal["CombinedExtractor"] 2688 extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]] = Field( 2689 ..., 2690 description="The record extractors to combine. At least one is required. Each sub-extractor is given the same HTTP response and decodes it independently, so the response is parsed once per sub-extractor. Streaming decoders (CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder, IterableDecoder) can only read the response body once and are rejected.", 2691 title="Extractors", 2692 ) 2693 mode: Optional[CombineMode] = Field( 2694 CombineMode.union, 2695 description='How the records of the sub-extractors are combined. "union" (default) yields every record of every sub-extractor, in the order the extractors are declared. "first_match" yields the records of the first sub-extractor that produces at least one record and skips the remaining ones; nothing is yielded if none of them produces a record. Note that a paginator which counts the records of a page counts the combined records: under "union" that is the sum over all sub-extractors, which overshoots the API page size, so "union" is rejected with an OffsetIncrement paginator because the offset would skip records. Under "first_match" the count is the count of the winning sub-extractor, which is usually the number of records the API returned for the page.', 2696 title="Combine Mode", 2697 ) 2698 skip_empty_records: Optional[bool] = Field( 2699 False, 2700 description='Whether to drop empty records - null, {}, [], "" - yielded by a sub-extractor before the records are combined. Off by default, so the output of a sub-extractor is passed through as it is. Turn it on when the API can return nulls in the middle of a record list, which a GraphQL API does when it answers a partial response and reports the failure in a sibling error field. Under "first_match" this also changes which sub-extractor wins: a sub-extractor whose records are all empty no longer counts as a match, so the next one is tried. The dropped records still count toward the page size of an OffsetIncrement or PageIncrement paginator, so a full page holding a null does not end pagination early.', 2701 title="Skip Empty Records", 2702 ) 2703 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2704 2705 2706class ZipfileDecoder(BaseModel): 2707 class Config: 2708 extra = Extra.allow 2709 2710 type: Literal["ZipfileDecoder"] 2711 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2712 ..., 2713 description="Parser to parse the decompressed data from the zipfile(s).", 2714 title="Parser", 2715 ) 2716 2717 2718class RecordSelector(BaseModel): 2719 type: Literal["RecordSelector"] 2720 extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 2721 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2722 None, 2723 description="Responsible for filtering records to be emitted by the Source.", 2724 title="Record Filter", 2725 ) 2726 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2727 None, 2728 description="Responsible for normalization according to the schema.", 2729 title="Schema Normalization", 2730 ) 2731 transform_before_filtering: Optional[bool] = Field( 2732 None, 2733 description="If true, transformation will be applied before record filtering.", 2734 title="Transform Before Filtering", 2735 ) 2736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2737 2738 2739class ConfigMigration(BaseModel): 2740 type: Literal["ConfigMigration"] 2741 description: Optional[str] = Field( 2742 None, description="The description/purpose of the config migration." 2743 ) 2744 transformations: List[ 2745 Union[ 2746 ConfigRemapField, 2747 ConfigAddFields, 2748 ConfigRemoveFields, 2749 CustomConfigTransformation, 2750 ] 2751 ] = Field( 2752 ..., 2753 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2754 title="Transformations", 2755 ) 2756 2757 2758class ConfigNormalizationRules(BaseModel): 2759 class Config: 2760 extra = Extra.forbid 2761 2762 type: Literal["ConfigNormalizationRules"] 2763 config_migrations: Optional[List[ConfigMigration]] = Field( 2764 [], 2765 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2766 title="Config Migrations", 2767 ) 2768 transformations: Optional[ 2769 List[ 2770 Union[ 2771 ConfigRemapField, 2772 ConfigAddFields, 2773 ConfigRemoveFields, 2774 CustomConfigTransformation, 2775 ] 2776 ] 2777 ] = Field( 2778 [], 2779 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2780 title="Transformations", 2781 ) 2782 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2783 [], 2784 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2785 title="Validations", 2786 ) 2787 2788 2789class Spec(BaseModel): 2790 type: Literal["Spec"] 2791 connection_specification: Dict[str, Any] = Field( 2792 ..., 2793 description="A connection specification describing how a the connector can be configured.", 2794 title="Connection Specification", 2795 ) 2796 documentation_url: Optional[str] = Field( 2797 None, 2798 description="URL of the connector's documentation page.", 2799 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2800 title="Documentation URL", 2801 ) 2802 advanced_auth: Optional[AuthFlow] = Field( 2803 None, 2804 description="Advanced specification for configuring the authentication flow.", 2805 title="Advanced Auth", 2806 ) 2807 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2808 None, title="Config Normalization Rules" 2809 ) 2810 2811 2812class DeclarativeSource1(BaseModel): 2813 class Config: 2814 extra = Extra.forbid 2815 2816 type: Literal["DeclarativeSource"] 2817 check: Union[CheckStream, CheckDynamicStream] 2818 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2819 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2820 version: str = Field( 2821 ..., 2822 description="The version of the Airbyte CDK used to build and test the source.", 2823 ) 2824 schemas: Optional[Schemas] = None 2825 definitions: Optional[Dict[str, Any]] = None 2826 spec: Optional[Spec] = None 2827 concurrency_level: Optional[ConcurrencyLevel] = None 2828 api_budget: Optional[HTTPAPIBudget] = None 2829 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2830 None, 2831 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2832 title="Stream Groups", 2833 ) 2834 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2835 None, 2836 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2837 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2838 title="Maximum Concurrent Asynchronous Jobs", 2839 ) 2840 metadata: Optional[Dict[str, Any]] = Field( 2841 None, 2842 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2843 ) 2844 description: Optional[str] = Field( 2845 None, 2846 description="A description of the connector. It will be presented on the Source documentation page.", 2847 ) 2848 2849 2850class DeclarativeSource2(BaseModel): 2851 class Config: 2852 extra = Extra.forbid 2853 2854 type: Literal["DeclarativeSource"] 2855 check: Union[CheckStream, CheckDynamicStream] 2856 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2857 None 2858 ) 2859 dynamic_streams: List[DynamicDeclarativeStream] 2860 version: str = Field( 2861 ..., 2862 description="The version of the Airbyte CDK used to build and test the source.", 2863 ) 2864 schemas: Optional[Schemas] = None 2865 definitions: Optional[Dict[str, Any]] = None 2866 spec: Optional[Spec] = None 2867 concurrency_level: Optional[ConcurrencyLevel] = None 2868 api_budget: Optional[HTTPAPIBudget] = None 2869 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2870 None, 2871 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2872 title="Stream Groups", 2873 ) 2874 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2875 None, 2876 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2877 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2878 title="Maximum Concurrent Asynchronous Jobs", 2879 ) 2880 metadata: Optional[Dict[str, Any]] = Field( 2881 None, 2882 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2883 ) 2884 description: Optional[str] = Field( 2885 None, 2886 description="A description of the connector. It will be presented on the Source documentation page.", 2887 ) 2888 2889 2890class DeclarativeSource(BaseModel): 2891 class Config: 2892 extra = Extra.forbid 2893 2894 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2895 ..., 2896 description="An API source that extracts data according to its declarative components.", 2897 title="DeclarativeSource", 2898 ) 2899 2900 2901class SelectiveAuthenticator(BaseModel): 2902 class Config: 2903 extra = Extra.allow 2904 2905 type: Literal["SelectiveAuthenticator"] 2906 authenticator_selection_path: List[str] = Field( 2907 ..., 2908 description="Path of the field in config with selected authenticator name", 2909 examples=[["auth"], ["auth", "type"]], 2910 title="Authenticator Selection Path", 2911 ) 2912 authenticators: Dict[ 2913 str, 2914 Union[ 2915 ApiKeyAuthenticator, 2916 BasicHttpAuthenticator, 2917 BearerAuthenticator, 2918 OAuthAuthenticator, 2919 JwtAuthenticator, 2920 SessionTokenAuthenticator, 2921 LegacySessionTokenAuthenticator, 2922 CustomAuthenticator, 2923 NoAuth, 2924 RateLimitedMultipleTokenAuthenticator, 2925 ], 2926 ] = Field( 2927 ..., 2928 description="Authenticators to select from.", 2929 examples=[ 2930 { 2931 "authenticators": { 2932 "token": "#/definitions/ApiKeyAuthenticator", 2933 "oauth": "#/definitions/OAuthAuthenticator", 2934 "jwt": "#/definitions/JwtAuthenticator", 2935 } 2936 } 2937 ], 2938 title="Authenticators", 2939 ) 2940 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2941 2942 2943class ConditionalStreams(BaseModel): 2944 type: Literal["ConditionalStreams"] 2945 condition: str = Field( 2946 ..., 2947 description="Condition that will be evaluated to determine if a set of streams should be available.", 2948 examples=["{{ config['is_sandbox'] }}"], 2949 title="Condition", 2950 ) 2951 streams: List[DeclarativeStream] = Field( 2952 ..., 2953 description="Streams that will be used during an operation based on the condition.", 2954 title="Streams", 2955 ) 2956 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2957 2958 2959class FileUploader(BaseModel): 2960 type: Literal["FileUploader"] 2961 requester: Union[HttpRequester, CustomRequester] = Field( 2962 ..., 2963 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2964 ) 2965 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = ( 2966 Field( 2967 ..., 2968 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2969 ) 2970 ) 2971 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2972 None, 2973 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2974 ) 2975 filename_extractor: Optional[str] = Field( 2976 None, 2977 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2978 examples=[ 2979 "{{ record.id }}/{{ record.file_name }}/", 2980 "{{ record.id }}_{{ record.file_name }}/", 2981 ], 2982 ) 2983 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2984 2985 2986class SyncTokenCursor(BaseModel): 2987 class Config: 2988 extra = Extra.allow 2989 2990 type: Literal["SyncTokenCursor"] 2991 token_path: List[str] = Field( 2992 ..., 2993 description="Path to the next token in the decoded response that ends a pagination pass. The value must be a string or an integer, and is stored and sent as a string. A missing, null or empty value means the response carried no token.", 2994 examples=[["syncToken"], ["nextSyncToken"], ["meta", "sync_token"]], 2995 title="Token Path", 2996 ) 2997 token_option: RequestOption = Field( 2998 ..., 2999 description="Where to send the stored token on each request of a pass: the JSON body, the query string, a header or the form body.", 3000 title="Send Token In", 3001 ) 3002 completion_condition: Optional[str] = Field( 3003 None, 3004 description="Must be true on the last response of a pass for its token to be committed. Defaults to a token being present at token_path. Set it when the API repeats the old token on intermediate pages or returns a position on every page.", 3005 examples=["{{ not response.moreDataAvailable }}"], 3006 title="Completion Condition", 3007 ) 3008 initial_token: Optional[str] = Field( 3009 None, 3010 description='Value to send when no token is stored, for an API that needs one to start a full pass, such as Todoist\'s "*". When unset, nothing is sent.', 3011 examples=["*"], 3012 title="Initial Token", 3013 ) 3014 cursor_field: Optional[str] = Field( 3015 None, 3016 description="Optional top-level record field advertised as the stream's cursor, so destinations can order versions of a record when deduplicating. It is never used to filter records or to build state.", 3017 title="Destination Cursor Field", 3018 ) 3019 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3020 3021 3022class DeclarativeStream(BaseModel): 3023 class Config: 3024 extra = Extra.allow 3025 3026 type: Literal["DeclarativeStream"] 3027 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 3028 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3029 ..., 3030 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3031 title="Retriever", 3032 ) 3033 incremental_sync: Optional[ 3034 Union[DatetimeBasedCursor, IncrementingCountCursor, SyncTokenCursor] 3035 ] = Field( 3036 None, 3037 description="Component used to fetch data incrementally based on a time field in the data.", 3038 title="Incremental Sync", 3039 ) 3040 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 3041 schema_loader: Optional[ 3042 Union[ 3043 InlineSchemaLoader, 3044 DynamicSchemaLoader, 3045 JsonFileSchemaLoader, 3046 List[ 3047 Union[ 3048 InlineSchemaLoader, 3049 DynamicSchemaLoader, 3050 JsonFileSchemaLoader, 3051 CustomSchemaLoader, 3052 ] 3053 ], 3054 CustomSchemaLoader, 3055 ] 3056 ] = Field( 3057 None, 3058 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 3059 title="Schema Loader", 3060 ) 3061 transformations: Optional[ 3062 List[ 3063 Union[ 3064 AddFields, 3065 RemoveFields, 3066 KeysToLower, 3067 KeysToSnakeCase, 3068 FlattenFields, 3069 DpathFlattenFields, 3070 KeysReplace, 3071 CustomTransformation, 3072 ] 3073 ] 3074 ] = Field( 3075 None, 3076 description="A list of transformations to be applied to each output record.", 3077 title="Transformations", 3078 ) 3079 state_migrations: Optional[ 3080 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 3081 ] = Field( 3082 [], 3083 description="Array of state migrations to be applied on the input state", 3084 title="State Migrations", 3085 ) 3086 file_uploader: Optional[FileUploader] = Field( 3087 None, 3088 description="(experimental) Describes how to fetch a file", 3089 title="File Uploader", 3090 ) 3091 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3092 3093 3094class SessionTokenAuthenticator(BaseModel): 3095 type: Literal["SessionTokenAuthenticator"] 3096 login_requester: HttpRequester = Field( 3097 ..., 3098 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 3099 examples=[ 3100 { 3101 "type": "HttpRequester", 3102 "url_base": "https://my_api.com", 3103 "path": "/login", 3104 "authenticator": { 3105 "type": "BasicHttpAuthenticator", 3106 "username": "{{ config.username }}", 3107 "password": "{{ config.password }}", 3108 }, 3109 } 3110 ], 3111 title="Login Requester", 3112 ) 3113 session_token_path: List[str] = Field( 3114 ..., 3115 description="The path in the response body returned from the login requester to the session token.", 3116 examples=[["access_token"], ["result", "token"]], 3117 title="Session Token Path", 3118 ) 3119 expiration_duration: Optional[str] = Field( 3120 None, 3121 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 3122 examples=["PT1H", "P1D"], 3123 title="Expiration Duration", 3124 ) 3125 request_authentication: Union[ 3126 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 3127 ] = Field( 3128 ..., 3129 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 3130 title="Data Request Authentication", 3131 ) 3132 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 3133 None, description="Component used to decode the response.", title="Decoder" 3134 ) 3135 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3136 3137 3138class HttpRequester(BaseModelWithDeprecations): 3139 type: Literal["HttpRequester"] 3140 url_base: Optional[str] = Field( 3141 None, 3142 deprecated=True, 3143 deprecation_message="Use `url` field instead.", 3144 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3145 examples=[ 3146 "https://connect.squareup.com/v2", 3147 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 3148 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3149 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3150 ], 3151 title="API Base URL", 3152 ) 3153 url: Optional[str] = Field( 3154 None, 3155 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3156 examples=[ 3157 "https://connect.squareup.com/v2", 3158 "{{ config['url'] or 'https://app.posthog.com'}}/api", 3159 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3160 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3161 ], 3162 title="API Endpoint URL", 3163 ) 3164 path: Optional[str] = Field( 3165 None, 3166 deprecated=True, 3167 deprecation_message="Use `url` field instead.", 3168 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3169 examples=[ 3170 "/products", 3171 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3172 "/trades/{{ config['symbol_id'] }}/history", 3173 ], 3174 title="URL Path", 3175 ) 3176 http_method: Optional[HttpMethod] = Field( 3177 HttpMethod.GET, 3178 description="The HTTP method used to fetch data from the source (can be GET or POST).", 3179 examples=["GET", "POST"], 3180 title="HTTP Method", 3181 ) 3182 authenticator: Optional[ 3183 Union[ 3184 ApiKeyAuthenticator, 3185 BasicHttpAuthenticator, 3186 BearerAuthenticator, 3187 OAuthAuthenticator, 3188 JwtAuthenticator, 3189 SessionTokenAuthenticator, 3190 SelectiveAuthenticator, 3191 CustomAuthenticator, 3192 NoAuth, 3193 LegacySessionTokenAuthenticator, 3194 RateLimitedMultipleTokenAuthenticator, 3195 ] 3196 ] = Field( 3197 None, 3198 description="Authentication method to use for requests sent to the API.", 3199 title="Authenticator", 3200 ) 3201 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 3202 None, 3203 deprecated=True, 3204 deprecation_message="Use `query_properties` field instead.", 3205 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 3206 title="Fetch Properties from Endpoint", 3207 ) 3208 query_properties: Optional[QueryProperties] = Field( 3209 None, 3210 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 3211 title="Query Properties", 3212 ) 3213 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 3214 None, 3215 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3216 examples=[ 3217 {"unit": "day"}, 3218 { 3219 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3220 }, 3221 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3222 {"sort_by[asc]": "updated_at"}, 3223 ], 3224 title="Query Parameters", 3225 ) 3226 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3227 None, 3228 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3229 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3230 title="Request Headers", 3231 ) 3232 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3233 None, 3234 deprecated=True, 3235 deprecation_message="Use `request_body` field instead.", 3236 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3237 examples=[ 3238 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3239 ], 3240 title="Request Body Payload (Non-JSON)", 3241 ) 3242 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3243 None, 3244 deprecated=True, 3245 deprecation_message="Use `request_body` field instead.", 3246 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3247 examples=[ 3248 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3249 {"key": "{{ config['value'] }}"}, 3250 {"sort": {"field": "updated_at", "order": "ascending"}}, 3251 ], 3252 title="Request Body JSON Payload", 3253 ) 3254 request_body: Optional[ 3255 Union[ 3256 RequestBodyPlainText, 3257 RequestBodyUrlEncodedForm, 3258 RequestBodyJsonObject, 3259 RequestBodyGraphQL, 3260 ] 3261 ] = Field( 3262 None, 3263 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3264 title="Request Body", 3265 ) 3266 error_handler: Optional[ 3267 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3268 ] = Field( 3269 None, 3270 description="Error handler component that defines how to handle errors.", 3271 title="Error Handler", 3272 ) 3273 use_cache: Optional[bool] = Field( 3274 False, 3275 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3276 title="Use Cache", 3277 ) 3278 connect_timeout_in_seconds: Optional[float] = Field( 3279 None, 3280 description="Maximum time to establish the connection (TCP + TLS handshake). When unset, no connect timeout is applied.", 3281 examples=[30], 3282 gt=0, 3283 title="Connect Timeout (seconds)", 3284 ) 3285 read_timeout_in_seconds: Optional[float] = Field( 3286 None, 3287 description="The maximum time to wait for the server to send any data (the first byte of the response, or the next chunk while the body is read). It is NOT a limit on the total request duration. When unset, no read timeout is applied. Only set it when every endpoint served by this requester responds well within the value; synchronous report or export endpoints can stay silent for minutes. Each timed-out attempt counts against the error handler's retry window (600 s for manifest connectors), so a value near 300 allows only a few retries.", 3288 examples=[300], 3289 gt=0, 3290 title="Read Timeout (seconds)", 3291 ) 3292 use_tcp_keepalive: Optional[bool] = Field( 3293 False, 3294 description="Enables TCP keepalive (60 s idle, 10 s probe interval, 6 probes) on this requester's connections, so dead or NAT-dropped connections are detected and fail instead of hanging forever. Does not fail slow but alive servers; use read_timeout_in_seconds to bound those.", 3295 examples=[True], 3296 title="Use TCP Keepalive", 3297 ) 3298 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3299 3300 3301class DynamicSchemaLoader(BaseModel): 3302 type: Literal["DynamicSchemaLoader"] 3303 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3304 ..., 3305 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3306 title="Retriever", 3307 ) 3308 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3309 None, 3310 description="Responsible for filtering fields to be added to json schema.", 3311 title="Schema Filter", 3312 ) 3313 schema_transformations: Optional[ 3314 List[ 3315 Union[ 3316 AddFields, 3317 RemoveFields, 3318 KeysToLower, 3319 KeysToSnakeCase, 3320 FlattenFields, 3321 DpathFlattenFields, 3322 KeysReplace, 3323 CustomTransformation, 3324 ] 3325 ] 3326 ] = Field( 3327 None, 3328 description="A list of transformations to be applied to the schema.", 3329 title="Schema Transformations", 3330 ) 3331 schema_type_identifier: SchemaTypeIdentifier 3332 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3333 3334 3335class ParentStreamConfig(BaseModel): 3336 type: Literal["ParentStreamConfig"] 3337 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3338 ..., description="Reference to the parent stream.", title="Parent Stream" 3339 ) 3340 parent_key: str = Field( 3341 ..., 3342 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3343 examples=["id", "{{ config['parent_record_id'] }}"], 3344 title="Parent Key", 3345 ) 3346 partition_field: str = Field( 3347 ..., 3348 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3349 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3350 title="Current Parent Key Value Identifier", 3351 ) 3352 request_option: Optional[RequestOption] = Field( 3353 None, 3354 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3355 title="Request Option", 3356 ) 3357 incremental_dependency: Optional[bool] = Field( 3358 False, 3359 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3360 title="Incremental Dependency", 3361 ) 3362 lazy_read_pointer: Optional[List[str]] = Field( 3363 [], 3364 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3365 title="Lazy Read Pointer", 3366 ) 3367 extra_fields: Optional[List[List[str]]] = Field( 3368 None, 3369 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3370 title="Extra Fields", 3371 ) 3372 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3373 3374 3375class PropertiesFromEndpoint(BaseModel): 3376 type: Literal["PropertiesFromEndpoint"] 3377 property_field_path: List[str] = Field( 3378 ..., 3379 description="Describes the path to the field that should be extracted", 3380 examples=[["name"]], 3381 ) 3382 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3383 ..., 3384 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3385 ) 3386 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3387 3388 3389class QueryProperties(BaseModel): 3390 type: Literal["QueryProperties"] 3391 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3392 ..., 3393 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3394 title="Property List", 3395 ) 3396 always_include_properties: Optional[List[str]] = Field( 3397 None, 3398 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3399 title="Always Include Properties", 3400 ) 3401 property_chunking: Optional[PropertyChunking] = Field( 3402 None, 3403 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3404 title="Property Chunking", 3405 ) 3406 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3407 None, 3408 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3409 title="Property Selector", 3410 ) 3411 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3412 3413 3414class StateDelegatingStream(BaseModel): 3415 type: Literal["StateDelegatingStream"] 3416 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3417 full_refresh_stream: DeclarativeStream = Field( 3418 ..., 3419 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3420 title="Full Refresh Stream", 3421 ) 3422 incremental_stream: DeclarativeStream = Field( 3423 ..., 3424 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3425 title="Incremental Stream", 3426 ) 3427 api_retention_period: Optional[str] = Field( 3428 None, 3429 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", 3430 examples=["P30D", "P90D", "P1Y"], 3431 title="API Retention Period", 3432 ) 3433 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3434 3435 3436class SimpleRetriever(BaseModel): 3437 type: Literal["SimpleRetriever"] 3438 requester: Union[HttpRequester, CustomRequester] = Field( 3439 ..., 3440 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3441 ) 3442 decoder: Optional[ 3443 Union[ 3444 JsonDecoder, 3445 JsonItemsDecoder, 3446 XmlDecoder, 3447 CsvDecoder, 3448 JsonlDecoder, 3449 GzipDecoder, 3450 IterableDecoder, 3451 ZipfileDecoder, 3452 CustomDecoder, 3453 ] 3454 ] = Field( 3455 None, 3456 description="Component decoding the response so records can be extracted.", 3457 title="HTTP Response Format", 3458 ) 3459 record_selector: RecordSelector = Field( 3460 ..., 3461 description="Component that describes how to extract records from a HTTP response.", 3462 ) 3463 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3464 None, 3465 description="Paginator component that describes how to navigate through the API's pages.", 3466 ) 3467 pagination_reset: Optional[PaginationReset] = Field( 3468 None, 3469 description="Describes what triggers pagination reset and how to handle it.", 3470 ) 3471 page_size_reduction: Optional[PageSizeReduction] = Field( 3472 None, 3473 description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", 3474 ) 3475 request_window_splitting: Optional[RequestWindowSplitting] = Field( 3476 None, 3477 description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page size, and applies even when the API rejects the request before returning a single page.", 3478 ) 3479 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3480 False, 3481 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3482 ) 3483 partition_router: Optional[ 3484 Union[ 3485 SubstreamPartitionRouter, 3486 ListPartitionRouter, 3487 GroupingPartitionRouter, 3488 UnionPartitionRouter, 3489 CustomPartitionRouter, 3490 List[ 3491 Union[ 3492 SubstreamPartitionRouter, 3493 ListPartitionRouter, 3494 GroupingPartitionRouter, 3495 UnionPartitionRouter, 3496 CustomPartitionRouter, 3497 ] 3498 ], 3499 ] 3500 ] = Field( 3501 None, 3502 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3503 title="Partition Router", 3504 ) 3505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3506 3507 3508class AsyncRetriever(BaseModel): 3509 type: Literal["AsyncRetriever"] 3510 record_selector: RecordSelector = Field( 3511 ..., 3512 description="Component that describes how to extract records from a HTTP response.", 3513 ) 3514 status_mapping: AsyncJobStatusMap = Field( 3515 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3516 ) 3517 status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = Field( 3518 ..., description="Responsible for fetching the actual status of the async job." 3519 ) 3520 download_target_extractor: Optional[ 3521 Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 3522 ] = Field( 3523 None, 3524 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3525 ) 3526 download_extractor: Optional[ 3527 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor] 3528 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3529 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3530 ..., 3531 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3532 ) 3533 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3534 ..., 3535 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3536 ) 3537 polling_job_timeout: Optional[Union[int, str]] = Field( 3538 None, 3539 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3540 ) 3541 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3542 None, 3543 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3544 ge=1, 3545 ) 3546 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3547 None, 3548 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3549 ) 3550 download_requester: Union[HttpRequester, CustomRequester] = Field( 3551 ..., 3552 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3553 ) 3554 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3555 None, 3556 description="Paginator component that describes how to navigate through the API's pages during download.", 3557 ) 3558 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3559 None, 3560 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3561 ) 3562 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3563 None, 3564 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3565 ) 3566 partition_router: Optional[ 3567 Union[ 3568 ListPartitionRouter, 3569 SubstreamPartitionRouter, 3570 GroupingPartitionRouter, 3571 UnionPartitionRouter, 3572 CustomPartitionRouter, 3573 List[ 3574 Union[ 3575 ListPartitionRouter, 3576 SubstreamPartitionRouter, 3577 GroupingPartitionRouter, 3578 UnionPartitionRouter, 3579 CustomPartitionRouter, 3580 ] 3581 ], 3582 ] 3583 ] = Field( 3584 [], 3585 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3586 title="Partition Router", 3587 ) 3588 decoder: Optional[ 3589 Union[ 3590 CsvDecoder, 3591 GzipDecoder, 3592 JsonDecoder, 3593 JsonItemsDecoder, 3594 JsonlDecoder, 3595 IterableDecoder, 3596 XmlDecoder, 3597 ZipfileDecoder, 3598 CustomDecoder, 3599 ] 3600 ] = Field( 3601 None, 3602 description="Component decoding the response so records can be extracted.", 3603 title="HTTP Response Format", 3604 ) 3605 download_decoder: Optional[ 3606 Union[ 3607 CsvDecoder, 3608 GzipDecoder, 3609 JsonDecoder, 3610 JsonItemsDecoder, 3611 JsonlDecoder, 3612 IterableDecoder, 3613 XmlDecoder, 3614 ZipfileDecoder, 3615 CustomDecoder, 3616 ] 3617 ] = Field( 3618 None, 3619 description="Component decoding the download response so records can be extracted.", 3620 title="Download HTTP Response Format", 3621 ) 3622 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3623 3624 3625class StreamGroup(BaseModel): 3626 streams: List[DeclarativeStream] = Field( 3627 ..., 3628 description="List of references to streams that belong to this group.\n", 3629 title="Streams", 3630 ) 3631 action: BlockSimultaneousSyncsAction = Field( 3632 ..., description="The action to apply to streams in this group.", title="Action" 3633 ) 3634 3635 3636class SubstreamPartitionRouter(BaseModel): 3637 type: Literal["SubstreamPartitionRouter"] 3638 parent_stream_configs: List[ParentStreamConfig] = Field( 3639 ..., 3640 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3641 title="Parent Stream Configs", 3642 ) 3643 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3644 3645 3646class GroupingPartitionRouter(BaseModel): 3647 type: Literal["GroupingPartitionRouter"] 3648 group_size: int = Field( 3649 ..., 3650 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3651 examples=[10, 50], 3652 title="Group Size", 3653 ) 3654 underlying_partition_router: Union[ 3655 ListPartitionRouter, 3656 SubstreamPartitionRouter, 3657 UnionPartitionRouter, 3658 CustomPartitionRouter, 3659 ] = Field( 3660 ..., 3661 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3662 title="Underlying Partition Router", 3663 ) 3664 deduplicate: Optional[bool] = Field( 3665 True, 3666 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3667 title="Deduplicate Partitions", 3668 ) 3669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3670 3671 3672class UnionPartitionRouter(BaseModel): 3673 type: Literal["UnionPartitionRouter"] 3674 partition_field: str = Field( 3675 ..., 3676 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3677 examples=["repository", "{{ config['partition_field'] }}"], 3678 title="Partition Field", 3679 ) 3680 partition_routers: List[ 3681 Union[ 3682 ListPartitionRouter, 3683 SubstreamPartitionRouter, 3684 UnionPartitionRouter, 3685 CustomPartitionRouter, 3686 ] 3687 ] = Field( 3688 ..., 3689 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3690 title="Partition Routers", 3691 ) 3692 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3693 3694 3695class HttpComponentsResolver(BaseModel): 3696 type: Literal["HttpComponentsResolver"] 3697 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3698 ..., 3699 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3700 title="Retriever", 3701 ) 3702 components_mapping: List[ComponentMappingDefinition] 3703 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3704 3705 3706class DynamicDeclarativeStream(BaseModel): 3707 type: Literal["DynamicDeclarativeStream"] 3708 name: Optional[str] = Field( 3709 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3710 ) 3711 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3712 ..., description="Reference to the stream template.", title="Stream Template" 3713 ) 3714 components_resolver: Union[ 3715 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3716 ] = Field( 3717 ..., 3718 description="Component resolve and populates stream templates with components values.", 3719 title="Components Resolver", 3720 ) 3721 use_parent_parameters: Optional[bool] = Field( 3722 True, 3723 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3724 title="Use Parent Parameters", 3725 ) 3726 3727 3728ComplexFieldType.update_forward_refs() 3729GzipDecoder.update_forward_refs() 3730CompositeErrorHandler.update_forward_refs() 3731CombinedExtractor.update_forward_refs() 3732DeclarativeSource1.update_forward_refs() 3733DeclarativeSource2.update_forward_refs() 3734SelectiveAuthenticator.update_forward_refs() 3735ConditionalStreams.update_forward_refs() 3736FileUploader.update_forward_refs() 3737DeclarativeStream.update_forward_refs() 3738DpathExtractor.update_forward_refs() 3739RecordExpander.update_forward_refs() 3740SessionTokenAuthenticator.update_forward_refs() 3741HttpRequester.update_forward_refs() 3742DynamicSchemaLoader.update_forward_refs() 3743ParentStreamConfig.update_forward_refs() 3744PropertiesFromEndpoint.update_forward_refs() 3745SimpleRetriever.update_forward_refs() 3746AsyncRetriever.update_forward_refs() 3747GroupingPartitionRouter.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
22class BasicHttpAuthenticator(BaseModel): 23 type: Literal["BasicHttpAuthenticator"] 24 username: str = Field( 25 ..., 26 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 27 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 28 title="Username", 29 ) 30 password: Optional[str] = Field( 31 "", 32 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['password'] }}", ""], 34 title="Password", 35 ) 36 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
39class BearerAuthenticator(BaseModel): 40 type: Literal["BearerAuthenticator"] 41 api_token: str = Field( 42 ..., 43 description="Token to inject as request header for authenticating with the API.", 44 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 45 title="Bearer Token", 46 ) 47 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
QuotaStatusSource(pydantic.v1.main.BaseModel):
55class QuotaStatusSource(BaseModel): 56 type: Literal["QuotaStatusSource"] 57 url: str = Field( 58 ..., 59 description="The full URL of the quota status endpoint.", 60 examples=[ 61 "https://api.github.com/rate_limit", 62 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 63 ], 64 title="URL", 65 ) 66 http_method: Optional[HttpMethod] = Field( 67 HttpMethod.GET, 68 description="The HTTP method used to fetch the quota status.", 69 title="HTTP Method", 70 ) 71 request_headers: Optional[Dict[str, str]] = Field( 72 None, 73 description="Additional headers to send with the quota status request.", 74 title="Request Headers", 75 ) 76 unavailable_status_codes: Optional[List[int]] = Field( 77 None, 78 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 79 examples=[[404]], 80 title="Unavailable Status Codes", 81 unique_items=True, 82 ) 83 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
http_method: Optional[HttpMethod]
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
86class DynamicStreamCheckConfig(BaseModel): 87 type: Literal["DynamicStreamCheckConfig"] 88 dynamic_stream_name: str = Field( 89 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 90 ) 91 stream_count: Optional[int] = Field( 92 None, 93 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 94 ge=1, 95 title="Stream Count", 96 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
99class CheckDynamicStream(BaseModel): 100 type: Literal["CheckDynamicStream"] 101 stream_count: int = Field( 102 ..., 103 description="Numbers of the streams to try reading from when running a check operation.", 104 title="Stream Count", 105 ) 106 use_check_availability: Optional[bool] = Field( 107 True, 108 description="Enables stream check availability. This field is automatically set by the CDK.", 109 title="Use Check Availability", 110 ) 111 config_overrides: Optional[Dict[str, Any]] = Field( 112 None, 113 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 114 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 115 title="Config Overrides", 116 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
119class ConcurrencyLevel(BaseModel): 120 type: Optional[Literal["ConcurrencyLevel"]] = None 121 default_concurrency: Union[int, str] = Field( 122 ..., 123 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 124 examples=[10, "{{ config['num_workers'] or 10 }}"], 125 title="Default Concurrency", 126 ) 127 max_concurrency: Optional[int] = Field( 128 None, 129 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 130 examples=[20, 100], 131 title="Max Concurrency", 132 ) 133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
136class ConstantBackoffStrategy(BaseModel): 137 type: Literal["ConstantBackoffStrategy"] 138 backoff_time_in_seconds: Union[float, str] = Field( 139 ..., 140 description="Backoff time in seconds.", 141 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 142 title="Backoff Time", 143 ) 144 jitter_range_in_seconds: Optional[float] = Field( 145 None, 146 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 147 examples=[15], 148 ge=0, 149 title="Jitter Range", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
154class CursorPagination(BaseModel): 155 type: Literal["CursorPagination"] 156 cursor_value: str = Field( 157 ..., 158 description="Value of the cursor defining the next page to fetch.", 159 examples=[ 160 "{{ headers.link.next.cursor }}", 161 "{{ last_record['key'] }}", 162 "{{ response['nextPage'] }}", 163 ], 164 title="Cursor Value", 165 ) 166 page_size: Optional[Union[int, str]] = Field( 167 None, 168 description="The number of records to include in each pages.", 169 examples=[100, "{{ config['page_size'] }}"], 170 title="Page Size", 171 ) 172 stop_condition: Optional[str] = Field( 173 None, 174 description="Template string evaluating when to stop paginating. Compare last_page_size against page_size rather than against a hardcoded number: page_size is the page size that was actually requested, so the condition stays correct when page_size_reduction shrinks it. It is only bound when this strategy declares page_size, and a condition that reads it without one is rejected, so add page_size alongside the condition. Testing the page for emptiness with last_page_size == 0 is equally safe and needs no page_size. A stream that enables page_size_reduction is rejected when its stop condition compares last_page_size against anything else, since a full page at a reduced size would then read as a short page.", 175 examples=[ 176 "{{ response.data.has_more is false }}", 177 "{{ 'next' not in headers['link'] }}", 178 "{{ last_page_size < page_size }}", 179 ], 180 title="Stop Condition", 181 ) 182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
185class CustomAuthenticator(BaseModel): 186 class Config: 187 extra = Extra.allow 188 189 type: Literal["CustomAuthenticator"] 190 class_name: str = Field( 191 ..., 192 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 193 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 194 title="Class Name", 195 ) 196 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
199class CustomBackoffStrategy(BaseModel): 200 class Config: 201 extra = Extra.allow 202 203 type: Literal["CustomBackoffStrategy"] 204 class_name: str = Field( 205 ..., 206 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 207 examples=["source_railz.components.MyCustomBackoffStrategy"], 208 title="Class Name", 209 ) 210 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
213class CustomErrorHandler(BaseModel): 214 class Config: 215 extra = Extra.allow 216 217 type: Literal["CustomErrorHandler"] 218 class_name: str = Field( 219 ..., 220 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 221 examples=["source_railz.components.MyCustomErrorHandler"], 222 title="Class Name", 223 ) 224 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
227class CustomPaginationStrategy(BaseModel): 228 class Config: 229 extra = Extra.allow 230 231 type: Literal["CustomPaginationStrategy"] 232 class_name: str = Field( 233 ..., 234 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 235 examples=["source_railz.components.MyCustomPaginationStrategy"], 236 title="Class Name", 237 ) 238 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
241class CustomRecordExtractor(BaseModel): 242 class Config: 243 extra = Extra.allow 244 245 type: Literal["CustomRecordExtractor"] 246 class_name: str = Field( 247 ..., 248 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 249 examples=["source_railz.components.MyCustomRecordExtractor"], 250 title="Class Name", 251 ) 252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
255class CustomRecordFilter(BaseModel): 256 class Config: 257 extra = Extra.allow 258 259 type: Literal["CustomRecordFilter"] 260 class_name: str = Field( 261 ..., 262 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 263 examples=["source_railz.components.MyCustomCustomRecordFilter"], 264 title="Class Name", 265 ) 266 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
269class CustomRequester(BaseModel): 270 class Config: 271 extra = Extra.allow 272 273 type: Literal["CustomRequester"] 274 class_name: str = Field( 275 ..., 276 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 277 examples=["source_railz.components.MyCustomRecordExtractor"], 278 title="Class Name", 279 ) 280 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
283class CustomRetriever(BaseModel): 284 class Config: 285 extra = Extra.allow 286 287 type: Literal["CustomRetriever"] 288 class_name: str = Field( 289 ..., 290 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 291 examples=["source_railz.components.MyCustomRetriever"], 292 title="Class Name", 293 ) 294 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
297class CustomPartitionRouter(BaseModel): 298 class Config: 299 extra = Extra.allow 300 301 type: Literal["CustomPartitionRouter"] 302 class_name: str = Field( 303 ..., 304 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 305 examples=["source_railz.components.MyCustomPartitionRouter"], 306 title="Class Name", 307 ) 308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
311class CustomSchemaLoader(BaseModel): 312 class Config: 313 extra = Extra.allow 314 315 type: Literal["CustomSchemaLoader"] 316 class_name: str = Field( 317 ..., 318 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 319 examples=["source_railz.components.MyCustomSchemaLoader"], 320 title="Class Name", 321 ) 322 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
325class CustomSchemaNormalization(BaseModel): 326 class Config: 327 extra = Extra.allow 328 329 type: Literal["CustomSchemaNormalization"] 330 class_name: str = Field( 331 ..., 332 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 333 examples=[ 334 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 335 ], 336 title="Class Name", 337 ) 338 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
341class CustomStateMigration(BaseModel): 342 class Config: 343 extra = Extra.allow 344 345 type: Literal["CustomStateMigration"] 346 class_name: str = Field( 347 ..., 348 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 349 examples=["source_railz.components.MyCustomStateMigration"], 350 title="Class Name", 351 ) 352 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
355class CustomTransformation(BaseModel): 356 class Config: 357 extra = Extra.allow 358 359 type: Literal["CustomTransformation"] 360 class_name: str = Field( 361 ..., 362 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 363 examples=["source_railz.components.MyCustomTransformation"], 364 title="Class Name", 365 ) 366 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
386class Algorithm(Enum): 387 HS256 = "HS256" 388 HS384 = "HS384" 389 HS512 = "HS512" 390 ES256 = "ES256" 391 ES256K = "ES256K" 392 ES384 = "ES384" 393 ES512 = "ES512" 394 RS256 = "RS256" 395 RS384 = "RS384" 396 RS512 = "RS512" 397 PS256 = "PS256" 398 PS384 = "PS384" 399 PS512 = "PS512" 400 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
403class JwtHeaders(BaseModel): 404 class Config: 405 extra = Extra.forbid 406 407 kid: Optional[str] = Field( 408 None, 409 description="Private key ID for user account.", 410 examples=["{{ config['kid'] }}"], 411 title="Key Identifier", 412 ) 413 typ: Optional[str] = Field( 414 "JWT", 415 description="The media type of the complete JWT.", 416 examples=["JWT"], 417 title="Type", 418 ) 419 cty: Optional[str] = Field( 420 None, 421 description="Content type of JWT header.", 422 examples=["JWT"], 423 title="Content Type", 424 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
427class JwtPayload(BaseModel): 428 class Config: 429 extra = Extra.forbid 430 431 iss: Optional[str] = Field( 432 None, 433 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 434 examples=["{{ config['iss'] }}"], 435 title="Issuer", 436 ) 437 sub: Optional[str] = Field( 438 None, 439 description="The subject of the JWT. Commonly defined by the API.", 440 title="Subject", 441 ) 442 aud: Optional[str] = Field( 443 None, 444 description="The recipient that the JWT is intended for. Commonly defined by the API.", 445 examples=["appstoreconnect-v1"], 446 title="Audience", 447 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
450class RefreshTokenUpdater(BaseModel): 451 refresh_token_name: Optional[str] = Field( 452 "refresh_token", 453 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 454 examples=["refresh_token"], 455 title="Refresh Token Property Name", 456 ) 457 access_token_config_path: Optional[List[str]] = Field( 458 ["credentials", "access_token"], 459 description="Config path to the access token. Make sure the field actually exists in the config.", 460 examples=[["credentials", "access_token"], ["access_token"]], 461 title="Config Path To Access Token", 462 ) 463 refresh_token_config_path: Optional[List[str]] = Field( 464 ["credentials", "refresh_token"], 465 description="Config path to the access token. Make sure the field actually exists in the config.", 466 examples=[["credentials", "refresh_token"], ["refresh_token"]], 467 title="Config Path To Refresh Token", 468 ) 469 token_expiry_date_config_path: Optional[List[str]] = Field( 470 ["credentials", "token_expiry_date"], 471 description="Config path to the expiry date. Make sure actually exists in the config.", 472 examples=[["credentials", "token_expiry_date"]], 473 title="Config Path To Expiry Date", 474 ) 475 refresh_token_error_status_codes: Optional[List[int]] = Field( 476 [], 477 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 478 examples=[[400, 500]], 479 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 480 ) 481 refresh_token_error_key: Optional[str] = Field( 482 "", 483 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 484 examples=["error"], 485 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 486 ) 487 refresh_token_error_values: Optional[List[str]] = Field( 488 [], 489 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 490 examples=[["invalid_grant", "invalid_permissions"]], 491 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 492 )
class
Rate(pydantic.v1.main.BaseModel):
495class Rate(BaseModel): 496 class Config: 497 extra = Extra.allow 498 499 limit: Union[int, str] = Field( 500 ..., 501 description="The maximum number of calls allowed within the interval.", 502 title="Limit", 503 ) 504 interval: str = Field( 505 ..., 506 description="The time interval for the rate limit.", 507 examples=["PT1H", "P1D"], 508 title="Interval", 509 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
512class HttpRequestRegexMatcher(BaseModel): 513 class Config: 514 extra = Extra.allow 515 516 method: Optional[str] = Field( 517 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 518 ) 519 url_base: Optional[str] = Field( 520 None, 521 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 522 title="URL Base", 523 ) 524 url_path_pattern: Optional[str] = Field( 525 None, 526 description="A regular expression pattern to match the URL path.", 527 title="URL Path Pattern", 528 ) 529 params: Optional[Dict[str, Any]] = Field( 530 None, description="The query parameters to match.", title="Parameters" 531 ) 532 headers: Optional[Dict[str, Any]] = Field( 533 None, description="The headers to match.", title="Headers" 534 ) 535 weight: Optional[Union[int, str]] = Field( 536 None, 537 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", 538 title="Weight", 539 )
class
HttpRequestRegexMatcher.Config:
class
CombineMode(enum.Enum):
union =
<CombineMode.union: 'union'>
first_match =
<CombineMode.first_match: 'first_match'>
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
547class ResponseToFileExtractor(BaseModel): 548 type: Literal["ResponseToFileExtractor"] 549 preserve_na_values: Optional[bool] = Field( 550 False, 551 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 552 title="Preserve NA Values", 553 ) 554 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ParentFieldPath(pydantic.v1.main.BaseModel):
562class ParentFieldPath(BaseModel): 563 type: Literal["ParentFieldPath"] 564 parent_path: List[str] = Field( 565 ..., 566 description="Path to the value on the record being expanded.", 567 examples=[["url"], ["id"], ["repository", "name"]], 568 title="Parent Path", 569 ) 570 record_path: List[str] = Field( 571 ..., 572 description="Path on the expanded item to write the value to. An existing value is overwritten. Intermediate objects are created as needed, so a path may not pass through a value the item already holds as a scalar, and a numeric path segment creates an array rather than an object - the same behavior as `AddFields`.", 573 examples=[["pull_request_url"], ["comment_id"], ["parent", "id"]], 574 title="Record Path", 575 ) 576 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
579class ExponentialBackoffStrategy(BaseModel): 580 type: Literal["ExponentialBackoffStrategy"] 581 factor: Optional[Union[float, str]] = Field( 582 5, 583 description="Multiplicative constant applied on each retry.", 584 examples=[5, 5.5, "10"], 585 title="Factor", 586 ) 587 jitter_range_in_seconds: Optional[float] = Field( 588 None, 589 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 590 examples=[2], 591 ge=0, 592 title="Jitter Range", 593 ) 594 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
597class GroupByKeyMergeStrategy(BaseModel): 598 type: Literal["GroupByKeyMergeStrategy"] 599 key: Union[str, List[str]] = Field( 600 ..., 601 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 602 examples=["id", ["parent_id", "end_date"]], 603 title="Key", 604 ) 605 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
Action(enum.Enum):
612class Action(Enum): 613 SUCCESS = "SUCCESS" 614 FAIL = "FAIL" 615 RETRY = "RETRY" 616 IGNORE = "IGNORE" 617 RESET_PAGINATION = "RESET_PAGINATION" 618 RATE_LIMITED = "RATE_LIMITED" 619 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 620 REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" 621 SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
REDUCE_PAGE_SIZE =
<Action.REDUCE_PAGE_SIZE: 'REDUCE_PAGE_SIZE'>
SPLIT_REQUEST_WINDOW =
<Action.SPLIT_REQUEST_WINDOW: 'SPLIT_REQUEST_WINDOW'>
class
FailureType(enum.Enum):
624class FailureType(Enum): 625 system_error = "system_error" 626 config_error = "config_error" 627 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
630class HttpResponseFilter(BaseModel): 631 type: Literal["HttpResponseFilter"] 632 action: Optional[Action] = Field( 633 None, 634 description="Action to execute if a response matches the filter.", 635 examples=[ 636 "SUCCESS", 637 "FAIL", 638 "RETRY", 639 "IGNORE", 640 "RESET_PAGINATION", 641 "RATE_LIMITED", 642 "REFRESH_TOKEN_THEN_RETRY", 643 "REDUCE_PAGE_SIZE", 644 "SPLIT_REQUEST_WINDOW", 645 ], 646 title="Action", 647 ) 648 failure_type: Optional[FailureType] = Field( 649 None, 650 description="Failure type of traced exception if a response matches the filter.", 651 examples=["system_error", "config_error", "transient_error"], 652 title="Failure Type", 653 ) 654 error_message: Optional[str] = Field( 655 None, 656 description="Error Message to display if the response matches the filter.", 657 examples=["Skipping {{ request_url }} (HTTP 404)."], 658 title="Error Message", 659 ) 660 error_message_contains: Optional[str] = Field( 661 None, 662 description="Match the response if its error message contains the substring.", 663 example=["This API operation is not enabled for this site"], 664 title="Error Message Substring", 665 ) 666 http_codes: Optional[List[int]] = Field( 667 None, 668 description="Match the response if its HTTP code is included in this list.", 669 examples=[[420, 429], [500]], 670 title="HTTP Codes", 671 unique_items=True, 672 ) 673 predicate: Optional[str] = Field( 674 None, 675 description="Match the response if the predicate evaluates to true.", 676 examples=[ 677 "{{ 'Too much requests' in response }}", 678 "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", 679 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 680 ], 681 title="Predicate", 682 ) 683 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
686class ComplexFieldType(BaseModel): 687 field_type: str 688 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
691class TypesMap(BaseModel): 692 target_type: Union[str, List[str], ComplexFieldType] 693 current_type: Union[str, List[str]] 694 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
697class SchemaTypeIdentifier(BaseModel): 698 type: Optional[Literal["SchemaTypeIdentifier"]] = None 699 schema_pointer: Optional[List[str]] = Field( 700 [], 701 description="List of nested fields defining the schema field path to extract. Defaults to [].", 702 title="Schema Path", 703 ) 704 key_pointer: List[str] = Field( 705 ..., 706 description="List of potentially nested fields describing the full path of the field key to extract.", 707 title="Key Path", 708 ) 709 type_pointer: Optional[List[str]] = Field( 710 None, 711 description="List of potentially nested fields describing the full path of the field type to extract.", 712 title="Type Path", 713 ) 714 types_mapping: Optional[List[TypesMap]] = None 715 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
718class InlineSchemaLoader(BaseModel): 719 type: Literal["InlineSchemaLoader"] 720 schema_: Optional[Dict[str, Any]] = Field( 721 None, 722 alias="schema", 723 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 724 title="Schema", 725 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
728class JsonFileSchemaLoader(BaseModel): 729 type: Literal["JsonFileSchemaLoader"] 730 file_path: Optional[str] = Field( 731 None, 732 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 733 example=["./schemas/users.json"], 734 title="File Path", 735 ) 736 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
743class JsonItemsDecoder(BaseModel): 744 type: Literal["JsonItemsDecoder"] 745 items_path: str = Field( 746 ..., 747 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", 748 examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], 749 title="Items Path", 750 ) 751 encoding: Optional[str] = Field( 752 "utf-8", 753 description="Text encoding used to decode the streamed bytes before JSON parsing.", 754 title="Encoding", 755 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
772class FlattenFields(BaseModel): 773 type: Literal["FlattenFields"] 774 flatten_lists: Optional[bool] = Field( 775 True, 776 description="Whether to flatten lists or leave it as is. Default is True.", 777 title="Flatten Lists", 778 ) 779 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
782class KeyTransformation(BaseModel): 783 type: Literal["KeyTransformation"] 784 prefix: Optional[str] = Field( 785 None, 786 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 787 examples=["flattened_"], 788 title="Key Prefix", 789 ) 790 suffix: Optional[str] = Field( 791 None, 792 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 793 examples=["_flattened"], 794 title="Key Suffix", 795 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
798class DpathFlattenFields(BaseModel): 799 type: Literal["DpathFlattenFields"] 800 field_path: List[str] = Field( 801 ..., 802 description="A path to field that needs to be flattened.", 803 examples=[["data"], ["data", "*", "field"]], 804 title="Field Path", 805 ) 806 delete_origin_value: Optional[bool] = Field( 807 None, 808 description="Whether to delete the origin value or keep it. Default is False.", 809 title="Delete Origin Value", 810 ) 811 replace_record: Optional[bool] = Field( 812 None, 813 description="Whether to replace the origin record or not. Default is False.", 814 title="Replace Origin Record", 815 ) 816 key_transformation: Optional[KeyTransformation] = Field( 817 None, 818 description="Transformation for object keys. If not provided, original key will be used.", 819 title="Key transformation", 820 ) 821 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
824class KeysReplace(BaseModel): 825 type: Literal["KeysReplace"] 826 old: str = Field( 827 ..., 828 description="Old value to replace.", 829 examples=[ 830 " ", 831 "{{ record.id }}", 832 "{{ config['id'] }}", 833 "{{ stream_slice['id'] }}", 834 ], 835 title="Old value", 836 ) 837 new: str = Field( 838 ..., 839 description="New value to set.", 840 examples=[ 841 "_", 842 "{{ record.id }}", 843 "{{ config['id'] }}", 844 "{{ stream_slice['id'] }}", 845 ], 846 title="New value", 847 ) 848 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
859class CustomDecoder(BaseModel): 860 class Config: 861 extra = Extra.allow 862 863 type: Literal["CustomDecoder"] 864 class_name: str = Field( 865 ..., 866 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 867 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 868 title="Class Name", 869 ) 870 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
873class MinMaxDatetime(BaseModel): 874 type: Literal["MinMaxDatetime"] 875 datetime: str = Field( 876 ..., 877 description="Datetime value.", 878 examples=[ 879 "2021-01-01", 880 "2021-01-01T00:00:00Z", 881 "{{ config['start_time'] }}", 882 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 883 ], 884 title="Datetime", 885 ) 886 datetime_format: Optional[str] = Field( 887 "", 888 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 889 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 890 title="Datetime Format", 891 ) 892 max_datetime: Optional[str] = Field( 893 None, 894 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 895 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 896 title="Max Datetime", 897 ) 898 min_datetime: Optional[str] = Field( 899 None, 900 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 901 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 902 title="Min Datetime", 903 ) 904 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
Scope(pydantic.v1.main.BaseModel):
class
Scope.Config:
class
OptionalScope(pydantic.v1.main.BaseModel):
class
OptionalScope.Config:
class
ScopesJoinStrategy(enum.Enum):
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
944class OauthConnectorInputSpecification(BaseModel): 945 class Config: 946 extra = Extra.allow 947 948 consent_url: str = Field( 949 ..., 950 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 951 examples=[ 952 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 953 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 954 ], 955 title="Consent URL", 956 ) 957 scope: Optional[str] = Field( 958 None, 959 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 960 examples=["user:read user:read_orders workspaces:read"], 961 title="Scopes", 962 ) 963 scopes: Optional[List[Scope]] = Field( 964 None, 965 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 966 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 967 title="Scopes", 968 ) 969 optional_scopes: Optional[List[OptionalScope]] = Field( 970 None, 971 description="Optional OAuth scope objects that may or may not be granted.", 972 examples=[[{"scope": "admin:read"}]], 973 title="Optional Scopes", 974 ) 975 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 976 ScopesJoinStrategy.space, 977 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 978 title="Scopes Join Strategy", 979 ) 980 access_token_url: str = Field( 981 ..., 982 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 983 examples=[ 984 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 985 ], 986 title="Access Token URL", 987 ) 988 access_token_headers: Optional[Dict[str, Any]] = Field( 989 None, 990 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 991 examples=[ 992 { 993 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 994 } 995 ], 996 title="Access Token Headers", 997 ) 998 access_token_params: Optional[Dict[str, Any]] = Field( 999 None, 1000 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1001 examples=[ 1002 { 1003 "{{ auth_code_key }}": "{{ auth_code_value }}", 1004 "{{ client_id_key }}": "{{ client_id_value }}", 1005 "{{ client_secret_key }}": "{{ client_secret_value }}", 1006 } 1007 ], 1008 title="Access Token Query Params (Json Encoded)", 1009 ) 1010 extract_output: Optional[List[str]] = Field( 1011 None, 1012 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1013 examples=[["access_token", "refresh_token", "other_field"]], 1014 title="Extract Output", 1015 ) 1016 state: Optional[State] = Field( 1017 None, 1018 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1019 examples=[{"min": 7, "max": 128}], 1020 title="Configurable State Query Param", 1021 ) 1022 client_id_key: Optional[str] = Field( 1023 None, 1024 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1025 examples=["my_custom_client_id_key_name"], 1026 title="Client ID Key Override", 1027 ) 1028 client_secret_key: Optional[str] = Field( 1029 None, 1030 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1031 examples=["my_custom_client_secret_key_name"], 1032 title="Client Secret Key Override", 1033 ) 1034 scope_key: Optional[str] = Field( 1035 None, 1036 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1037 examples=["my_custom_scope_key_key_name"], 1038 title="Scopes Key Override", 1039 ) 1040 state_key: Optional[str] = Field( 1041 None, 1042 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1043 examples=["my_custom_state_key_key_name"], 1044 title="State Key Override", 1045 ) 1046 auth_code_key: Optional[str] = Field( 1047 None, 1048 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1049 examples=["my_custom_auth_code_key_name"], 1050 title="Auth Code Key Override", 1051 ) 1052 redirect_uri_key: Optional[str] = Field( 1053 None, 1054 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1055 examples=["my_custom_redirect_uri_key_name"], 1056 title="Redirect URI Key Override", 1057 )
scopes: Optional[List[Scope]]
optional_scopes: Optional[List[OptionalScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1060class OAuthConfigSpecification(BaseModel): 1061 class Config: 1062 extra = Extra.allow 1063 1064 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1065 None, 1066 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1067 examples=[ 1068 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1069 { 1070 "app_id": { 1071 "type": "string", 1072 "path_in_connector_config": ["info", "app_id"], 1073 } 1074 }, 1075 ], 1076 title="OAuth user input", 1077 ) 1078 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1079 None, 1080 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1081 title="DeclarativeOAuth Connector Specification", 1082 ) 1083 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1084 None, 1085 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1086 examples=[ 1087 { 1088 "refresh_token": { 1089 "type": "string,", 1090 "path_in_connector_config": ["credentials", "refresh_token"], 1091 } 1092 } 1093 ], 1094 title="OAuth output specification", 1095 ) 1096 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1097 None, 1098 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1099 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1100 title="OAuth input specification", 1101 ) 1102 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1103 None, 1104 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1105 examples=[ 1106 { 1107 "client_id": { 1108 "type": "string,", 1109 "path_in_connector_config": ["credentials", "client_id"], 1110 }, 1111 "client_secret": { 1112 "type": "string,", 1113 "path_in_connector_config": ["credentials", "client_secret"], 1114 }, 1115 } 1116 ], 1117 title="OAuth server output specification", 1118 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1121class OffsetIncrement(BaseModel): 1122 type: Literal["OffsetIncrement"] 1123 page_size: Optional[Union[int, str]] = Field( 1124 None, 1125 description="The number of records to include in each pages.", 1126 examples=[100, "{{ config['page_size'] }}"], 1127 title="Limit", 1128 ) 1129 inject_on_first_request: Optional[bool] = Field( 1130 False, 1131 description="Using the `offset` with value `0` during the first request", 1132 title="Inject Offset on First Request", 1133 ) 1134 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1137class PageIncrement(BaseModel): 1138 type: Literal["PageIncrement"] 1139 page_size: Optional[Union[int, str]] = Field( 1140 None, 1141 description="The number of records to include in each pages.", 1142 examples=[100, "100", "{{ config['page_size'] }}"], 1143 title="Page Size", 1144 ) 1145 start_from_page: Optional[int] = Field( 1146 0, 1147 description="Index of the first page to request.", 1148 examples=[0, 1], 1149 title="Start From Page", 1150 ) 1151 inject_on_first_request: Optional[bool] = Field( 1152 False, 1153 description="Using the `page number` with value defined by `start_from_page` during the first request", 1154 title="Inject Page Number on First Request", 1155 ) 1156 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1159class PrimaryKey(BaseModel): 1160 __root__: Union[str, List[str], List[List[str]]] = Field( 1161 ..., 1162 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1163 examples=["id", ["code", "type"]], 1164 title="Primary Key", 1165 )
class
PropertyLimitType(enum.Enum):
1168class PropertyLimitType(Enum): 1169 characters = "characters" 1170 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1173class PropertyChunking(BaseModel): 1174 type: Literal["PropertyChunking"] 1175 property_limit_type: PropertyLimitType = Field( 1176 ..., 1177 description="The type used to determine the maximum number of properties per chunk", 1178 title="Property Limit Type", 1179 ) 1180 property_limit: Optional[int] = Field( 1181 None, 1182 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1183 title="Property Limit", 1184 ) 1185 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1186 None, 1187 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1188 title="Record Merge Strategy", 1189 ) 1190 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1193class RecordFilter(BaseModel): 1194 type: Literal["RecordFilter"] 1195 condition: Optional[str] = Field( 1196 "", 1197 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1198 examples=[ 1199 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1200 "{{ record.status in ['active', 'expired'] }}", 1201 ], 1202 title="Condition", 1203 ) 1204 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1212class RemoveFields(BaseModel): 1213 type: Literal["RemoveFields"] 1214 condition: Optional[str] = Field( 1215 "", 1216 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1217 examples=[ 1218 "{{ property|string == '' }}", 1219 "{{ property is integer }}", 1220 "{{ property|length > 5 }}", 1221 "{{ property == 'some_string_to_match' }}", 1222 ], 1223 ) 1224 field_pointers: List[List[str]] = Field( 1225 ..., 1226 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1227 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1228 title="Field Paths", 1229 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1236class InjectInto(Enum): 1237 request_parameter = "request_parameter" 1238 header = "header" 1239 body_data = "body_data" 1240 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1243class RequestOption(BaseModel): 1244 type: Literal["RequestOption"] 1245 inject_into: InjectInto = Field( 1246 ..., 1247 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1248 examples=["request_parameter", "header", "body_data", "body_json"], 1249 title="Inject Into", 1250 ) 1251 field_name: Optional[str] = Field( 1252 None, 1253 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1254 examples=["segment_id"], 1255 title="Field Name", 1256 ) 1257 field_path: Optional[List[str]] = Field( 1258 None, 1259 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1260 examples=[["data", "viewer", "id"]], 1261 title="Field Path", 1262 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1272class LegacySessionTokenAuthenticator(BaseModel): 1273 type: Literal["LegacySessionTokenAuthenticator"] 1274 header: str = Field( 1275 ..., 1276 description="The name of the session token header that will be injected in the request", 1277 examples=["X-Session"], 1278 title="Session Request Header", 1279 ) 1280 login_url: str = Field( 1281 ..., 1282 description="Path of the login URL (do not include the base URL)", 1283 examples=["session"], 1284 title="Login Path", 1285 ) 1286 session_token: Optional[str] = Field( 1287 None, 1288 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1289 example=["{{ config['session_token'] }}"], 1290 title="Session Token", 1291 ) 1292 session_token_response_key: str = Field( 1293 ..., 1294 description="Name of the key of the session token to be extracted from the response", 1295 examples=["id"], 1296 title="Response Token Response Key", 1297 ) 1298 username: Optional[str] = Field( 1299 None, 1300 description="Username used to authenticate and obtain a session token", 1301 examples=[" {{ config['username'] }}"], 1302 title="Username", 1303 ) 1304 password: Optional[str] = Field( 1305 "", 1306 description="Password used to authenticate and obtain a session token", 1307 examples=["{{ config['password'] }}", ""], 1308 title="Password", 1309 ) 1310 validate_session_url: str = Field( 1311 ..., 1312 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1313 examples=["user/current"], 1314 title="Validate Session Path", 1315 ) 1316 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
ResetPolicy(enum.Enum):
1329class ResetPolicy(Enum): 1330 NEVER = "NEVER" 1331 AFTER_SUCCESSFUL_PAGE = "AFTER_SUCCESSFUL_PAGE"
NEVER =
<ResetPolicy.NEVER: 'NEVER'>
AFTER_SUCCESSFUL_PAGE =
<ResetPolicy.AFTER_SUCCESSFUL_PAGE: 'AFTER_SUCCESSFUL_PAGE'>
class
PageSizeReduction(pydantic.v1.main.BaseModel):
1334class PageSizeReduction(BaseModel): 1335 type: Literal["PageSizeReduction"] 1336 reduction_factor: Optional[float] = Field( 1337 2, 1338 description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", 1339 examples=[2, 4], 1340 gt=1, 1341 title="Reduction Factor", 1342 ) 1343 minimum_page_size: Optional[int] = Field( 1344 1, 1345 description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", 1346 examples=[1, 10], 1347 ge=1, 1348 title="Minimum Page Size", 1349 ) 1350 max_attempts: Optional[int] = Field( 1351 5, 1352 description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", 1353 examples=[5, 10], 1354 ge=1, 1355 title="Maximum Reduction Attempts", 1356 ) 1357 backoff_seconds: Optional[float] = Field( 1358 0.5, 1359 description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', 1360 examples=[0.5, 5], 1361 ge=0, 1362 title="Backoff Seconds", 1363 ) 1364 retries_at_minimum_page_size: Optional[int] = Field( 1365 0, 1366 description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", 1367 examples=[0, 3], 1368 ge=0, 1369 title="Retries At Minimum Page Size", 1370 ) 1371 failure_message: Optional[str] = Field( 1372 None, 1373 description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because max_attempts was reached or because the page size is already at minimum_page_size. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every page size the connector asked for.", 1374 examples=[ 1375 "Narrow the sync down by selecting fewer fields on this stream.", 1376 "Set a more recent start date so that each page covers less data.", 1377 ], 1378 title="Failure Message", 1379 ) 1380 reset_policy: Optional[ResetPolicy] = Field( 1381 ResetPolicy.NEVER, 1382 description="When to restore the page size configured on the pagination strategy. NEVER keeps the reduced page size for the rest of the partition. AFTER_SUCCESSFUL_PAGE restores it as soon as one page succeeds, which means hitting the same error again on every page - use it only when the reduction is worth one extra request per page, for instance because the configured page size usually works and only some pages are too heavy. It only controls the page size: the max_attempts budget restarts on every page that succeeds under both policies, so there is no limit on how many reductions a partition may make in total. What is bounded is the reductions that get no page through.", 1383 title="Reset Policy", 1384 )
reset_policy: Optional[ResetPolicy]
class
RequestWindowSplitting(pydantic.v1.main.BaseModel):
1387class RequestWindowSplitting(BaseModel): 1388 type: Literal["RequestWindowSplitting"] 1389 failure_message: Optional[str] = Field( 1390 None, 1391 description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", 1392 examples=[ 1393 "Narrow the sync down by selecting fewer fields on this stream.", 1394 "Lower time_window so that each request covers less data.", 1395 ], 1396 title="Failure Message", 1397 ) 1398 min_split_window: Optional[str] = Field( 1399 None, 1400 description="Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or below this size is not split further, though the result can end up a bit smaller than this, not larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", 1401 examples=["P1D", "PT1H"], 1402 title="Minimum Split Window", 1403 )
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
1413class AsyncJobStatusMap(BaseModel): 1414 type: Optional[Literal["AsyncJobStatusMap"]] = None 1415 running: List[str] 1416 completed: List[str] 1417 failed: List[str] 1418 timeout: List[str] 1419 skipped: Optional[List[str]] = Field( 1420 None, 1421 description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", 1422 )
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1429class ValueType(Enum): 1430 string = "string" 1431 number = "number" 1432 integer = "integer" 1433 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1436class WaitTimeFromHeader(BaseModel): 1437 type: Literal["WaitTimeFromHeader"] 1438 header: str = Field( 1439 ..., 1440 description="The name of the response header defining how long to wait before retrying.", 1441 examples=["Retry-After"], 1442 title="Response Header Name", 1443 ) 1444 regex: Optional[str] = Field( 1445 None, 1446 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1447 examples=["([-+]?\\d+)"], 1448 title="Extraction Regex", 1449 ) 1450 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1451 None, 1452 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1453 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1454 title="Max Waiting Time in Seconds", 1455 ) 1456 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitTimeFromResponse(pydantic.v1.main.BaseModel):
1459class WaitTimeFromResponse(BaseModel): 1460 type: Literal["WaitTimeFromResponse"] 1461 wait_time_in_seconds: str = Field( 1462 ..., 1463 description="The expression defining how long to wait before retrying, in seconds relative to now (convert an absolute timestamp with `timestamp(...) - now_utc().timestamp()`). An empty, zero, negative or non-numeric result, or an expression that cannot be evaluated on the response, defers to the next backoff strategy. Nothing bounds the wait unless `max_waiting_time_in_seconds` is set, so setting it is strongly recommended.", 1464 examples=[ 1465 "{{ response.retry_after_seconds }}", 1466 "{{ max(1, timestamp(response.error.message | regex_search('Retry after (\\S+Z)')) - now_utc().timestamp()) if (response is mapping and response.get('error') is mapping and ((response.error.message or '') | string | regex_search('Retry after (\\S+Z)'))) else '' }}", 1467 ], 1468 title="Wait Time in Seconds", 1469 ) 1470 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1471 None, 1472 description="Stop the stream instead of waiting, when the computed wait is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1473 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1474 title="Max Waiting Time in Seconds", 1475 ) 1476 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1479class WaitUntilTimeFromHeader(BaseModel): 1480 type: Literal["WaitUntilTimeFromHeader"] 1481 header: str = Field( 1482 ..., 1483 description="The name of the response header defining how long to wait before retrying.", 1484 examples=["wait_time"], 1485 title="Response Header", 1486 ) 1487 min_wait: Optional[Union[float, str]] = Field( 1488 None, 1489 description="Minimum time to wait before retrying.", 1490 examples=[10, "60"], 1491 title="Minimum Wait Time", 1492 ) 1493 regex: Optional[str] = Field( 1494 None, 1495 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1496 examples=["([-+]?\\d+)"], 1497 title="Extraction Regex", 1498 ) 1499 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1500 None, 1501 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1502 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1503 title="Max Waiting Time in Seconds", 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1508class ComponentMappingDefinition(BaseModel): 1509 type: Literal["ComponentMappingDefinition"] 1510 field_path: List[str] = Field( 1511 ..., 1512 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1513 examples=[ 1514 ["name"], 1515 ["retriever", "requester", "url"], 1516 ["retriever", "requester", "{{ components_values.field }}"], 1517 ["*", "**", "name"], 1518 ], 1519 title="Field Path", 1520 ) 1521 value: str = Field( 1522 ..., 1523 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1524 examples=[ 1525 "{{ components_values['updates'] }}", 1526 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1527 "{{ config['segment_id'] }}", 1528 "{{ stream_slice['parent_id'] }}", 1529 "{{ stream_slice['extra_fields']['name'] }}", 1530 ], 1531 title="Value", 1532 ) 1533 value_type: Optional[ValueType] = Field( 1534 None, 1535 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1536 title="Value Type", 1537 ) 1538 create_or_update: Optional[bool] = Field( 1539 False, 1540 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1541 title="Create or Update", 1542 ) 1543 condition: Optional[str] = Field( 1544 None, 1545 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1546 examples=[ 1547 "{{ components_values.get('cursor_field', None) }}", 1548 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1549 ], 1550 title="Condition", 1551 ) 1552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1555class StreamConfig(BaseModel): 1556 type: Literal["StreamConfig"] 1557 configs_pointer: List[str] = Field( 1558 ..., 1559 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1560 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1561 title="Configs Pointer", 1562 ) 1563 default_values: Optional[List[Dict[str, Any]]] = Field( 1564 None, 1565 description="A list of default values, each matching the structure expected from the parsed component value.", 1566 title="Default Values", 1567 ) 1568 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1571class ConfigComponentsResolver(BaseModel): 1572 type: Literal["ConfigComponentsResolver"] 1573 stream_config: Union[List[StreamConfig], StreamConfig] 1574 components_mapping: List[ComponentMappingDefinition] 1575 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1578class StreamParametersDefinition(BaseModel): 1579 type: Literal["StreamParametersDefinition"] 1580 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1581 ..., 1582 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1583 examples=[ 1584 [ 1585 { 1586 "name": "test stream", 1587 "$parameters": {"entity": "test entity"}, 1588 "primary_key": "test key", 1589 } 1590 ] 1591 ], 1592 title="Stream Parameters", 1593 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1596class ParametrizedComponentsResolver(BaseModel): 1597 type: Literal["ParametrizedComponentsResolver"] 1598 stream_parameters: StreamParametersDefinition 1599 components_mapping: List[ComponentMappingDefinition] 1600 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1625class ValidateAdheresToSchema(BaseModel): 1626 type: Literal["ValidateAdheresToSchema"] 1627 base_schema: Union[str, Dict[str, Any]] = Field( 1628 ..., 1629 description="The base JSON schema against which the user-provided schema will be validated.", 1630 examples=[ 1631 "{{ config['report_validation_schema'] }}", 1632 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1633 { 1634 "$schema": "http://json-schema.org/draft-07/schema#", 1635 "title": "Person", 1636 "type": "object", 1637 "properties": { 1638 "name": {"type": "string", "description": "The person's name"}, 1639 "age": { 1640 "type": "integer", 1641 "minimum": 0, 1642 "description": "The person's age", 1643 }, 1644 }, 1645 "required": ["name", "age"], 1646 }, 1647 ], 1648 title="Base JSON Schema", 1649 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1652class CustomValidationStrategy(BaseModel): 1653 class Config: 1654 extra = Extra.allow 1655 1656 type: Literal["CustomValidationStrategy"] 1657 class_name: str = Field( 1658 ..., 1659 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1660 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1661 title="Class Name", 1662 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1665class ConfigRemapField(BaseModel): 1666 type: Literal["ConfigRemapField"] 1667 map: Union[Dict[str, Any], str] = Field( 1668 ..., 1669 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1670 examples=[ 1671 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1672 "{{ config['status_mapping'] }}", 1673 ], 1674 title="Value Mapping", 1675 ) 1676 field_path: List[str] = Field( 1677 ..., 1678 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1679 examples=[ 1680 ["status"], 1681 ["data", "status"], 1682 ["data", "{{ config.name }}", "status"], 1683 ["data", "*", "status"], 1684 ], 1685 title="Field Path", 1686 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1689class ConfigRemoveFields(BaseModel): 1690 type: Literal["ConfigRemoveFields"] 1691 field_pointers: List[List[str]] = Field( 1692 ..., 1693 description="A list of field pointers to be removed from the config.", 1694 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1695 title="Field Pointers", 1696 ) 1697 condition: Optional[str] = Field( 1698 "", 1699 description="Fields will be removed if expression is evaluated to True.", 1700 examples=[ 1701 "{{ config['environemnt'] == 'sandbox' }}", 1702 "{{ property is integer }}", 1703 "{{ property|length > 5 }}", 1704 "{{ property == 'some_string_to_match' }}", 1705 ], 1706 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1709class CustomConfigTransformation(BaseModel): 1710 type: Literal["CustomConfigTransformation"] 1711 class_name: str = Field( 1712 ..., 1713 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1714 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1715 ) 1716 parameters: Optional[Dict[str, Any]] = Field( 1717 None, 1718 alias="$parameters", 1719 description="Additional parameters to be passed to the custom config transformation.", 1720 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1723class AddedFieldDefinition(BaseModel): 1724 type: Literal["AddedFieldDefinition"] 1725 path: List[str] = Field( 1726 ..., 1727 description="List of strings defining the path where to add the value on the record.", 1728 examples=[["segment_id"], ["metadata", "segment_id"]], 1729 title="Path", 1730 ) 1731 value: str = Field( 1732 ..., 1733 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1734 examples=[ 1735 "{{ record['updates'] }}", 1736 "{{ record['MetaData']['LastUpdatedTime'] }}", 1737 "{{ stream_partition['segment_id'] }}", 1738 ], 1739 title="Value", 1740 ) 1741 value_type: Optional[ValueType] = Field( 1742 None, 1743 description="Type of the value. If not specified, the type will be inferred from the value.", 1744 title="Value Type", 1745 ) 1746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1749class AddFields(BaseModel): 1750 type: Literal["AddFields"] 1751 fields: List[AddedFieldDefinition] = Field( 1752 ..., 1753 description="List of transformations (path and corresponding value) that will be added to the record.", 1754 title="Fields", 1755 ) 1756 condition: Optional[str] = Field( 1757 "", 1758 description="Fields will be added if expression is evaluated to True.", 1759 examples=[ 1760 "{{ property|string == '' }}", 1761 "{{ property is integer }}", 1762 "{{ property|length > 5 }}", 1763 "{{ property == 'some_string_to_match' }}", 1764 ], 1765 ) 1766 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1769class ApiKeyAuthenticator(BaseModel): 1770 type: Literal["ApiKeyAuthenticator"] 1771 api_token: Optional[str] = Field( 1772 None, 1773 description="The API key to inject in the request. Fill it in the user inputs.", 1774 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1775 title="API Key", 1776 ) 1777 header: Optional[str] = Field( 1778 None, 1779 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1780 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1781 title="Header Name", 1782 ) 1783 inject_into: Optional[RequestOption] = Field( 1784 None, 1785 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1786 examples=[ 1787 {"inject_into": "header", "field_name": "Authorization"}, 1788 {"inject_into": "request_parameter", "field_name": "authKey"}, 1789 ], 1790 title="Inject API Key Into Outgoing HTTP Request", 1791 ) 1792 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1795class AuthFlow(BaseModel): 1796 auth_flow_type: Optional[AuthFlowType] = Field( 1797 None, description="The type of auth to use", title="Auth flow type" 1798 ) 1799 predicate_key: Optional[List[str]] = Field( 1800 None, 1801 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1802 examples=[["credentials", "auth_type"]], 1803 title="Predicate key", 1804 ) 1805 predicate_value: Optional[str] = Field( 1806 None, 1807 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1808 examples=["Oauth"], 1809 title="Predicate value", 1810 ) 1811 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
TokenQuota(pydantic.v1.main.BaseModel):
1814class TokenQuota(BaseModel): 1815 type: Literal["TokenQuota"] 1816 name: str = Field( 1817 ..., 1818 description="Name of the quota pool.", 1819 examples=["rest", "graphql"], 1820 title="Name", 1821 ) 1822 remaining_path: List[str] = Field( 1823 ..., 1824 description="Path to the remaining call count for this pool in the quota status response.", 1825 examples=[["resources", "core", "remaining"]], 1826 title="Remaining Path", 1827 ) 1828 reset_path: List[str] = Field( 1829 ..., 1830 description="Path to the quota reset timestamp for this pool in the quota status response.", 1831 examples=[["resources", "core", "reset"]], 1832 title="Reset Path", 1833 ) 1834 limit_path: Optional[List[str]] = Field( 1835 None, 1836 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 1837 examples=[["resources", "core", "limit"]], 1838 title="Limit Path", 1839 ) 1840 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 1841 None, 1842 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 1843 title="Matchers", 1844 ) 1845 remaining_header: Optional[str] = Field( 1846 None, 1847 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 1848 examples=["X-RateLimit-Remaining"], 1849 title="Remaining Header", 1850 ) 1851 reset_header: Optional[str] = Field( 1852 None, 1853 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 1854 examples=["X-RateLimit-Reset"], 1855 title="Reset Header", 1856 ) 1857 limit_header: Optional[str] = Field( 1858 None, 1859 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 1860 examples=["X-RateLimit-Limit"], 1861 title="Limit Header", 1862 ) 1863 exhaustion_status_codes: Optional[List[int]] = Field( 1864 None, 1865 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 1866 examples=[[429]], 1867 title="Exhaustion Status Codes", 1868 ) 1869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
matchers: Optional[List[HttpRequestRegexMatcher]]
class
CheckStream(pydantic.v1.main.BaseModel):
1872class CheckStream(BaseModel): 1873 type: Literal["CheckStream"] 1874 stream_names: Optional[List[str]] = Field( 1875 None, 1876 description="Names of the streams to try reading from when running a check operation.", 1877 examples=[["users"], ["users", "contacts"]], 1878 title="Stream Names", 1879 ) 1880 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1881 config_overrides: Optional[Dict[str, Any]] = Field( 1882 None, 1883 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1884 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1885 title="Config Overrides", 1886 )
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1889class IncrementingCountCursor(BaseModel): 1890 type: Literal["IncrementingCountCursor"] 1891 cursor_field: str = Field( 1892 ..., 1893 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1894 examples=["created_at", "{{ config['record_cursor'] }}"], 1895 title="Cursor Field", 1896 ) 1897 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1898 None, 1899 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1900 title="Allow Catalog Defined Cursor Field", 1901 ) 1902 start_value: Optional[Union[str, int]] = Field( 1903 None, 1904 description="The value that determines the earliest record that should be synced.", 1905 examples=[0, "{{ config['start_value'] }}"], 1906 title="Start Value", 1907 ) 1908 start_value_option: Optional[RequestOption] = Field( 1909 None, 1910 description="Optionally configures how the start value will be sent in requests to the source API.", 1911 title="Inject Start Value Into Outgoing HTTP Request", 1912 ) 1913 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1916class DatetimeBasedCursor(BaseModel): 1917 type: Literal["DatetimeBasedCursor"] 1918 clamping: Optional[Clamping] = Field( 1919 None, 1920 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1921 title="Date Range Clamping", 1922 ) 1923 cursor_field: str = Field( 1924 ..., 1925 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1926 examples=["created_at", "{{ config['record_cursor'] }}"], 1927 title="Cursor Field", 1928 ) 1929 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1930 None, 1931 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1932 title="Allow Catalog Defined Cursor Field", 1933 ) 1934 cursor_datetime_formats: Optional[List[str]] = Field( 1935 None, 1936 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1937 examples=[ 1938 "%Y-%m-%d", 1939 "%Y-%m-%d %H:%M:%S", 1940 "%Y-%m-%dT%H:%M:%S", 1941 "%Y-%m-%dT%H:%M:%SZ", 1942 "%Y-%m-%dT%H:%M:%S%z", 1943 "%Y-%m-%dT%H:%M:%S.%fZ", 1944 "%Y-%m-%dT%H:%M:%S.%f%z", 1945 "%Y-%m-%d %H:%M:%S.%f+00:00", 1946 "%s", 1947 "%ms", 1948 ], 1949 title="Cursor Datetime Formats", 1950 ) 1951 start_datetime: Union[MinMaxDatetime, str] = Field( 1952 ..., 1953 description="The datetime that determines the earliest record that should be synced.", 1954 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1955 title="Start Datetime", 1956 ) 1957 start_time_option: Optional[RequestOption] = Field( 1958 None, 1959 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1960 title="Inject Start Time Into Outgoing HTTP Request", 1961 ) 1962 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1963 None, 1964 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1965 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1966 title="End Datetime", 1967 ) 1968 end_time_option: Optional[RequestOption] = Field( 1969 None, 1970 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1971 title="Inject End Time Into Outgoing HTTP Request", 1972 ) 1973 datetime_format: str = Field( 1974 ..., 1975 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1976 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1977 title="Outgoing Datetime Format", 1978 ) 1979 cursor_granularity: Optional[str] = Field( 1980 None, 1981 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1982 examples=["PT1S"], 1983 title="Cursor Granularity", 1984 ) 1985 is_data_feed: Optional[bool] = Field( 1986 None, 1987 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1988 title="Data Feed API", 1989 ) 1990 is_client_side_incremental: Optional[bool] = Field( 1991 None, 1992 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value minus the lookback window, if one is set (never before the start date), and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1993 title="Client-side Incremental Filtering", 1994 ) 1995 is_compare_strictly: Optional[bool] = Field( 1996 False, 1997 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1998 title="Strict Start-End Time Comparison", 1999 ) 2000 global_substream_cursor: Optional[bool] = Field( 2001 False, 2002 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 2003 title="Global Substream Cursor", 2004 ) 2005 lookback_window: Optional[str] = Field( 2006 None, 2007 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2008 examples=["P1D", "P{{ config['lookback_days'] }}D"], 2009 title="Lookback Window", 2010 ) 2011 partition_field_end: Optional[str] = Field( 2012 None, 2013 description="Name of the partition start time field.", 2014 examples=["ending_time"], 2015 title="Partition Field End", 2016 ) 2017 partition_field_start: Optional[str] = Field( 2018 None, 2019 description="Name of the partition end time field.", 2020 examples=["starting_time"], 2021 title="Partition Field Start", 2022 ) 2023 step: Optional[str] = Field( 2024 None, 2025 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2026 examples=["P1W", "{{ config['step_increment'] }}"], 2027 title="Step", 2028 ) 2029 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
2032class JwtAuthenticator(BaseModel): 2033 type: Literal["JwtAuthenticator"] 2034 secret_key: str = Field( 2035 ..., 2036 description="Secret used to sign the JSON web token.", 2037 examples=["{{ config['secret_key'] }}"], 2038 title="Secret Key", 2039 ) 2040 base64_encode_secret_key: Optional[bool] = Field( 2041 False, 2042 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 2043 title="Base64-encode Secret Key", 2044 ) 2045 algorithm: Algorithm = Field( 2046 ..., 2047 description="Algorithm used to sign the JSON web token.", 2048 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 2049 title="Algorithm", 2050 ) 2051 token_duration: Optional[int] = Field( 2052 1200, 2053 description="The amount of time in seconds a JWT token can be valid after being issued.", 2054 examples=[1200, 3600], 2055 title="Token Duration", 2056 ) 2057 header_prefix: Optional[str] = Field( 2058 None, 2059 description="The prefix to be used within the Authentication header.", 2060 examples=["Bearer", "Basic"], 2061 title="Header Prefix", 2062 ) 2063 jwt_headers: Optional[JwtHeaders] = Field( 2064 None, 2065 description="JWT headers used when signing JSON web token.", 2066 title="JWT Headers", 2067 ) 2068 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 2069 None, 2070 description="Additional headers to be included with the JWT headers object.", 2071 title="Additional JWT Headers", 2072 ) 2073 jwt_payload: Optional[JwtPayload] = Field( 2074 None, 2075 description="JWT Payload used when signing JSON web token.", 2076 title="JWT Payload", 2077 ) 2078 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2079 None, 2080 description="Additional properties to be added to the JWT payload.", 2081 title="Additional JWT Payload Properties", 2082 ) 2083 passphrase: Optional[str] = Field( 2084 None, 2085 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2086 examples=["{{ config['passphrase'] }}"], 2087 title="Passphrase", 2088 ) 2089 request_option: Optional[RequestOption] = Field( 2090 None, 2091 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2092 title="Request Option", 2093 ) 2094 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
2097class OAuthAuthenticator(BaseModel): 2098 type: Literal["OAuthAuthenticator"] 2099 client_id_name: Optional[str] = Field( 2100 "client_id", 2101 description="The name of the property to use to refresh the `access_token`.", 2102 examples=["custom_app_id"], 2103 title="Client ID Property Name", 2104 ) 2105 client_id: Optional[str] = Field( 2106 None, 2107 description="The OAuth client ID. Fill it in the user inputs.", 2108 examples=[ 2109 "{{ config['client_id'] }}", 2110 "{{ config['credentials']['client_id }}", 2111 ], 2112 title="Client ID", 2113 ) 2114 client_secret_name: Optional[str] = Field( 2115 "client_secret", 2116 description="The name of the property to use to refresh the `access_token`.", 2117 examples=["custom_app_secret"], 2118 title="Client Secret Property Name", 2119 ) 2120 client_secret: Optional[str] = Field( 2121 None, 2122 description="The OAuth client secret. Fill it in the user inputs.", 2123 examples=[ 2124 "{{ config['client_secret'] }}", 2125 "{{ config['credentials']['client_secret }}", 2126 ], 2127 title="Client Secret", 2128 ) 2129 refresh_token_name: Optional[str] = Field( 2130 "refresh_token", 2131 description="The name of the property to use to refresh the `access_token`.", 2132 examples=["custom_app_refresh_value"], 2133 title="Refresh Token Property Name", 2134 ) 2135 refresh_token: Optional[str] = Field( 2136 None, 2137 description="Credential artifact used to get a new access token.", 2138 examples=[ 2139 "{{ config['refresh_token'] }}", 2140 "{{ config['credentials]['refresh_token'] }}", 2141 ], 2142 title="Refresh Token", 2143 ) 2144 token_refresh_endpoint: Optional[str] = Field( 2145 None, 2146 description="The full URL to call to obtain a new access token.", 2147 examples=["https://connect.squareup.com/oauth2/token"], 2148 title="Token Refresh Endpoint", 2149 ) 2150 access_token_name: Optional[str] = Field( 2151 "access_token", 2152 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2153 examples=["access_token"], 2154 title="Access Token Property Name", 2155 ) 2156 access_token_value: Optional[str] = Field( 2157 None, 2158 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2159 examples=["secret_access_token_value"], 2160 title="Access Token Value", 2161 ) 2162 expires_in_name: Optional[str] = Field( 2163 "expires_in", 2164 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2165 examples=["expires_in"], 2166 title="Token Expiry Property Name", 2167 ) 2168 grant_type_name: Optional[str] = Field( 2169 "grant_type", 2170 description="The name of the property to use to refresh the `access_token`.", 2171 examples=["custom_grant_type"], 2172 title="Grant Type Property Name", 2173 ) 2174 grant_type: Optional[str] = Field( 2175 "refresh_token", 2176 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2177 examples=["refresh_token", "client_credentials"], 2178 title="Grant Type", 2179 ) 2180 refresh_request_body: Optional[Dict[str, Any]] = Field( 2181 None, 2182 description="Body of the request sent to get a new access token.", 2183 examples=[ 2184 { 2185 "applicationId": "{{ config['application_id'] }}", 2186 "applicationSecret": "{{ config['application_secret'] }}", 2187 "token": "{{ config['token'] }}", 2188 } 2189 ], 2190 title="Refresh Request Body", 2191 ) 2192 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2193 None, 2194 description="Headers of the request sent to get a new access token.", 2195 examples=[ 2196 { 2197 "Authorization": "<AUTH_TOKEN>", 2198 "Content-Type": "application/x-www-form-urlencoded", 2199 } 2200 ], 2201 title="Refresh Request Headers", 2202 ) 2203 send_refresh_request_as_query_params: Optional[bool] = Field( 2204 False, 2205 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2206 examples=[True], 2207 title="Send Refresh Request As Query Params", 2208 ) 2209 scopes: Optional[List[str]] = Field( 2210 None, 2211 description="List of scopes that should be granted to the access token.", 2212 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2213 title="Scopes", 2214 ) 2215 token_expiry_date: Optional[str] = Field( 2216 None, 2217 description="The access token expiry date.", 2218 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2219 title="Token Expiry Date", 2220 ) 2221 token_expiry_date_format: Optional[str] = Field( 2222 None, 2223 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2224 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2225 title="Token Expiry Date Format", 2226 ) 2227 refresh_token_error_status_codes: Optional[List[int]] = Field( 2228 None, 2229 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2230 examples=[[400, 500]], 2231 title="Refresh Token Error Status Codes", 2232 ) 2233 refresh_token_error_key: Optional[str] = Field( 2234 None, 2235 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2236 examples=["error"], 2237 title="Refresh Token Error Key", 2238 ) 2239 refresh_token_error_values: Optional[List[str]] = Field( 2240 None, 2241 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2242 examples=[["invalid_grant", "invalid_permissions"]], 2243 title="Refresh Token Error Values", 2244 ) 2245 refresh_token_error_messages: Optional[Dict[str, str]] = Field( 2246 None, 2247 description="Fixed user-facing messages for refresh token errors. Keys are matched in order against the provider error code leading error_description (a token such as ABC12345 followed by a word boundary), the whitespace-normalized full error_description, then the OAuth error value; the first match replaces the default re-authentication guidance, and the provider error detail is still appended. Only applies to responses already flagged as config errors by Refresh Token Error Status Codes, Refresh Token Error Key and Refresh Token Error Values (or their defaults); it does not flag any additional responses. Values are used verbatim, without interpolation, so the message is identical on every attempt.", 2248 examples=[ 2249 { 2250 "invalid_grant": "The refresh token expired or was revoked. Re-authenticate this source to restore syncs." 2251 } 2252 ], 2253 title="Refresh Token Error Messages", 2254 ) 2255 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2256 None, 2257 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2258 title="Refresh Token Updater", 2259 ) 2260 profile_assertion: Optional[JwtAuthenticator] = Field( 2261 None, 2262 description="The authenticator being used to authenticate the client authenticator.", 2263 title="Profile Assertion", 2264 ) 2265 use_profile_assertion: Optional[bool] = Field( 2266 False, 2267 description="Enable using profile assertion as a flow for OAuth authorization.", 2268 title="Use Profile Assertion", 2269 ) 2270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2273class FixedWindowCallRatePolicy(BaseModel): 2274 class Config: 2275 extra = Extra.allow 2276 2277 type: Literal["FixedWindowCallRatePolicy"] 2278 period: str = Field( 2279 ..., description="The time interval for the rate limit window.", title="Period" 2280 ) 2281 call_limit: int = Field( 2282 ..., 2283 description="The maximum number of calls allowed within the period.", 2284 title="Call Limit", 2285 ) 2286 matchers: List[HttpRequestRegexMatcher] = Field( 2287 ..., 2288 description="List of matchers that define which requests this policy applies to.", 2289 title="Matchers", 2290 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2293class MovingWindowCallRatePolicy(BaseModel): 2294 class Config: 2295 extra = Extra.allow 2296 2297 type: Literal["MovingWindowCallRatePolicy"] 2298 rates: List[Rate] = Field( 2299 ..., 2300 description="List of rates that define the call limits for different time intervals.", 2301 title="Rates", 2302 ) 2303 matchers: List[HttpRequestRegexMatcher] = Field( 2304 ..., 2305 description="List of matchers that define which requests this policy applies to.", 2306 title="Matchers", 2307 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2310class UnlimitedCallRatePolicy(BaseModel): 2311 class Config: 2312 extra = Extra.allow 2313 2314 type: Literal["UnlimitedCallRatePolicy"] 2315 matchers: List[HttpRequestRegexMatcher] = Field( 2316 ..., 2317 description="List of matchers that define which requests this policy applies to.", 2318 title="Matchers", 2319 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2322class DefaultErrorHandler(BaseModel): 2323 type: Literal["DefaultErrorHandler"] 2324 backoff_strategies: Optional[ 2325 List[ 2326 Union[ 2327 ConstantBackoffStrategy, 2328 ExponentialBackoffStrategy, 2329 WaitTimeFromHeader, 2330 WaitTimeFromResponse, 2331 WaitUntilTimeFromHeader, 2332 CustomBackoffStrategy, 2333 ] 2334 ] 2335 ] = Field( 2336 None, 2337 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2338 title="Backoff Strategies", 2339 ) 2340 max_retries: Optional[Union[int, str]] = Field( 2341 5, 2342 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2343 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2344 title="Max Retry Count", 2345 ) 2346 response_filters: Optional[List[HttpResponseFilter]] = Field( 2347 None, 2348 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2349 title="Response Filters", 2350 ) 2351 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitTimeFromResponse, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2354class DefaultPaginator(BaseModel): 2355 type: Literal["DefaultPaginator"] 2356 pagination_strategy: Union[ 2357 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2358 ] = Field( 2359 ..., 2360 description="Strategy defining how records are paginated.", 2361 title="Pagination Strategy", 2362 ) 2363 page_size_option: Optional[RequestOption] = Field( 2364 None, title="Inject Page Size Into Outgoing HTTP Request" 2365 ) 2366 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2367 None, 2368 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2369 title="Inject Page Token Into Outgoing HTTP Request", 2370 ) 2371 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2374class RecordExpander(BaseModel): 2375 type: Literal["RecordExpander"] 2376 expand_records_from_field: List[str] = Field( 2377 ..., 2378 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2379 examples=[ 2380 ["lines", "data"], 2381 ["items"], 2382 ["nested", "array"], 2383 ["sections", "*", "items"], 2384 ], 2385 title="Expand Records From Field", 2386 ) 2387 remain_original_record: Optional[bool] = Field( 2388 False, 2389 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2390 title="Remain Original Record", 2391 ) 2392 parent_fields: Optional[List[ParentFieldPath]] = Field( 2393 None, 2394 description="Named values to copy from the record being expanded onto each expanded item. Use this instead of `remain_original_record` when only a few parent fields are needed: it copies the named values rather than deep-copying the whole parent once per item, which matters when the parent record is large and the nested list is long. An existing value at `record_path` is overwritten, and a `parent_path` the parent does not have copies null. A copied object or array is deep-copied, so a transformation that writes inside it affects only that item. Independent of `remain_original_record`; both may be set. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2395 title="Parent Fields", 2396 ) 2397 merge_parent: Optional[bool] = Field( 2398 False, 2399 description="If true, each expanded item is the parent record shallow-merged with the item, the item's own keys winning on collision, and the expanded list removed from the parent's copy. Only the value at `expand_records_from_field` is removed, so for a multi-segment path the top-level key stays with its other fields. Each item receives its own deep copy of the merged parent fields, so a transformation that writes into a nested value affects only that item. The merge happens first, then `parent_fields` are copied, then `original_record` is embedded when `remain_original_record` is set; all three may be combined. Applies to items fetched through `truncated_list_retriever` as well as to embedded ones. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2400 title="Merge Parent", 2401 ) 2402 on_no_records: Optional[OnNoRecords] = Field( 2403 OnNoRecords.skip, 2404 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2405 title="On No Records", 2406 ) 2407 truncation_indicator_path: Optional[List[str]] = Field( 2408 None, 2409 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2410 examples=[["data", "object", "lines", "has_more"]], 2411 title="Truncation Indicator Path", 2412 ) 2413 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2414 None, 2415 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2416 title="Truncated List Retriever", 2417 ) 2418 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_fields: Optional[List[ParentFieldPath]]
on_no_records: Optional[OnNoRecords]
truncated_list_retriever: Union[SimpleRetriever, CustomRetriever, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2421class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2422 type: Literal["ApiKey"] 2423 inject_into: RequestOption = Field( 2424 ..., 2425 description="Configure how the API Key will be sent in requests to the source API.", 2426 examples=[ 2427 {"inject_into": "header", "field_name": "Authorization"}, 2428 {"inject_into": "request_parameter", "field_name": "authKey"}, 2429 ], 2430 title="Inject API Key Into Outgoing HTTP Request", 2431 ) 2432 api_token: Optional[str] = Field( 2433 "{{ session_token }}", 2434 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2435 examples=[ 2436 "{{ session_token }}", 2437 "Token {{ session_token }}", 2438 "Bearer {{ session_token }}", 2439 ], 2440 title="API Token Template", 2441 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2444class JsonSchemaPropertySelector(BaseModel): 2445 type: Literal["JsonSchemaPropertySelector"] 2446 transformations: Optional[ 2447 List[ 2448 Union[ 2449 AddFields, 2450 RemoveFields, 2451 KeysToLower, 2452 KeysToSnakeCase, 2453 FlattenFields, 2454 DpathFlattenFields, 2455 KeysReplace, 2456 CustomTransformation, 2457 ] 2458 ] 2459 ] = Field( 2460 None, 2461 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2462 title="Transformations", 2463 ) 2464 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2467class ListPartitionRouter(BaseModel): 2468 type: Literal["ListPartitionRouter"] 2469 cursor_field: str = Field( 2470 ..., 2471 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2472 examples=["section", "{{ config['section_key'] }}"], 2473 title="Current Partition Value Identifier", 2474 ) 2475 values: Union[str, List[str]] = Field( 2476 ..., 2477 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2478 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2479 title="Partition Values", 2480 ) 2481 request_option: Optional[RequestOption] = Field( 2482 None, 2483 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2484 title="Inject Partition Value Into Outgoing HTTP Request", 2485 ) 2486 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2489class PaginationReset(BaseModel): 2490 type: Literal["PaginationReset"] 2491 action: Action1 2492 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2495class GzipDecoder(BaseModel): 2496 type: Literal["GzipDecoder"] 2497 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2500class RequestBodyGraphQL(BaseModel): 2501 type: Literal["RequestBodyGraphQL"] 2502 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2505class DpathValidator(BaseModel): 2506 type: Literal["DpathValidator"] 2507 field_path: List[str] = Field( 2508 ..., 2509 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2510 examples=[ 2511 ["data"], 2512 ["data", "records"], 2513 ["data", "{{ parameters.name }}"], 2514 ["data", "*", "record"], 2515 ], 2516 title="Field Path", 2517 ) 2518 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2519 ..., 2520 description="The condition that the specified config value will be evaluated against", 2521 title="Validation Strategy", 2522 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2525class PredicateValidator(BaseModel): 2526 type: Literal["PredicateValidator"] 2527 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2528 ..., 2529 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2530 examples=[ 2531 "test-value", 2532 "{{ config['api_version'] }}", 2533 "{{ config['tenant_id'] }}", 2534 123, 2535 ], 2536 title="Value", 2537 ) 2538 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2539 ..., 2540 description="The validation strategy to apply to the value.", 2541 title="Validation Strategy", 2542 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2545class ConfigAddFields(BaseModel): 2546 type: Literal["ConfigAddFields"] 2547 fields: List[AddedFieldDefinition] = Field( 2548 ..., 2549 description="A list of transformations (path and corresponding value) that will be added to the config.", 2550 title="Fields", 2551 ) 2552 condition: Optional[str] = Field( 2553 "", 2554 description="Fields will be added if expression is evaluated to True.", 2555 examples=[ 2556 "{{ config['environemnt'] == 'sandbox' }}", 2557 "{{ property is integer }}", 2558 "{{ property|length > 5 }}", 2559 "{{ property == 'some_string_to_match' }}", 2560 ], 2561 )
fields: List[AddedFieldDefinition]
class
RateLimitedMultipleTokenAuthenticator(pydantic.v1.main.BaseModel):
2564class RateLimitedMultipleTokenAuthenticator(BaseModel): 2565 type: Literal["RateLimitedMultipleTokenAuthenticator"] 2566 tokens: Union[str, List[str]] = Field( 2567 ..., 2568 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 2569 examples=[ 2570 "{{ config['credentials']['personal_access_token'] }}", 2571 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 2572 ], 2573 title="Tokens", 2574 ) 2575 token_delimiter: Optional[str] = Field( 2576 ",", 2577 description="Delimiter used to split a single token string into multiple tokens.", 2578 title="Token Delimiter", 2579 ) 2580 auth_method: Optional[str] = Field( 2581 "Bearer", 2582 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 2583 examples=["Bearer", "token"], 2584 title="Auth Method", 2585 ) 2586 header: Optional[str] = Field( 2587 "Authorization", 2588 description="The name of the HTTP header in which to inject the token.", 2589 title="Header Name", 2590 ) 2591 quota_status_source: QuotaStatusSource = Field( 2592 ..., 2593 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 2594 title="Quota Status Source", 2595 ) 2596 quotas: List[TokenQuota] = Field( 2597 ..., 2598 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 2599 min_items=1, 2600 title="Quota Pools", 2601 ) 2602 max_wait_time: Optional[str] = Field( 2603 "PT2H", 2604 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 2605 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 2606 title="Maximum Wait Time", 2607 ) 2608 budget_reserve_fraction: Optional[float] = Field( 2609 0.1, 2610 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 2611 title="Budget Reserve Fraction", 2612 ) 2613 budget_min_reserve: Optional[int] = Field( 2614 50, 2615 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 2616 title="Budget Minimum Reserve", 2617 ) 2618 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
quota_status_source: QuotaStatusSource
quotas: List[TokenQuota]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2621class CompositeErrorHandler(BaseModel): 2622 type: Literal["CompositeErrorHandler"] 2623 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2624 Field( 2625 ..., 2626 description="List of error handlers to iterate on to determine how to handle a failed response.", 2627 title="Error Handlers", 2628 ) 2629 ) 2630 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2633class HTTPAPIBudget(BaseModel): 2634 class Config: 2635 extra = Extra.allow 2636 2637 type: Literal["HTTPAPIBudget"] 2638 policies: List[ 2639 Union[ 2640 FixedWindowCallRatePolicy, 2641 MovingWindowCallRatePolicy, 2642 UnlimitedCallRatePolicy, 2643 ] 2644 ] = Field( 2645 ..., 2646 description="List of call rate policies that define how many calls are allowed.", 2647 title="Policies", 2648 ) 2649 ratelimit_reset_header: Optional[str] = Field( 2650 "ratelimit-reset", 2651 description="The HTTP response header name that indicates when the rate limit resets.", 2652 title="Rate Limit Reset Header", 2653 ) 2654 ratelimit_remaining_header: Optional[str] = Field( 2655 "ratelimit-remaining", 2656 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2657 title="Rate Limit Remaining Header", 2658 ) 2659 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2660 [429], 2661 description="List of HTTP status codes that indicate a rate limit has been hit.", 2662 title="Status Codes for Rate Limit Hit", 2663 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2666class DpathExtractor(BaseModel): 2667 type: Literal["DpathExtractor"] 2668 field_path: List[str] = Field( 2669 ..., 2670 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2671 examples=[ 2672 ["data"], 2673 ["data", "records"], 2674 ["data", "{{ parameters.name }}"], 2675 ["data", "*", "record"], 2676 ], 2677 title="Field Path", 2678 ) 2679 record_expander: Optional[RecordExpander] = Field( 2680 None, 2681 description="Optional component to expand records by extracting items from nested array fields.", 2682 title="Record Expander", 2683 ) 2684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
CombinedExtractor(pydantic.v1.main.BaseModel):
2687class CombinedExtractor(BaseModel): 2688 type: Literal["CombinedExtractor"] 2689 extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]] = Field( 2690 ..., 2691 description="The record extractors to combine. At least one is required. Each sub-extractor is given the same HTTP response and decodes it independently, so the response is parsed once per sub-extractor. Streaming decoders (CsvDecoder, JsonlDecoder, JsonItemsDecoder, GzipDecoder, IterableDecoder) can only read the response body once and are rejected.", 2692 title="Extractors", 2693 ) 2694 mode: Optional[CombineMode] = Field( 2695 CombineMode.union, 2696 description='How the records of the sub-extractors are combined. "union" (default) yields every record of every sub-extractor, in the order the extractors are declared. "first_match" yields the records of the first sub-extractor that produces at least one record and skips the remaining ones; nothing is yielded if none of them produces a record. Note that a paginator which counts the records of a page counts the combined records: under "union" that is the sum over all sub-extractors, which overshoots the API page size, so "union" is rejected with an OffsetIncrement paginator because the offset would skip records. Under "first_match" the count is the count of the winning sub-extractor, which is usually the number of records the API returned for the page.', 2697 title="Combine Mode", 2698 ) 2699 skip_empty_records: Optional[bool] = Field( 2700 False, 2701 description='Whether to drop empty records - null, {}, [], "" - yielded by a sub-extractor before the records are combined. Off by default, so the output of a sub-extractor is passed through as it is. Turn it on when the API can return nulls in the middle of a record list, which a GraphQL API does when it answers a partial response and reports the failure in a sibling error field. Under "first_match" this also changes which sub-extractor wins: a sub-extractor whose records are all empty no longer counts as a match, so the next one is tried. The dropped records still count toward the page size of an OffsetIncrement or PageIncrement paginator, so a full page holding a null does not end pagination early.', 2702 title="Skip Empty Records", 2703 ) 2704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractors: List[Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]]
mode: Optional[CombineMode]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2707class ZipfileDecoder(BaseModel): 2708 class Config: 2709 extra = Extra.allow 2710 2711 type: Literal["ZipfileDecoder"] 2712 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2713 ..., 2714 description="Parser to parse the decompressed data from the zipfile(s).", 2715 title="Parser", 2716 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2719class RecordSelector(BaseModel): 2720 type: Literal["RecordSelector"] 2721 extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 2722 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2723 None, 2724 description="Responsible for filtering records to be emitted by the Source.", 2725 title="Record Filter", 2726 ) 2727 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2728 None, 2729 description="Responsible for normalization according to the schema.", 2730 title="Schema Normalization", 2731 ) 2732 transform_before_filtering: Optional[bool] = Field( 2733 None, 2734 description="If true, transformation will be applied before record filtering.", 2735 title="Transform Before Filtering", 2736 ) 2737 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2740class ConfigMigration(BaseModel): 2741 type: Literal["ConfigMigration"] 2742 description: Optional[str] = Field( 2743 None, description="The description/purpose of the config migration." 2744 ) 2745 transformations: List[ 2746 Union[ 2747 ConfigRemapField, 2748 ConfigAddFields, 2749 ConfigRemoveFields, 2750 CustomConfigTransformation, 2751 ] 2752 ] = Field( 2753 ..., 2754 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2755 title="Transformations", 2756 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2759class ConfigNormalizationRules(BaseModel): 2760 class Config: 2761 extra = Extra.forbid 2762 2763 type: Literal["ConfigNormalizationRules"] 2764 config_migrations: Optional[List[ConfigMigration]] = Field( 2765 [], 2766 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2767 title="Config Migrations", 2768 ) 2769 transformations: Optional[ 2770 List[ 2771 Union[ 2772 ConfigRemapField, 2773 ConfigAddFields, 2774 ConfigRemoveFields, 2775 CustomConfigTransformation, 2776 ] 2777 ] 2778 ] = Field( 2779 [], 2780 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2781 title="Transformations", 2782 ) 2783 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2784 [], 2785 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2786 title="Validations", 2787 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2790class Spec(BaseModel): 2791 type: Literal["Spec"] 2792 connection_specification: Dict[str, Any] = Field( 2793 ..., 2794 description="A connection specification describing how a the connector can be configured.", 2795 title="Connection Specification", 2796 ) 2797 documentation_url: Optional[str] = Field( 2798 None, 2799 description="URL of the connector's documentation page.", 2800 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2801 title="Documentation URL", 2802 ) 2803 advanced_auth: Optional[AuthFlow] = Field( 2804 None, 2805 description="Advanced specification for configuring the authentication flow.", 2806 title="Advanced Auth", 2807 ) 2808 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2809 None, title="Config Normalization Rules" 2810 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2813class DeclarativeSource1(BaseModel): 2814 class Config: 2815 extra = Extra.forbid 2816 2817 type: Literal["DeclarativeSource"] 2818 check: Union[CheckStream, CheckDynamicStream] 2819 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2820 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2821 version: str = Field( 2822 ..., 2823 description="The version of the Airbyte CDK used to build and test the source.", 2824 ) 2825 schemas: Optional[Schemas] = None 2826 definitions: Optional[Dict[str, Any]] = None 2827 spec: Optional[Spec] = None 2828 concurrency_level: Optional[ConcurrencyLevel] = None 2829 api_budget: Optional[HTTPAPIBudget] = None 2830 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2831 None, 2832 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2833 title="Stream Groups", 2834 ) 2835 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2836 None, 2837 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2838 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2839 title="Maximum Concurrent Asynchronous Jobs", 2840 ) 2841 metadata: Optional[Dict[str, Any]] = Field( 2842 None, 2843 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2844 ) 2845 description: Optional[str] = Field( 2846 None, 2847 description="A description of the connector. It will be presented on the Source documentation page.", 2848 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2851class DeclarativeSource2(BaseModel): 2852 class Config: 2853 extra = Extra.forbid 2854 2855 type: Literal["DeclarativeSource"] 2856 check: Union[CheckStream, CheckDynamicStream] 2857 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2858 None 2859 ) 2860 dynamic_streams: List[DynamicDeclarativeStream] 2861 version: str = Field( 2862 ..., 2863 description="The version of the Airbyte CDK used to build and test the source.", 2864 ) 2865 schemas: Optional[Schemas] = None 2866 definitions: Optional[Dict[str, Any]] = None 2867 spec: Optional[Spec] = None 2868 concurrency_level: Optional[ConcurrencyLevel] = None 2869 api_budget: Optional[HTTPAPIBudget] = None 2870 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2871 None, 2872 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", 2873 title="Stream Groups", 2874 ) 2875 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2876 None, 2877 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2878 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2879 title="Maximum Concurrent Asynchronous Jobs", 2880 ) 2881 metadata: Optional[Dict[str, Any]] = Field( 2882 None, 2883 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2884 ) 2885 description: Optional[str] = Field( 2886 None, 2887 description="A description of the connector. It will be presented on the Source documentation page.", 2888 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2891class DeclarativeSource(BaseModel): 2892 class Config: 2893 extra = Extra.forbid 2894 2895 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2896 ..., 2897 description="An API source that extracts data according to its declarative components.", 2898 title="DeclarativeSource", 2899 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2902class SelectiveAuthenticator(BaseModel): 2903 class Config: 2904 extra = Extra.allow 2905 2906 type: Literal["SelectiveAuthenticator"] 2907 authenticator_selection_path: List[str] = Field( 2908 ..., 2909 description="Path of the field in config with selected authenticator name", 2910 examples=[["auth"], ["auth", "type"]], 2911 title="Authenticator Selection Path", 2912 ) 2913 authenticators: Dict[ 2914 str, 2915 Union[ 2916 ApiKeyAuthenticator, 2917 BasicHttpAuthenticator, 2918 BearerAuthenticator, 2919 OAuthAuthenticator, 2920 JwtAuthenticator, 2921 SessionTokenAuthenticator, 2922 LegacySessionTokenAuthenticator, 2923 CustomAuthenticator, 2924 NoAuth, 2925 RateLimitedMultipleTokenAuthenticator, 2926 ], 2927 ] = Field( 2928 ..., 2929 description="Authenticators to select from.", 2930 examples=[ 2931 { 2932 "authenticators": { 2933 "token": "#/definitions/ApiKeyAuthenticator", 2934 "oauth": "#/definitions/OAuthAuthenticator", 2935 "jwt": "#/definitions/JwtAuthenticator", 2936 } 2937 } 2938 ], 2939 title="Authenticators", 2940 ) 2941 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2944class ConditionalStreams(BaseModel): 2945 type: Literal["ConditionalStreams"] 2946 condition: str = Field( 2947 ..., 2948 description="Condition that will be evaluated to determine if a set of streams should be available.", 2949 examples=["{{ config['is_sandbox'] }}"], 2950 title="Condition", 2951 ) 2952 streams: List[DeclarativeStream] = Field( 2953 ..., 2954 description="Streams that will be used during an operation based on the condition.", 2955 title="Streams", 2956 ) 2957 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2960class FileUploader(BaseModel): 2961 type: Literal["FileUploader"] 2962 requester: Union[HttpRequester, CustomRequester] = Field( 2963 ..., 2964 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2965 ) 2966 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = ( 2967 Field( 2968 ..., 2969 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2970 ) 2971 ) 2972 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2973 None, 2974 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2975 ) 2976 filename_extractor: Optional[str] = Field( 2977 None, 2978 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2979 examples=[ 2980 "{{ record.id }}/{{ record.file_name }}/", 2981 "{{ record.id }}_{{ record.file_name }}/", 2982 ], 2983 ) 2984 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
SyncTokenCursor(pydantic.v1.main.BaseModel):
2987class SyncTokenCursor(BaseModel): 2988 class Config: 2989 extra = Extra.allow 2990 2991 type: Literal["SyncTokenCursor"] 2992 token_path: List[str] = Field( 2993 ..., 2994 description="Path to the next token in the decoded response that ends a pagination pass. The value must be a string or an integer, and is stored and sent as a string. A missing, null or empty value means the response carried no token.", 2995 examples=[["syncToken"], ["nextSyncToken"], ["meta", "sync_token"]], 2996 title="Token Path", 2997 ) 2998 token_option: RequestOption = Field( 2999 ..., 3000 description="Where to send the stored token on each request of a pass: the JSON body, the query string, a header or the form body.", 3001 title="Send Token In", 3002 ) 3003 completion_condition: Optional[str] = Field( 3004 None, 3005 description="Must be true on the last response of a pass for its token to be committed. Defaults to a token being present at token_path. Set it when the API repeats the old token on intermediate pages or returns a position on every page.", 3006 examples=["{{ not response.moreDataAvailable }}"], 3007 title="Completion Condition", 3008 ) 3009 initial_token: Optional[str] = Field( 3010 None, 3011 description='Value to send when no token is stored, for an API that needs one to start a full pass, such as Todoist\'s "*". When unset, nothing is sent.', 3012 examples=["*"], 3013 title="Initial Token", 3014 ) 3015 cursor_field: Optional[str] = Field( 3016 None, 3017 description="Optional top-level record field advertised as the stream's cursor, so destinations can order versions of a record when deduplicating. It is never used to filter records or to build state.", 3018 title="Destination Cursor Field", 3019 ) 3020 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
token_option: RequestOption
class
SyncTokenCursor.Config:
class
DeclarativeStream(pydantic.v1.main.BaseModel):
3023class DeclarativeStream(BaseModel): 3024 class Config: 3025 extra = Extra.allow 3026 3027 type: Literal["DeclarativeStream"] 3028 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 3029 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3030 ..., 3031 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3032 title="Retriever", 3033 ) 3034 incremental_sync: Optional[ 3035 Union[DatetimeBasedCursor, IncrementingCountCursor, SyncTokenCursor] 3036 ] = Field( 3037 None, 3038 description="Component used to fetch data incrementally based on a time field in the data.", 3039 title="Incremental Sync", 3040 ) 3041 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 3042 schema_loader: Optional[ 3043 Union[ 3044 InlineSchemaLoader, 3045 DynamicSchemaLoader, 3046 JsonFileSchemaLoader, 3047 List[ 3048 Union[ 3049 InlineSchemaLoader, 3050 DynamicSchemaLoader, 3051 JsonFileSchemaLoader, 3052 CustomSchemaLoader, 3053 ] 3054 ], 3055 CustomSchemaLoader, 3056 ] 3057 ] = Field( 3058 None, 3059 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 3060 title="Schema Loader", 3061 ) 3062 transformations: Optional[ 3063 List[ 3064 Union[ 3065 AddFields, 3066 RemoveFields, 3067 KeysToLower, 3068 KeysToSnakeCase, 3069 FlattenFields, 3070 DpathFlattenFields, 3071 KeysReplace, 3072 CustomTransformation, 3073 ] 3074 ] 3075 ] = Field( 3076 None, 3077 description="A list of transformations to be applied to each output record.", 3078 title="Transformations", 3079 ) 3080 state_migrations: Optional[ 3081 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 3082 ] = Field( 3083 [], 3084 description="Array of state migrations to be applied on the input state", 3085 title="State Migrations", 3086 ) 3087 file_uploader: Optional[FileUploader] = Field( 3088 None, 3089 description="(experimental) Describes how to fetch a file", 3090 title="File Uploader", 3091 ) 3092 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, SyncTokenCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
3095class SessionTokenAuthenticator(BaseModel): 3096 type: Literal["SessionTokenAuthenticator"] 3097 login_requester: HttpRequester = Field( 3098 ..., 3099 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 3100 examples=[ 3101 { 3102 "type": "HttpRequester", 3103 "url_base": "https://my_api.com", 3104 "path": "/login", 3105 "authenticator": { 3106 "type": "BasicHttpAuthenticator", 3107 "username": "{{ config.username }}", 3108 "password": "{{ config.password }}", 3109 }, 3110 } 3111 ], 3112 title="Login Requester", 3113 ) 3114 session_token_path: List[str] = Field( 3115 ..., 3116 description="The path in the response body returned from the login requester to the session token.", 3117 examples=[["access_token"], ["result", "token"]], 3118 title="Session Token Path", 3119 ) 3120 expiration_duration: Optional[str] = Field( 3121 None, 3122 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 3123 examples=["PT1H", "P1D"], 3124 title="Expiration Duration", 3125 ) 3126 request_authentication: Union[ 3127 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 3128 ] = Field( 3129 ..., 3130 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 3131 title="Data Request Authentication", 3132 ) 3133 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 3134 None, description="Component used to decode the response.", title="Decoder" 3135 ) 3136 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
3139class HttpRequester(BaseModelWithDeprecations): 3140 type: Literal["HttpRequester"] 3141 url_base: Optional[str] = Field( 3142 None, 3143 deprecated=True, 3144 deprecation_message="Use `url` field instead.", 3145 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3146 examples=[ 3147 "https://connect.squareup.com/v2", 3148 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 3149 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3150 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3151 ], 3152 title="API Base URL", 3153 ) 3154 url: Optional[str] = Field( 3155 None, 3156 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3157 examples=[ 3158 "https://connect.squareup.com/v2", 3159 "{{ config['url'] or 'https://app.posthog.com'}}/api", 3160 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3161 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 3162 ], 3163 title="API Endpoint URL", 3164 ) 3165 path: Optional[str] = Field( 3166 None, 3167 deprecated=True, 3168 deprecation_message="Use `url` field instead.", 3169 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 3170 examples=[ 3171 "/products", 3172 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 3173 "/trades/{{ config['symbol_id'] }}/history", 3174 ], 3175 title="URL Path", 3176 ) 3177 http_method: Optional[HttpMethod] = Field( 3178 HttpMethod.GET, 3179 description="The HTTP method used to fetch data from the source (can be GET or POST).", 3180 examples=["GET", "POST"], 3181 title="HTTP Method", 3182 ) 3183 authenticator: Optional[ 3184 Union[ 3185 ApiKeyAuthenticator, 3186 BasicHttpAuthenticator, 3187 BearerAuthenticator, 3188 OAuthAuthenticator, 3189 JwtAuthenticator, 3190 SessionTokenAuthenticator, 3191 SelectiveAuthenticator, 3192 CustomAuthenticator, 3193 NoAuth, 3194 LegacySessionTokenAuthenticator, 3195 RateLimitedMultipleTokenAuthenticator, 3196 ] 3197 ] = Field( 3198 None, 3199 description="Authentication method to use for requests sent to the API.", 3200 title="Authenticator", 3201 ) 3202 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 3203 None, 3204 deprecated=True, 3205 deprecation_message="Use `query_properties` field instead.", 3206 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 3207 title="Fetch Properties from Endpoint", 3208 ) 3209 query_properties: Optional[QueryProperties] = Field( 3210 None, 3211 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 3212 title="Query Properties", 3213 ) 3214 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 3215 None, 3216 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3217 examples=[ 3218 {"unit": "day"}, 3219 { 3220 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3221 }, 3222 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3223 {"sort_by[asc]": "updated_at"}, 3224 ], 3225 title="Query Parameters", 3226 ) 3227 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3228 None, 3229 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3230 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3231 title="Request Headers", 3232 ) 3233 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3234 None, 3235 deprecated=True, 3236 deprecation_message="Use `request_body` field instead.", 3237 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3238 examples=[ 3239 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3240 ], 3241 title="Request Body Payload (Non-JSON)", 3242 ) 3243 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3244 None, 3245 deprecated=True, 3246 deprecation_message="Use `request_body` field instead.", 3247 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3248 examples=[ 3249 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3250 {"key": "{{ config['value'] }}"}, 3251 {"sort": {"field": "updated_at", "order": "ascending"}}, 3252 ], 3253 title="Request Body JSON Payload", 3254 ) 3255 request_body: Optional[ 3256 Union[ 3257 RequestBodyPlainText, 3258 RequestBodyUrlEncodedForm, 3259 RequestBodyJsonObject, 3260 RequestBodyGraphQL, 3261 ] 3262 ] = Field( 3263 None, 3264 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3265 title="Request Body", 3266 ) 3267 error_handler: Optional[ 3268 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3269 ] = Field( 3270 None, 3271 description="Error handler component that defines how to handle errors.", 3272 title="Error Handler", 3273 ) 3274 use_cache: Optional[bool] = Field( 3275 False, 3276 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3277 title="Use Cache", 3278 ) 3279 connect_timeout_in_seconds: Optional[float] = Field( 3280 None, 3281 description="Maximum time to establish the connection (TCP + TLS handshake). When unset, no connect timeout is applied.", 3282 examples=[30], 3283 gt=0, 3284 title="Connect Timeout (seconds)", 3285 ) 3286 read_timeout_in_seconds: Optional[float] = Field( 3287 None, 3288 description="The maximum time to wait for the server to send any data (the first byte of the response, or the next chunk while the body is read). It is NOT a limit on the total request duration. When unset, no read timeout is applied. Only set it when every endpoint served by this requester responds well within the value; synchronous report or export endpoints can stay silent for minutes. Each timed-out attempt counts against the error handler's retry window (600 s for manifest connectors), so a value near 300 allows only a few retries.", 3289 examples=[300], 3290 gt=0, 3291 title="Read Timeout (seconds)", 3292 ) 3293 use_tcp_keepalive: Optional[bool] = Field( 3294 False, 3295 description="Enables TCP keepalive (60 s idle, 10 s probe interval, 6 probes) on this requester's connections, so dead or NAT-dropped connections are detected and fail instead of hanging forever. Does not fail slow but alive servers; use read_timeout_in_seconds to bound those.", 3296 examples=[True], 3297 title="Use TCP Keepalive", 3298 ) 3299 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, RateLimitedMultipleTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
3302class DynamicSchemaLoader(BaseModel): 3303 type: Literal["DynamicSchemaLoader"] 3304 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3305 ..., 3306 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3307 title="Retriever", 3308 ) 3309 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3310 None, 3311 description="Responsible for filtering fields to be added to json schema.", 3312 title="Schema Filter", 3313 ) 3314 schema_transformations: Optional[ 3315 List[ 3316 Union[ 3317 AddFields, 3318 RemoveFields, 3319 KeysToLower, 3320 KeysToSnakeCase, 3321 FlattenFields, 3322 DpathFlattenFields, 3323 KeysReplace, 3324 CustomTransformation, 3325 ] 3326 ] 3327 ] = Field( 3328 None, 3329 description="A list of transformations to be applied to the schema.", 3330 title="Schema Transformations", 3331 ) 3332 schema_type_identifier: SchemaTypeIdentifier 3333 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
3336class ParentStreamConfig(BaseModel): 3337 type: Literal["ParentStreamConfig"] 3338 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3339 ..., description="Reference to the parent stream.", title="Parent Stream" 3340 ) 3341 parent_key: str = Field( 3342 ..., 3343 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3344 examples=["id", "{{ config['parent_record_id'] }}"], 3345 title="Parent Key", 3346 ) 3347 partition_field: str = Field( 3348 ..., 3349 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3350 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3351 title="Current Parent Key Value Identifier", 3352 ) 3353 request_option: Optional[RequestOption] = Field( 3354 None, 3355 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3356 title="Request Option", 3357 ) 3358 incremental_dependency: Optional[bool] = Field( 3359 False, 3360 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3361 title="Incremental Dependency", 3362 ) 3363 lazy_read_pointer: Optional[List[str]] = Field( 3364 [], 3365 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3366 title="Lazy Read Pointer", 3367 ) 3368 extra_fields: Optional[List[List[str]]] = Field( 3369 None, 3370 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3371 title="Extra Fields", 3372 ) 3373 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
3376class PropertiesFromEndpoint(BaseModel): 3377 type: Literal["PropertiesFromEndpoint"] 3378 property_field_path: List[str] = Field( 3379 ..., 3380 description="Describes the path to the field that should be extracted", 3381 examples=[["name"]], 3382 ) 3383 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3384 ..., 3385 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3386 ) 3387 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
3390class QueryProperties(BaseModel): 3391 type: Literal["QueryProperties"] 3392 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3393 ..., 3394 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3395 title="Property List", 3396 ) 3397 always_include_properties: Optional[List[str]] = Field( 3398 None, 3399 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3400 title="Always Include Properties", 3401 ) 3402 property_chunking: Optional[PropertyChunking] = Field( 3403 None, 3404 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3405 title="Property Chunking", 3406 ) 3407 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3408 None, 3409 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3410 title="Property Selector", 3411 ) 3412 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3415class StateDelegatingStream(BaseModel): 3416 type: Literal["StateDelegatingStream"] 3417 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3418 full_refresh_stream: DeclarativeStream = Field( 3419 ..., 3420 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3421 title="Full Refresh Stream", 3422 ) 3423 incremental_stream: DeclarativeStream = Field( 3424 ..., 3425 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3426 title="Incremental Stream", 3427 ) 3428 api_retention_period: Optional[str] = Field( 3429 None, 3430 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", 3431 examples=["P30D", "P90D", "P1Y"], 3432 title="API Retention Period", 3433 ) 3434 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3437class SimpleRetriever(BaseModel): 3438 type: Literal["SimpleRetriever"] 3439 requester: Union[HttpRequester, CustomRequester] = Field( 3440 ..., 3441 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3442 ) 3443 decoder: Optional[ 3444 Union[ 3445 JsonDecoder, 3446 JsonItemsDecoder, 3447 XmlDecoder, 3448 CsvDecoder, 3449 JsonlDecoder, 3450 GzipDecoder, 3451 IterableDecoder, 3452 ZipfileDecoder, 3453 CustomDecoder, 3454 ] 3455 ] = Field( 3456 None, 3457 description="Component decoding the response so records can be extracted.", 3458 title="HTTP Response Format", 3459 ) 3460 record_selector: RecordSelector = Field( 3461 ..., 3462 description="Component that describes how to extract records from a HTTP response.", 3463 ) 3464 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3465 None, 3466 description="Paginator component that describes how to navigate through the API's pages.", 3467 ) 3468 pagination_reset: Optional[PaginationReset] = Field( 3469 None, 3470 description="Describes what triggers pagination reset and how to handle it.", 3471 ) 3472 page_size_reduction: Optional[PageSizeReduction] = Field( 3473 None, 3474 description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", 3475 ) 3476 request_window_splitting: Optional[RequestWindowSplitting] = Field( 3477 None, 3478 description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page size, and applies even when the API rejects the request before returning a single page.", 3479 ) 3480 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3481 False, 3482 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3483 ) 3484 partition_router: Optional[ 3485 Union[ 3486 SubstreamPartitionRouter, 3487 ListPartitionRouter, 3488 GroupingPartitionRouter, 3489 UnionPartitionRouter, 3490 CustomPartitionRouter, 3491 List[ 3492 Union[ 3493 SubstreamPartitionRouter, 3494 ListPartitionRouter, 3495 GroupingPartitionRouter, 3496 UnionPartitionRouter, 3497 CustomPartitionRouter, 3498 ] 3499 ], 3500 ] 3501 ] = Field( 3502 None, 3503 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3504 title="Partition Router", 3505 ) 3506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
page_size_reduction: Optional[PageSizeReduction]
request_window_splitting: Optional[RequestWindowSplitting]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3509class AsyncRetriever(BaseModel): 3510 type: Literal["AsyncRetriever"] 3511 record_selector: RecordSelector = Field( 3512 ..., 3513 description="Component that describes how to extract records from a HTTP response.", 3514 ) 3515 status_mapping: AsyncJobStatusMap = Field( 3516 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3517 ) 3518 status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] = Field( 3519 ..., description="Responsible for fetching the actual status of the async job." 3520 ) 3521 download_target_extractor: Optional[ 3522 Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor] 3523 ] = Field( 3524 None, 3525 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3526 ) 3527 download_extractor: Optional[ 3528 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor] 3529 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3530 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3531 ..., 3532 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3533 ) 3534 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3535 ..., 3536 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3537 ) 3538 polling_job_timeout: Optional[Union[int, str]] = Field( 3539 None, 3540 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3541 ) 3542 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3543 None, 3544 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3545 ge=1, 3546 ) 3547 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3548 None, 3549 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3550 ) 3551 download_requester: Union[HttpRequester, CustomRequester] = Field( 3552 ..., 3553 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3554 ) 3555 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3556 None, 3557 description="Paginator component that describes how to navigate through the API's pages during download.", 3558 ) 3559 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3560 None, 3561 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3562 ) 3563 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3564 None, 3565 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3566 ) 3567 partition_router: Optional[ 3568 Union[ 3569 ListPartitionRouter, 3570 SubstreamPartitionRouter, 3571 GroupingPartitionRouter, 3572 UnionPartitionRouter, 3573 CustomPartitionRouter, 3574 List[ 3575 Union[ 3576 ListPartitionRouter, 3577 SubstreamPartitionRouter, 3578 GroupingPartitionRouter, 3579 UnionPartitionRouter, 3580 CustomPartitionRouter, 3581 ] 3582 ], 3583 ] 3584 ] = Field( 3585 [], 3586 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3587 title="Partition Router", 3588 ) 3589 decoder: Optional[ 3590 Union[ 3591 CsvDecoder, 3592 GzipDecoder, 3593 JsonDecoder, 3594 JsonItemsDecoder, 3595 JsonlDecoder, 3596 IterableDecoder, 3597 XmlDecoder, 3598 ZipfileDecoder, 3599 CustomDecoder, 3600 ] 3601 ] = Field( 3602 None, 3603 description="Component decoding the response so records can be extracted.", 3604 title="HTTP Response Format", 3605 ) 3606 download_decoder: Optional[ 3607 Union[ 3608 CsvDecoder, 3609 GzipDecoder, 3610 JsonDecoder, 3611 JsonItemsDecoder, 3612 JsonlDecoder, 3613 IterableDecoder, 3614 XmlDecoder, 3615 ZipfileDecoder, 3616 CustomDecoder, 3617 ] 3618 ] = Field( 3619 None, 3620 description="Component decoding the download response so records can be extracted.", 3621 title="Download HTTP Response Format", 3622 ) 3623 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, CombinedExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, CombinedExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
StreamGroup(pydantic.v1.main.BaseModel):
3626class StreamGroup(BaseModel): 3627 streams: List[DeclarativeStream] = Field( 3628 ..., 3629 description="List of references to streams that belong to this group.\n", 3630 title="Streams", 3631 ) 3632 action: BlockSimultaneousSyncsAction = Field( 3633 ..., description="The action to apply to streams in this group.", title="Action" 3634 )
streams: List[DeclarativeStream]
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3637class SubstreamPartitionRouter(BaseModel): 3638 type: Literal["SubstreamPartitionRouter"] 3639 parent_stream_configs: List[ParentStreamConfig] = Field( 3640 ..., 3641 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3642 title="Parent Stream Configs", 3643 ) 3644 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3647class GroupingPartitionRouter(BaseModel): 3648 type: Literal["GroupingPartitionRouter"] 3649 group_size: int = Field( 3650 ..., 3651 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3652 examples=[10, 50], 3653 title="Group Size", 3654 ) 3655 underlying_partition_router: Union[ 3656 ListPartitionRouter, 3657 SubstreamPartitionRouter, 3658 UnionPartitionRouter, 3659 CustomPartitionRouter, 3660 ] = Field( 3661 ..., 3662 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3663 title="Underlying Partition Router", 3664 ) 3665 deduplicate: Optional[bool] = Field( 3666 True, 3667 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3668 title="Deduplicate Partitions", 3669 ) 3670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]
class
UnionPartitionRouter(pydantic.v1.main.BaseModel):
3673class UnionPartitionRouter(BaseModel): 3674 type: Literal["UnionPartitionRouter"] 3675 partition_field: str = Field( 3676 ..., 3677 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3678 examples=["repository", "{{ config['partition_field'] }}"], 3679 title="Partition Field", 3680 ) 3681 partition_routers: List[ 3682 Union[ 3683 ListPartitionRouter, 3684 SubstreamPartitionRouter, 3685 UnionPartitionRouter, 3686 CustomPartitionRouter, 3687 ] 3688 ] = Field( 3689 ..., 3690 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3691 title="Partition Routers", 3692 ) 3693 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
partition_routers: List[Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3696class HttpComponentsResolver(BaseModel): 3697 type: Literal["HttpComponentsResolver"] 3698 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3699 ..., 3700 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3701 title="Retriever", 3702 ) 3703 components_mapping: List[ComponentMappingDefinition] 3704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3707class DynamicDeclarativeStream(BaseModel): 3708 type: Literal["DynamicDeclarativeStream"] 3709 name: Optional[str] = Field( 3710 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3711 ) 3712 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3713 ..., description="Reference to the stream template.", title="Stream Template" 3714 ) 3715 components_resolver: Union[ 3716 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3717 ] = Field( 3718 ..., 3719 description="Component resolve and populates stream templates with components values.", 3720 title="Components Resolver", 3721 ) 3722 use_parent_parameters: Optional[bool] = Field( 3723 True, 3724 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3725 title="Use Parent Parameters", 3726 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]