airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 config_overrides: Optional[Dict[str, Any]] = Field( 81 None, 82 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 83 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 84 title="Config Overrides", 85 ) 86 87 88class ConcurrencyLevel(BaseModel): 89 type: Optional[Literal["ConcurrencyLevel"]] = None 90 default_concurrency: Union[int, str] = Field( 91 ..., 92 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 93 examples=[10, "{{ config['num_workers'] or 10 }}"], 94 title="Default Concurrency", 95 ) 96 max_concurrency: Optional[int] = Field( 97 None, 98 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 99 examples=[20, 100], 100 title="Max Concurrency", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class ConstantBackoffStrategy(BaseModel): 106 type: Literal["ConstantBackoffStrategy"] 107 backoff_time_in_seconds: Union[float, str] = Field( 108 ..., 109 description="Backoff time in seconds.", 110 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 111 title="Backoff Time", 112 ) 113 jitter_range_in_seconds: Optional[float] = Field( 114 None, 115 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 116 examples=[15], 117 ge=0, 118 title="Jitter Range", 119 ) 120 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 121 122 123class CursorPagination(BaseModel): 124 type: Literal["CursorPagination"] 125 cursor_value: str = Field( 126 ..., 127 description="Value of the cursor defining the next page to fetch.", 128 examples=[ 129 "{{ headers.link.next.cursor }}", 130 "{{ last_record['key'] }}", 131 "{{ response['nextPage'] }}", 132 ], 133 title="Cursor Value", 134 ) 135 page_size: Optional[Union[int, str]] = Field( 136 None, 137 description="The number of records to include in each pages.", 138 examples=[100, "{{ config['page_size'] }}"], 139 title="Page Size", 140 ) 141 stop_condition: Optional[str] = Field( 142 None, 143 description="Template string evaluating when to stop paginating.", 144 examples=[ 145 "{{ response.data.has_more is false }}", 146 "{{ 'next' not in headers['link'] }}", 147 ], 148 title="Stop Condition", 149 ) 150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 151 152 153class CustomAuthenticator(BaseModel): 154 class Config: 155 extra = Extra.allow 156 157 type: Literal["CustomAuthenticator"] 158 class_name: str = Field( 159 ..., 160 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 161 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 162 title="Class Name", 163 ) 164 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 165 166 167class CustomBackoffStrategy(BaseModel): 168 class Config: 169 extra = Extra.allow 170 171 type: Literal["CustomBackoffStrategy"] 172 class_name: str = Field( 173 ..., 174 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 175 examples=["source_railz.components.MyCustomBackoffStrategy"], 176 title="Class Name", 177 ) 178 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 179 180 181class CustomErrorHandler(BaseModel): 182 class Config: 183 extra = Extra.allow 184 185 type: Literal["CustomErrorHandler"] 186 class_name: str = Field( 187 ..., 188 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 189 examples=["source_railz.components.MyCustomErrorHandler"], 190 title="Class Name", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomPaginationStrategy(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomPaginationStrategy"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomPaginationStrategy"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordExtractor(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordExtractor"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomRecordExtractor"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRecordFilter(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRecordFilter"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomCustomRecordFilter"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRequester(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRequester"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRecordExtractor"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomRetriever(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomRetriever"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomRetriever"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomPartitionRouter(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomPartitionRouter"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomPartitionRouter"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaLoader(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaLoader"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 287 examples=["source_railz.components.MyCustomSchemaLoader"], 288 title="Class Name", 289 ) 290 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 291 292 293class CustomSchemaNormalization(BaseModel): 294 class Config: 295 extra = Extra.allow 296 297 type: Literal["CustomSchemaNormalization"] 298 class_name: str = Field( 299 ..., 300 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 301 examples=[ 302 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 303 ], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomStateMigration(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomStateMigration"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomStateMigration"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class CustomTransformation(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Literal["CustomTransformation"] 328 class_name: str = Field( 329 ..., 330 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 331 examples=["source_railz.components.MyCustomTransformation"], 332 title="Class Name", 333 ) 334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 335 336 337class LegacyToPerPartitionStateMigration(BaseModel): 338 class Config: 339 extra = Extra.allow 340 341 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 342 343 344class Clamping(BaseModel): 345 target: str = Field( 346 ..., 347 description="The period of time that datetime windows will be clamped by", 348 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 349 title="Target", 350 ) 351 target_details: Optional[Dict[str, Any]] = None 352 353 354class Algorithm(Enum): 355 HS256 = "HS256" 356 HS384 = "HS384" 357 HS512 = "HS512" 358 ES256 = "ES256" 359 ES256K = "ES256K" 360 ES384 = "ES384" 361 ES512 = "ES512" 362 RS256 = "RS256" 363 RS384 = "RS384" 364 RS512 = "RS512" 365 PS256 = "PS256" 366 PS384 = "PS384" 367 PS512 = "PS512" 368 EdDSA = "EdDSA" 369 370 371class JwtHeaders(BaseModel): 372 class Config: 373 extra = Extra.forbid 374 375 kid: Optional[str] = Field( 376 None, 377 description="Private key ID for user account.", 378 examples=["{{ config['kid'] }}"], 379 title="Key Identifier", 380 ) 381 typ: Optional[str] = Field( 382 "JWT", 383 description="The media type of the complete JWT.", 384 examples=["JWT"], 385 title="Type", 386 ) 387 cty: Optional[str] = Field( 388 None, 389 description="Content type of JWT header.", 390 examples=["JWT"], 391 title="Content Type", 392 ) 393 394 395class JwtPayload(BaseModel): 396 class Config: 397 extra = Extra.forbid 398 399 iss: Optional[str] = Field( 400 None, 401 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 402 examples=["{{ config['iss'] }}"], 403 title="Issuer", 404 ) 405 sub: Optional[str] = Field( 406 None, 407 description="The subject of the JWT. Commonly defined by the API.", 408 title="Subject", 409 ) 410 aud: Optional[str] = Field( 411 None, 412 description="The recipient that the JWT is intended for. Commonly defined by the API.", 413 examples=["appstoreconnect-v1"], 414 title="Audience", 415 ) 416 417 418class RefreshTokenUpdater(BaseModel): 419 refresh_token_name: Optional[str] = Field( 420 "refresh_token", 421 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 422 examples=["refresh_token"], 423 title="Refresh Token Property Name", 424 ) 425 access_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "access_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "access_token"], ["access_token"]], 429 title="Config Path To Access Token", 430 ) 431 refresh_token_config_path: Optional[List[str]] = Field( 432 ["credentials", "refresh_token"], 433 description="Config path to the access token. Make sure the field actually exists in the config.", 434 examples=[["credentials", "refresh_token"], ["refresh_token"]], 435 title="Config Path To Refresh Token", 436 ) 437 token_expiry_date_config_path: Optional[List[str]] = Field( 438 ["credentials", "token_expiry_date"], 439 description="Config path to the expiry date. Make sure actually exists in the config.", 440 examples=[["credentials", "token_expiry_date"]], 441 title="Config Path To Expiry Date", 442 ) 443 refresh_token_error_status_codes: Optional[List[int]] = Field( 444 [], 445 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 446 examples=[[400, 500]], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 448 ) 449 refresh_token_error_key: Optional[str] = Field( 450 "", 451 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 452 examples=["error"], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 454 ) 455 refresh_token_error_values: Optional[List[str]] = Field( 456 [], 457 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 458 examples=[["invalid_grant", "invalid_permissions"]], 459 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 460 ) 461 462 463class Rate(BaseModel): 464 class Config: 465 extra = Extra.allow 466 467 limit: Union[int, str] = Field( 468 ..., 469 description="The maximum number of calls allowed within the interval.", 470 title="Limit", 471 ) 472 interval: str = Field( 473 ..., 474 description="The time interval for the rate limit.", 475 examples=["PT1H", "P1D"], 476 title="Interval", 477 ) 478 479 480class HttpRequestRegexMatcher(BaseModel): 481 class Config: 482 extra = Extra.allow 483 484 method: Optional[str] = Field( 485 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 486 ) 487 url_base: Optional[str] = Field( 488 None, 489 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 490 title="URL Base", 491 ) 492 url_path_pattern: Optional[str] = Field( 493 None, 494 description="A regular expression pattern to match the URL path.", 495 title="URL Path Pattern", 496 ) 497 params: Optional[Dict[str, Any]] = Field( 498 None, description="The query parameters to match.", title="Parameters" 499 ) 500 headers: Optional[Dict[str, Any]] = Field( 501 None, description="The headers to match.", title="Headers" 502 ) 503 weight: Optional[Union[int, str]] = Field( 504 None, 505 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 506 title="Weight", 507 ) 508 509 510class ResponseToFileExtractor(BaseModel): 511 type: Literal["ResponseToFileExtractor"] 512 preserve_na_values: Optional[bool] = Field( 513 False, 514 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 515 title="Preserve NA Values", 516 ) 517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 518 519 520class OnNoRecords(Enum): 521 skip = "skip" 522 emit_parent = "emit_parent" 523 524 525class ExponentialBackoffStrategy(BaseModel): 526 type: Literal["ExponentialBackoffStrategy"] 527 factor: Optional[Union[float, str]] = Field( 528 5, 529 description="Multiplicative constant applied on each retry.", 530 examples=[5, 5.5, "10"], 531 title="Factor", 532 ) 533 jitter_range_in_seconds: Optional[float] = Field( 534 None, 535 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 536 examples=[2], 537 ge=0, 538 title="Jitter Range", 539 ) 540 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 541 542 543class GroupByKeyMergeStrategy(BaseModel): 544 type: Literal["GroupByKeyMergeStrategy"] 545 key: Union[str, List[str]] = Field( 546 ..., 547 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 548 examples=["id", ["parent_id", "end_date"]], 549 title="Key", 550 ) 551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 552 553 554class SessionTokenRequestBearerAuthenticator(BaseModel): 555 type: Literal["Bearer"] 556 557 558class HttpMethod(Enum): 559 GET = "GET" 560 POST = "POST" 561 562 563class QuotaStatusSource(BaseModel): 564 type: Literal["QuotaStatusSource"] 565 url: str = Field( 566 ..., 567 description="The full URL of the quota status endpoint.", 568 examples=[ 569 "https://api.github.com/rate_limit", 570 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 571 ], 572 title="URL", 573 ) 574 http_method: Optional[HttpMethod] = Field( 575 HttpMethod.GET, 576 description="The HTTP method used to fetch the quota status.", 577 title="HTTP Method", 578 ) 579 request_headers: Optional[Dict[str, str]] = Field( 580 None, 581 description="Additional headers to send with the quota status request.", 582 title="Request Headers", 583 ) 584 unavailable_status_codes: Optional[List[int]] = Field( 585 None, 586 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 587 examples=[[404]], 588 title="Unavailable Status Codes", 589 unique_items=True, 590 ) 591 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 592 593 594class TokenQuota(BaseModel): 595 type: Literal["TokenQuota"] 596 name: str = Field( 597 ..., 598 description="Name of the quota pool.", 599 examples=["rest", "graphql"], 600 title="Name", 601 ) 602 remaining_path: List[str] = Field( 603 ..., 604 description="Path to the remaining call count for this pool in the quota status response.", 605 examples=[["resources", "core", "remaining"]], 606 title="Remaining Path", 607 ) 608 reset_path: List[str] = Field( 609 ..., 610 description="Path to the quota reset timestamp for this pool in the quota status response.", 611 examples=[["resources", "core", "reset"]], 612 title="Reset Path", 613 ) 614 limit_path: Optional[List[str]] = Field( 615 None, 616 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 617 examples=[["resources", "core", "limit"]], 618 title="Limit Path", 619 ) 620 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 621 None, 622 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 623 title="Matchers", 624 ) 625 remaining_header: Optional[str] = Field( 626 None, 627 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 628 examples=["X-RateLimit-Remaining"], 629 title="Remaining Header", 630 ) 631 reset_header: Optional[str] = Field( 632 None, 633 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 634 examples=["X-RateLimit-Reset"], 635 title="Reset Header", 636 ) 637 limit_header: Optional[str] = Field( 638 None, 639 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 640 examples=["X-RateLimit-Limit"], 641 title="Limit Header", 642 ) 643 exhaustion_status_codes: Optional[List[int]] = Field( 644 None, 645 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 646 examples=[[429]], 647 title="Exhaustion Status Codes", 648 ) 649 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 650 651 652class RateLimitedMultipleTokenAuthenticator(BaseModel): 653 type: Literal["RateLimitedMultipleTokenAuthenticator"] 654 tokens: Union[str, List[str]] = Field( 655 ..., 656 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 657 examples=[ 658 "{{ config['credentials']['personal_access_token'] }}", 659 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 660 ], 661 title="Tokens", 662 ) 663 token_delimiter: Optional[str] = Field( 664 ",", 665 description="Delimiter used to split a single token string into multiple tokens.", 666 title="Token Delimiter", 667 ) 668 auth_method: Optional[str] = Field( 669 "Bearer", 670 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 671 examples=["Bearer", "token"], 672 title="Auth Method", 673 ) 674 header: Optional[str] = Field( 675 "Authorization", 676 description="The name of the HTTP header in which to inject the token.", 677 title="Header Name", 678 ) 679 quota_status_source: QuotaStatusSource = Field( 680 ..., 681 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 682 title="Quota Status Source", 683 ) 684 quotas: List[TokenQuota] = Field( 685 ..., 686 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 687 min_items=1, 688 title="Quota Pools", 689 ) 690 max_wait_time: Optional[str] = Field( 691 "PT2H", 692 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 693 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 694 title="Maximum Wait Time", 695 ) 696 budget_reserve_fraction: Optional[float] = Field( 697 0.1, 698 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 699 title="Budget Reserve Fraction", 700 ) 701 budget_min_reserve: Optional[int] = Field( 702 50, 703 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 704 title="Budget Minimum Reserve", 705 ) 706 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 707 708 709class Action(Enum): 710 SUCCESS = "SUCCESS" 711 FAIL = "FAIL" 712 RETRY = "RETRY" 713 IGNORE = "IGNORE" 714 RESET_PAGINATION = "RESET_PAGINATION" 715 RATE_LIMITED = "RATE_LIMITED" 716 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 717 718 719class FailureType(Enum): 720 system_error = "system_error" 721 config_error = "config_error" 722 transient_error = "transient_error" 723 724 725class HttpResponseFilter(BaseModel): 726 type: Literal["HttpResponseFilter"] 727 action: Optional[Action] = Field( 728 None, 729 description="Action to execute if a response matches the filter.", 730 examples=[ 731 "SUCCESS", 732 "FAIL", 733 "RETRY", 734 "IGNORE", 735 "RESET_PAGINATION", 736 "RATE_LIMITED", 737 "REFRESH_TOKEN_THEN_RETRY", 738 ], 739 title="Action", 740 ) 741 failure_type: Optional[FailureType] = Field( 742 None, 743 description="Failure type of traced exception if a response matches the filter.", 744 examples=["system_error", "config_error", "transient_error"], 745 title="Failure Type", 746 ) 747 error_message: Optional[str] = Field( 748 None, 749 description="Error Message to display if the response matches the filter.", 750 title="Error Message", 751 ) 752 error_message_contains: Optional[str] = Field( 753 None, 754 description="Match the response if its error message contains the substring.", 755 example=["This API operation is not enabled for this site"], 756 title="Error Message Substring", 757 ) 758 http_codes: Optional[List[int]] = Field( 759 None, 760 description="Match the response if its HTTP code is included in this list.", 761 examples=[[420, 429], [500]], 762 title="HTTP Codes", 763 unique_items=True, 764 ) 765 predicate: Optional[str] = Field( 766 None, 767 description="Match the response if the predicate evaluates to true.", 768 examples=[ 769 "{{ 'Too much requests' in response }}", 770 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 771 ], 772 title="Predicate", 773 ) 774 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 775 776 777class ComplexFieldType(BaseModel): 778 field_type: str 779 items: Optional[Union[str, ComplexFieldType]] = None 780 781 782class TypesMap(BaseModel): 783 target_type: Union[str, List[str], ComplexFieldType] 784 current_type: Union[str, List[str]] 785 condition: Optional[str] = None 786 787 788class SchemaTypeIdentifier(BaseModel): 789 type: Optional[Literal["SchemaTypeIdentifier"]] = None 790 schema_pointer: Optional[List[str]] = Field( 791 [], 792 description="List of nested fields defining the schema field path to extract. Defaults to [].", 793 title="Schema Path", 794 ) 795 key_pointer: List[str] = Field( 796 ..., 797 description="List of potentially nested fields describing the full path of the field key to extract.", 798 title="Key Path", 799 ) 800 type_pointer: Optional[List[str]] = Field( 801 None, 802 description="List of potentially nested fields describing the full path of the field type to extract.", 803 title="Type Path", 804 ) 805 types_mapping: Optional[List[TypesMap]] = None 806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 807 808 809class InlineSchemaLoader(BaseModel): 810 type: Literal["InlineSchemaLoader"] 811 schema_: Optional[Dict[str, Any]] = Field( 812 None, 813 alias="schema", 814 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 815 title="Schema", 816 ) 817 818 819class JsonFileSchemaLoader(BaseModel): 820 type: Literal["JsonFileSchemaLoader"] 821 file_path: Optional[str] = Field( 822 None, 823 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 824 example=["./schemas/users.json"], 825 title="File Path", 826 ) 827 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 828 829 830class JsonDecoder(BaseModel): 831 type: Literal["JsonDecoder"] 832 833 834class JsonItemsDecoder(BaseModel): 835 type: Literal["JsonItemsDecoder"] 836 items_path: str = Field( 837 ..., 838 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 839 title="Items Path", 840 ) 841 encoding: Optional[str] = Field( 842 "utf-8", 843 description="The character encoding of the JSON data. Defaults to UTF-8.", 844 title="Encoding", 845 ) 846 847 848class JsonlDecoder(BaseModel): 849 type: Literal["JsonlDecoder"] 850 851 852class KeysToLower(BaseModel): 853 type: Literal["KeysToLower"] 854 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 855 856 857class KeysToSnakeCase(BaseModel): 858 type: Literal["KeysToSnakeCase"] 859 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 860 861 862class FlattenFields(BaseModel): 863 type: Literal["FlattenFields"] 864 flatten_lists: Optional[bool] = Field( 865 True, 866 description="Whether to flatten lists or leave it as is. Default is True.", 867 title="Flatten Lists", 868 ) 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class KeyTransformation(BaseModel): 873 type: Literal["KeyTransformation"] 874 prefix: Optional[str] = Field( 875 None, 876 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 877 examples=["flattened_"], 878 title="Key Prefix", 879 ) 880 suffix: Optional[str] = Field( 881 None, 882 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 883 examples=["_flattened"], 884 title="Key Suffix", 885 ) 886 887 888class DpathFlattenFields(BaseModel): 889 type: Literal["DpathFlattenFields"] 890 field_path: List[str] = Field( 891 ..., 892 description="A path to field that needs to be flattened.", 893 examples=[["data"], ["data", "*", "field"]], 894 title="Field Path", 895 ) 896 delete_origin_value: Optional[bool] = Field( 897 None, 898 description="Whether to delete the origin value or keep it. Default is False.", 899 title="Delete Origin Value", 900 ) 901 replace_record: Optional[bool] = Field( 902 None, 903 description="Whether to replace the origin record or not. Default is False.", 904 title="Replace Origin Record", 905 ) 906 key_transformation: Optional[KeyTransformation] = Field( 907 None, 908 description="Transformation for object keys. If not provided, original key will be used.", 909 title="Key transformation", 910 ) 911 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 912 913 914class KeysReplace(BaseModel): 915 type: Literal["KeysReplace"] 916 old: str = Field( 917 ..., 918 description="Old value to replace.", 919 examples=[ 920 " ", 921 "{{ record.id }}", 922 "{{ config['id'] }}", 923 "{{ stream_slice['id'] }}", 924 ], 925 title="Old value", 926 ) 927 new: str = Field( 928 ..., 929 description="New value to set.", 930 examples=[ 931 "_", 932 "{{ record.id }}", 933 "{{ config['id'] }}", 934 "{{ stream_slice['id'] }}", 935 ], 936 title="New value", 937 ) 938 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 939 940 941class IterableDecoder(BaseModel): 942 type: Literal["IterableDecoder"] 943 944 945class XmlDecoder(BaseModel): 946 type: Literal["XmlDecoder"] 947 948 949class CustomDecoder(BaseModel): 950 class Config: 951 extra = Extra.allow 952 953 type: Literal["CustomDecoder"] 954 class_name: str = Field( 955 ..., 956 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 957 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 958 title="Class Name", 959 ) 960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 961 962 963class MinMaxDatetime(BaseModel): 964 type: Literal["MinMaxDatetime"] 965 datetime: str = Field( 966 ..., 967 description="Datetime value.", 968 examples=[ 969 "2021-01-01", 970 "2021-01-01T00:00:00Z", 971 "{{ config['start_time'] }}", 972 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 973 ], 974 title="Datetime", 975 ) 976 datetime_format: Optional[str] = Field( 977 "", 978 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 979 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 980 title="Datetime Format", 981 ) 982 max_datetime: Optional[str] = Field( 983 None, 984 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 985 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 986 title="Max Datetime", 987 ) 988 min_datetime: Optional[str] = Field( 989 None, 990 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 991 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 992 title="Min Datetime", 993 ) 994 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 995 996 997class NoAuth(BaseModel): 998 type: Literal["NoAuth"] 999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1000 1001 1002class NoPagination(BaseModel): 1003 type: Literal["NoPagination"] 1004 1005 1006class State(BaseModel): 1007 class Config: 1008 extra = Extra.allow 1009 1010 min: int 1011 max: int 1012 1013 1014class OAuthScope(BaseModel): 1015 class Config: 1016 extra = Extra.allow 1017 1018 scope: str = Field( 1019 ..., 1020 description="The OAuth scope string to request from the provider.", 1021 ) 1022 1023 1024class OauthConnectorInputSpecification(BaseModel): 1025 class Config: 1026 extra = Extra.allow 1027 1028 consent_url: str = Field( 1029 ..., 1030 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1031 examples=[ 1032 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1033 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1034 ], 1035 title="Consent URL", 1036 ) 1037 scope: Optional[str] = Field( 1038 None, 1039 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1040 examples=["user:read user:read_orders workspaces:read"], 1041 title="Scopes", 1042 ) 1043 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1044 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1045 # The CDK schema defines the manifest contract; the platform reads these fields 1046 # during the OAuth consent flow to build the authorization URL. 1047 scopes: Optional[List[OAuthScope]] = Field( 1048 None, 1049 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1050 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1051 title="Scopes", 1052 ) 1053 optional_scopes: Optional[List[OAuthScope]] = Field( 1054 None, 1055 description="Optional OAuth scope objects that may or may not be granted.", 1056 examples=[[{"scope": "admin:read"}]], 1057 title="Optional Scopes", 1058 ) 1059 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1060 ScopesJoinStrategy.space, 1061 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1062 title="Scopes Join Strategy", 1063 ) 1064 access_token_url: str = Field( 1065 ..., 1066 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1067 examples=[ 1068 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1069 ], 1070 title="Access Token URL", 1071 ) 1072 access_token_headers: Optional[Dict[str, Any]] = Field( 1073 None, 1074 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1075 examples=[ 1076 { 1077 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1078 } 1079 ], 1080 title="Access Token Headers", 1081 ) 1082 access_token_params: Optional[Dict[str, Any]] = Field( 1083 None, 1084 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1085 examples=[ 1086 { 1087 "{{ auth_code_key }}": "{{ auth_code_value }}", 1088 "{{ client_id_key }}": "{{ client_id_value }}", 1089 "{{ client_secret_key }}": "{{ client_secret_value }}", 1090 } 1091 ], 1092 title="Access Token Query Params (Json Encoded)", 1093 ) 1094 extract_output: Optional[List[str]] = Field( 1095 None, 1096 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1097 examples=[["access_token", "refresh_token", "other_field"]], 1098 title="Extract Output", 1099 ) 1100 state: Optional[State] = Field( 1101 None, 1102 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1103 examples=[{"min": 7, "max": 128}], 1104 title="Configurable State Query Param", 1105 ) 1106 client_id_key: Optional[str] = Field( 1107 None, 1108 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1109 examples=["my_custom_client_id_key_name"], 1110 title="Client ID Key Override", 1111 ) 1112 client_secret_key: Optional[str] = Field( 1113 None, 1114 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1115 examples=["my_custom_client_secret_key_name"], 1116 title="Client Secret Key Override", 1117 ) 1118 scope_key: Optional[str] = Field( 1119 None, 1120 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1121 examples=["my_custom_scope_key_key_name"], 1122 title="Scopes Key Override", 1123 ) 1124 state_key: Optional[str] = Field( 1125 None, 1126 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1127 examples=["my_custom_state_key_key_name"], 1128 title="State Key Override", 1129 ) 1130 auth_code_key: Optional[str] = Field( 1131 None, 1132 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1133 examples=["my_custom_auth_code_key_name"], 1134 title="Auth Code Key Override", 1135 ) 1136 redirect_uri_key: Optional[str] = Field( 1137 None, 1138 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1139 examples=["my_custom_redirect_uri_key_name"], 1140 title="Redirect URI Key Override", 1141 ) 1142 1143 1144class OAuthConfigSpecification(BaseModel): 1145 class Config: 1146 extra = Extra.allow 1147 1148 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1149 None, 1150 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1151 examples=[ 1152 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1153 { 1154 "app_id": { 1155 "type": "string", 1156 "path_in_connector_config": ["info", "app_id"], 1157 } 1158 }, 1159 ], 1160 title="OAuth user input", 1161 ) 1162 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1163 None, 1164 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1165 title="DeclarativeOAuth Connector Specification", 1166 ) 1167 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1168 None, 1169 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1170 examples=[ 1171 { 1172 "refresh_token": { 1173 "type": "string,", 1174 "path_in_connector_config": ["credentials", "refresh_token"], 1175 } 1176 } 1177 ], 1178 title="OAuth output specification", 1179 ) 1180 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1181 None, 1182 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1183 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1184 title="OAuth input specification", 1185 ) 1186 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1187 None, 1188 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1189 examples=[ 1190 { 1191 "client_id": { 1192 "type": "string,", 1193 "path_in_connector_config": ["credentials", "client_id"], 1194 }, 1195 "client_secret": { 1196 "type": "string,", 1197 "path_in_connector_config": ["credentials", "client_secret"], 1198 }, 1199 } 1200 ], 1201 title="OAuth server output specification", 1202 ) 1203 1204 1205class OffsetIncrement(BaseModel): 1206 type: Literal["OffsetIncrement"] 1207 page_size: Optional[Union[int, str]] = Field( 1208 None, 1209 description="The number of records to include in each pages.", 1210 examples=[100, "{{ config['page_size'] }}"], 1211 title="Limit", 1212 ) 1213 inject_on_first_request: Optional[bool] = Field( 1214 False, 1215 description="Using the `offset` with value `0` during the first request", 1216 title="Inject Offset on First Request", 1217 ) 1218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1219 1220 1221class PageIncrement(BaseModel): 1222 type: Literal["PageIncrement"] 1223 page_size: Optional[Union[int, str]] = Field( 1224 None, 1225 description="The number of records to include in each pages.", 1226 examples=[100, "100", "{{ config['page_size'] }}"], 1227 title="Page Size", 1228 ) 1229 start_from_page: Optional[int] = Field( 1230 0, 1231 description="Index of the first page to request.", 1232 examples=[0, 1], 1233 title="Start From Page", 1234 ) 1235 inject_on_first_request: Optional[bool] = Field( 1236 False, 1237 description="Using the `page number` with value defined by `start_from_page` during the first request", 1238 title="Inject Page Number on First Request", 1239 ) 1240 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1241 1242 1243class PrimaryKey(BaseModel): 1244 __root__: Union[str, List[str], List[List[str]]] = Field( 1245 ..., 1246 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1247 examples=["id", ["code", "type"]], 1248 title="Primary Key", 1249 ) 1250 1251 1252class PropertyLimitType(Enum): 1253 characters = "characters" 1254 property_count = "property_count" 1255 1256 1257class PropertyChunking(BaseModel): 1258 type: Literal["PropertyChunking"] 1259 property_limit_type: PropertyLimitType = Field( 1260 ..., 1261 description="The type used to determine the maximum number of properties per chunk", 1262 title="Property Limit Type", 1263 ) 1264 property_limit: Optional[int] = Field( 1265 None, 1266 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1267 title="Property Limit", 1268 ) 1269 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1270 None, 1271 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1272 title="Record Merge Strategy", 1273 ) 1274 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1275 1276 1277class RecordFilter(BaseModel): 1278 type: Literal["RecordFilter"] 1279 condition: Optional[str] = Field( 1280 "", 1281 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1282 examples=[ 1283 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1284 "{{ record.status in ['active', 'expired'] }}", 1285 ], 1286 title="Condition", 1287 ) 1288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1289 1290 1291class SchemaNormalization(Enum): 1292 Default = "Default" 1293 None_ = "None" 1294 1295 1296class RemoveFields(BaseModel): 1297 type: Literal["RemoveFields"] 1298 condition: Optional[str] = Field( 1299 "", 1300 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1301 examples=[ 1302 "{{ property|string == '' }}", 1303 "{{ property is integer }}", 1304 "{{ property|length > 5 }}", 1305 "{{ property == 'some_string_to_match' }}", 1306 ], 1307 ) 1308 field_pointers: List[List[str]] = Field( 1309 ..., 1310 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1311 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1312 title="Field Paths", 1313 ) 1314 1315 1316class RequestPath(BaseModel): 1317 type: Literal["RequestPath"] 1318 1319 1320class InjectInto(Enum): 1321 request_parameter = "request_parameter" 1322 header = "header" 1323 body_data = "body_data" 1324 body_json = "body_json" 1325 1326 1327class RequestOption(BaseModel): 1328 type: Literal["RequestOption"] 1329 inject_into: InjectInto = Field( 1330 ..., 1331 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1332 examples=["request_parameter", "header", "body_data", "body_json"], 1333 title="Inject Into", 1334 ) 1335 field_name: Optional[str] = Field( 1336 None, 1337 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1338 examples=["segment_id"], 1339 title="Field Name", 1340 ) 1341 field_path: Optional[List[str]] = Field( 1342 None, 1343 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1344 examples=[["data", "viewer", "id"]], 1345 title="Field Path", 1346 ) 1347 1348 1349class Schemas(BaseModel): 1350 pass 1351 1352 class Config: 1353 extra = Extra.allow 1354 1355 1356class LegacySessionTokenAuthenticator(BaseModel): 1357 type: Literal["LegacySessionTokenAuthenticator"] 1358 header: str = Field( 1359 ..., 1360 description="The name of the session token header that will be injected in the request", 1361 examples=["X-Session"], 1362 title="Session Request Header", 1363 ) 1364 login_url: str = Field( 1365 ..., 1366 description="Path of the login URL (do not include the base URL)", 1367 examples=["session"], 1368 title="Login Path", 1369 ) 1370 session_token: Optional[str] = Field( 1371 None, 1372 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1373 example=["{{ config['session_token'] }}"], 1374 title="Session Token", 1375 ) 1376 session_token_response_key: str = Field( 1377 ..., 1378 description="Name of the key of the session token to be extracted from the response", 1379 examples=["id"], 1380 title="Response Token Response Key", 1381 ) 1382 username: Optional[str] = Field( 1383 None, 1384 description="Username used to authenticate and obtain a session token", 1385 examples=[" {{ config['username'] }}"], 1386 title="Username", 1387 ) 1388 password: Optional[str] = Field( 1389 "", 1390 description="Password used to authenticate and obtain a session token", 1391 examples=["{{ config['password'] }}", ""], 1392 title="Password", 1393 ) 1394 validate_session_url: str = Field( 1395 ..., 1396 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1397 examples=["user/current"], 1398 title="Validate Session Path", 1399 ) 1400 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1401 1402 1403class Action1(Enum): 1404 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1405 RESET = "RESET" 1406 1407 1408class PaginationResetLimits(BaseModel): 1409 type: Literal["PaginationResetLimits"] 1410 number_of_records: Optional[int] = None 1411 1412 1413class CsvDecoder(BaseModel): 1414 type: Literal["CsvDecoder"] 1415 encoding: Optional[str] = "utf-8" 1416 delimiter: Optional[str] = "," 1417 set_values_to_none: Optional[List[str]] = None 1418 1419 1420class AsyncJobStatusMap(BaseModel): 1421 type: Optional[Literal["AsyncJobStatusMap"]] = None 1422 running: List[str] 1423 completed: List[str] 1424 failed: List[str] 1425 timeout: List[str] 1426 skipped: Optional[List[str]] = None 1427 1428 1429class ValueType(Enum): 1430 string = "string" 1431 number = "number" 1432 integer = "integer" 1433 boolean = "boolean" 1434 1435 1436class WaitTimeFromHeader(BaseModel): 1437 type: Literal["WaitTimeFromHeader"] 1438 header: str = Field( 1439 ..., 1440 description="The name of the response header defining how long to wait before retrying.", 1441 examples=["Retry-After"], 1442 title="Response Header Name", 1443 ) 1444 regex: Optional[str] = Field( 1445 None, 1446 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1447 examples=["([-+]?\\d+)"], 1448 title="Extraction Regex", 1449 ) 1450 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1451 None, 1452 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1453 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1454 title="Max Waiting Time in Seconds", 1455 ) 1456 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1457 1458 1459class WaitUntilTimeFromHeader(BaseModel): 1460 type: Literal["WaitUntilTimeFromHeader"] 1461 header: str = Field( 1462 ..., 1463 description="The name of the response header defining how long to wait before retrying.", 1464 examples=["wait_time"], 1465 title="Response Header", 1466 ) 1467 min_wait: Optional[Union[float, str]] = Field( 1468 None, 1469 description="Minimum time to wait before retrying.", 1470 examples=[10, "60"], 1471 title="Minimum Wait Time", 1472 ) 1473 regex: Optional[str] = Field( 1474 None, 1475 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1476 examples=["([-+]?\\d+)"], 1477 title="Extraction Regex", 1478 ) 1479 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1480 None, 1481 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1482 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1483 title="Max Waiting Time in Seconds", 1484 ) 1485 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1486 1487 1488class ComponentMappingDefinition(BaseModel): 1489 type: Literal["ComponentMappingDefinition"] 1490 field_path: List[str] = Field( 1491 ..., 1492 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1493 examples=[ 1494 ["name"], 1495 ["retriever", "requester", "url"], 1496 ["retriever", "requester", "{{ components_values.field }}"], 1497 ["*", "**", "name"], 1498 ], 1499 title="Field Path", 1500 ) 1501 value: str = Field( 1502 ..., 1503 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1504 examples=[ 1505 "{{ components_values['updates'] }}", 1506 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1507 "{{ config['segment_id'] }}", 1508 "{{ stream_slice['parent_id'] }}", 1509 "{{ stream_slice['extra_fields']['name'] }}", 1510 ], 1511 title="Value", 1512 ) 1513 value_type: Optional[ValueType] = Field( 1514 None, 1515 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1516 title="Value Type", 1517 ) 1518 create_or_update: Optional[bool] = Field( 1519 False, 1520 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1521 title="Create or Update", 1522 ) 1523 condition: Optional[str] = Field( 1524 None, 1525 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1526 examples=[ 1527 "{{ components_values.get('cursor_field', None) }}", 1528 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1529 ], 1530 title="Condition", 1531 ) 1532 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1533 1534 1535class StreamConfig(BaseModel): 1536 type: Literal["StreamConfig"] 1537 configs_pointer: List[str] = Field( 1538 ..., 1539 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1540 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1541 title="Configs Pointer", 1542 ) 1543 default_values: Optional[List[Dict[str, Any]]] = Field( 1544 None, 1545 description="A list of default values, each matching the structure expected from the parsed component value.", 1546 title="Default Values", 1547 ) 1548 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1549 1550 1551class ConfigComponentsResolver(BaseModel): 1552 type: Literal["ConfigComponentsResolver"] 1553 stream_config: Union[List[StreamConfig], StreamConfig] 1554 components_mapping: List[ComponentMappingDefinition] 1555 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1556 1557 1558class StreamParametersDefinition(BaseModel): 1559 type: Literal["StreamParametersDefinition"] 1560 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1561 ..., 1562 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1563 examples=[ 1564 [ 1565 { 1566 "name": "test stream", 1567 "$parameters": {"entity": "test entity"}, 1568 "primary_key": "test key", 1569 } 1570 ] 1571 ], 1572 title="Stream Parameters", 1573 ) 1574 1575 1576class ParametrizedComponentsResolver(BaseModel): 1577 type: Literal["ParametrizedComponentsResolver"] 1578 stream_parameters: StreamParametersDefinition 1579 components_mapping: List[ComponentMappingDefinition] 1580 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1581 1582 1583class RequestBodyPlainText(BaseModel): 1584 type: Literal["RequestBodyPlainText"] 1585 value: str 1586 1587 1588class RequestBodyUrlEncodedForm(BaseModel): 1589 type: Literal["RequestBodyUrlEncodedForm"] 1590 value: Dict[str, str] 1591 1592 1593class RequestBodyJsonObject(BaseModel): 1594 type: Literal["RequestBodyJsonObject"] 1595 value: Dict[str, Any] 1596 1597 1598class RequestBodyGraphQlQuery(BaseModel): 1599 class Config: 1600 extra = Extra.allow 1601 1602 query: str = Field(..., description="The GraphQL query to be executed") 1603 1604 1605class ValidateAdheresToSchema(BaseModel): 1606 type: Literal["ValidateAdheresToSchema"] 1607 base_schema: Union[str, Dict[str, Any]] = Field( 1608 ..., 1609 description="The base JSON schema against which the user-provided schema will be validated.", 1610 examples=[ 1611 "{{ config['report_validation_schema'] }}", 1612 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1613 { 1614 "$schema": "http://json-schema.org/draft-07/schema#", 1615 "title": "Person", 1616 "type": "object", 1617 "properties": { 1618 "name": {"type": "string", "description": "The person's name"}, 1619 "age": { 1620 "type": "integer", 1621 "minimum": 0, 1622 "description": "The person's age", 1623 }, 1624 }, 1625 "required": ["name", "age"], 1626 }, 1627 ], 1628 title="Base JSON Schema", 1629 ) 1630 1631 1632class CustomValidationStrategy(BaseModel): 1633 class Config: 1634 extra = Extra.allow 1635 1636 type: Literal["CustomValidationStrategy"] 1637 class_name: str = Field( 1638 ..., 1639 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1640 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1641 title="Class Name", 1642 ) 1643 1644 1645class ConfigRemapField(BaseModel): 1646 type: Literal["ConfigRemapField"] 1647 map: Union[Dict[str, Any], str] = Field( 1648 ..., 1649 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1650 examples=[ 1651 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1652 "{{ config['status_mapping'] }}", 1653 ], 1654 title="Value Mapping", 1655 ) 1656 field_path: List[str] = Field( 1657 ..., 1658 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1659 examples=[ 1660 ["status"], 1661 ["data", "status"], 1662 ["data", "{{ config.name }}", "status"], 1663 ["data", "*", "status"], 1664 ], 1665 title="Field Path", 1666 ) 1667 1668 1669class ConfigRemoveFields(BaseModel): 1670 type: Literal["ConfigRemoveFields"] 1671 field_pointers: List[List[str]] = Field( 1672 ..., 1673 description="A list of field pointers to be removed from the config.", 1674 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1675 title="Field Pointers", 1676 ) 1677 condition: Optional[str] = Field( 1678 "", 1679 description="Fields will be removed if expression is evaluated to True.", 1680 examples=[ 1681 "{{ config['environemnt'] == 'sandbox' }}", 1682 "{{ property is integer }}", 1683 "{{ property|length > 5 }}", 1684 "{{ property == 'some_string_to_match' }}", 1685 ], 1686 ) 1687 1688 1689class CustomConfigTransformation(BaseModel): 1690 type: Literal["CustomConfigTransformation"] 1691 class_name: str = Field( 1692 ..., 1693 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1694 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1695 ) 1696 parameters: Optional[Dict[str, Any]] = Field( 1697 None, 1698 alias="$parameters", 1699 description="Additional parameters to be passed to the custom config transformation.", 1700 ) 1701 1702 1703class AddedFieldDefinition(BaseModel): 1704 type: Literal["AddedFieldDefinition"] 1705 path: List[str] = Field( 1706 ..., 1707 description="List of strings defining the path where to add the value on the record.", 1708 examples=[["segment_id"], ["metadata", "segment_id"]], 1709 title="Path", 1710 ) 1711 value: str = Field( 1712 ..., 1713 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1714 examples=[ 1715 "{{ record['updates'] }}", 1716 "{{ record['MetaData']['LastUpdatedTime'] }}", 1717 "{{ stream_partition['segment_id'] }}", 1718 ], 1719 title="Value", 1720 ) 1721 value_type: Optional[ValueType] = Field( 1722 None, 1723 description="Type of the value. If not specified, the type will be inferred from the value.", 1724 title="Value Type", 1725 ) 1726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1727 1728 1729class AddFields(BaseModel): 1730 type: Literal["AddFields"] 1731 fields: List[AddedFieldDefinition] = Field( 1732 ..., 1733 description="List of transformations (path and corresponding value) that will be added to the record.", 1734 title="Fields", 1735 ) 1736 condition: Optional[str] = Field( 1737 "", 1738 description="Fields will be added if expression is evaluated to True.", 1739 examples=[ 1740 "{{ property|string == '' }}", 1741 "{{ property is integer }}", 1742 "{{ property|length > 5 }}", 1743 "{{ property == 'some_string_to_match' }}", 1744 ], 1745 ) 1746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1747 1748 1749class ApiKeyAuthenticator(BaseModel): 1750 type: Literal["ApiKeyAuthenticator"] 1751 api_token: Optional[str] = Field( 1752 None, 1753 description="The API key to inject in the request. Fill it in the user inputs.", 1754 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1755 title="API Key", 1756 ) 1757 header: Optional[str] = Field( 1758 None, 1759 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1760 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1761 title="Header Name", 1762 ) 1763 inject_into: Optional[RequestOption] = Field( 1764 None, 1765 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1766 examples=[ 1767 {"inject_into": "header", "field_name": "Authorization"}, 1768 {"inject_into": "request_parameter", "field_name": "authKey"}, 1769 ], 1770 title="Inject API Key Into Outgoing HTTP Request", 1771 ) 1772 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1773 1774 1775class AuthFlow(BaseModel): 1776 auth_flow_type: Optional[AuthFlowType] = Field( 1777 None, description="The type of auth to use", title="Auth flow type" 1778 ) 1779 predicate_key: Optional[List[str]] = Field( 1780 None, 1781 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1782 examples=[["credentials", "auth_type"]], 1783 title="Predicate key", 1784 ) 1785 predicate_value: Optional[str] = Field( 1786 None, 1787 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1788 examples=["Oauth"], 1789 title="Predicate value", 1790 ) 1791 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1792 1793 1794class CheckStream(BaseModel): 1795 type: Literal["CheckStream"] 1796 stream_names: Optional[List[str]] = Field( 1797 None, 1798 description="Names of the streams to try reading from when running a check operation.", 1799 examples=[["users"], ["users", "contacts"]], 1800 title="Stream Names", 1801 ) 1802 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1803 config_overrides: Optional[Dict[str, Any]] = Field( 1804 None, 1805 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1806 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1807 title="Config Overrides", 1808 ) 1809 1810 1811class IncrementingCountCursor(BaseModel): 1812 type: Literal["IncrementingCountCursor"] 1813 cursor_field: str = Field( 1814 ..., 1815 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1816 examples=["created_at", "{{ config['record_cursor'] }}"], 1817 title="Cursor Field", 1818 ) 1819 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1820 None, 1821 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1822 title="Allow Catalog Defined Cursor Field", 1823 ) 1824 start_value: Optional[Union[str, int]] = Field( 1825 None, 1826 description="The value that determines the earliest record that should be synced.", 1827 examples=[0, "{{ config['start_value'] }}"], 1828 title="Start Value", 1829 ) 1830 start_value_option: Optional[RequestOption] = Field( 1831 None, 1832 description="Optionally configures how the start value will be sent in requests to the source API.", 1833 title="Inject Start Value Into Outgoing HTTP Request", 1834 ) 1835 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1836 1837 1838class DatetimeBasedCursor(BaseModel): 1839 type: Literal["DatetimeBasedCursor"] 1840 clamping: Optional[Clamping] = Field( 1841 None, 1842 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1843 title="Date Range Clamping", 1844 ) 1845 cursor_field: str = Field( 1846 ..., 1847 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1848 examples=["created_at", "{{ config['record_cursor'] }}"], 1849 title="Cursor Field", 1850 ) 1851 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1852 None, 1853 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1854 title="Allow Catalog Defined Cursor Field", 1855 ) 1856 cursor_datetime_formats: Optional[List[str]] = Field( 1857 None, 1858 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1859 examples=[ 1860 "%Y-%m-%d", 1861 "%Y-%m-%d %H:%M:%S", 1862 "%Y-%m-%dT%H:%M:%S", 1863 "%Y-%m-%dT%H:%M:%SZ", 1864 "%Y-%m-%dT%H:%M:%S%z", 1865 "%Y-%m-%dT%H:%M:%S.%fZ", 1866 "%Y-%m-%dT%H:%M:%S.%f%z", 1867 "%Y-%m-%d %H:%M:%S.%f+00:00", 1868 "%s", 1869 "%ms", 1870 ], 1871 title="Cursor Datetime Formats", 1872 ) 1873 start_datetime: Union[MinMaxDatetime, str] = Field( 1874 ..., 1875 description="The datetime that determines the earliest record that should be synced.", 1876 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1877 title="Start Datetime", 1878 ) 1879 start_time_option: Optional[RequestOption] = Field( 1880 None, 1881 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1882 title="Inject Start Time Into Outgoing HTTP Request", 1883 ) 1884 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1885 None, 1886 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1887 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1888 title="End Datetime", 1889 ) 1890 end_time_option: Optional[RequestOption] = Field( 1891 None, 1892 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1893 title="Inject End Time Into Outgoing HTTP Request", 1894 ) 1895 datetime_format: str = Field( 1896 ..., 1897 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1898 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1899 title="Outgoing Datetime Format", 1900 ) 1901 cursor_granularity: Optional[str] = Field( 1902 None, 1903 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1904 examples=["PT1S"], 1905 title="Cursor Granularity", 1906 ) 1907 is_data_feed: Optional[bool] = Field( 1908 None, 1909 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1910 title="Data Feed API", 1911 ) 1912 is_client_side_incremental: Optional[bool] = Field( 1913 None, 1914 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1915 title="Client-side Incremental Filtering", 1916 ) 1917 is_compare_strictly: Optional[bool] = Field( 1918 False, 1919 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1920 title="Strict Start-End Time Comparison", 1921 ) 1922 global_substream_cursor: Optional[bool] = Field( 1923 False, 1924 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1925 title="Global Substream Cursor", 1926 ) 1927 lookback_window: Optional[str] = Field( 1928 None, 1929 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1930 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1931 title="Lookback Window", 1932 ) 1933 partition_field_end: Optional[str] = Field( 1934 None, 1935 description="Name of the partition start time field.", 1936 examples=["ending_time"], 1937 title="Partition Field End", 1938 ) 1939 partition_field_start: Optional[str] = Field( 1940 None, 1941 description="Name of the partition end time field.", 1942 examples=["starting_time"], 1943 title="Partition Field Start", 1944 ) 1945 step: Optional[str] = Field( 1946 None, 1947 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1948 examples=["P1W", "{{ config['step_increment'] }}"], 1949 title="Step", 1950 ) 1951 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1952 1953 1954class JwtAuthenticator(BaseModel): 1955 type: Literal["JwtAuthenticator"] 1956 secret_key: str = Field( 1957 ..., 1958 description="Secret used to sign the JSON web token.", 1959 examples=["{{ config['secret_key'] }}"], 1960 title="Secret Key", 1961 ) 1962 base64_encode_secret_key: Optional[bool] = Field( 1963 False, 1964 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1965 title="Base64-encode Secret Key", 1966 ) 1967 algorithm: Algorithm = Field( 1968 ..., 1969 description="Algorithm used to sign the JSON web token.", 1970 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1971 title="Algorithm", 1972 ) 1973 token_duration: Optional[int] = Field( 1974 1200, 1975 description="The amount of time in seconds a JWT token can be valid after being issued.", 1976 examples=[1200, 3600], 1977 title="Token Duration", 1978 ) 1979 header_prefix: Optional[str] = Field( 1980 None, 1981 description="The prefix to be used within the Authentication header.", 1982 examples=["Bearer", "Basic"], 1983 title="Header Prefix", 1984 ) 1985 jwt_headers: Optional[JwtHeaders] = Field( 1986 None, 1987 description="JWT headers used when signing JSON web token.", 1988 title="JWT Headers", 1989 ) 1990 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1991 None, 1992 description="Additional headers to be included with the JWT headers object.", 1993 title="Additional JWT Headers", 1994 ) 1995 jwt_payload: Optional[JwtPayload] = Field( 1996 None, 1997 description="JWT Payload used when signing JSON web token.", 1998 title="JWT Payload", 1999 ) 2000 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2001 None, 2002 description="Additional properties to be added to the JWT payload.", 2003 title="Additional JWT Payload Properties", 2004 ) 2005 passphrase: Optional[str] = Field( 2006 None, 2007 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2008 examples=["{{ config['passphrase'] }}"], 2009 title="Passphrase", 2010 ) 2011 request_option: Optional[RequestOption] = Field( 2012 None, 2013 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2014 title="Request Option", 2015 ) 2016 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2017 2018 2019class OAuthAuthenticator(BaseModel): 2020 type: Literal["OAuthAuthenticator"] 2021 client_id_name: Optional[str] = Field( 2022 "client_id", 2023 description="The name of the property to use to refresh the `access_token`.", 2024 examples=["custom_app_id"], 2025 title="Client ID Property Name", 2026 ) 2027 client_id: Optional[str] = Field( 2028 None, 2029 description="The OAuth client ID. Fill it in the user inputs.", 2030 examples=[ 2031 "{{ config['client_id'] }}", 2032 "{{ config['credentials']['client_id }}", 2033 ], 2034 title="Client ID", 2035 ) 2036 client_secret_name: Optional[str] = Field( 2037 "client_secret", 2038 description="The name of the property to use to refresh the `access_token`.", 2039 examples=["custom_app_secret"], 2040 title="Client Secret Property Name", 2041 ) 2042 client_secret: Optional[str] = Field( 2043 None, 2044 description="The OAuth client secret. Fill it in the user inputs.", 2045 examples=[ 2046 "{{ config['client_secret'] }}", 2047 "{{ config['credentials']['client_secret }}", 2048 ], 2049 title="Client Secret", 2050 ) 2051 refresh_token_name: Optional[str] = Field( 2052 "refresh_token", 2053 description="The name of the property to use to refresh the `access_token`.", 2054 examples=["custom_app_refresh_value"], 2055 title="Refresh Token Property Name", 2056 ) 2057 refresh_token: Optional[str] = Field( 2058 None, 2059 description="Credential artifact used to get a new access token.", 2060 examples=[ 2061 "{{ config['refresh_token'] }}", 2062 "{{ config['credentials]['refresh_token'] }}", 2063 ], 2064 title="Refresh Token", 2065 ) 2066 token_refresh_endpoint: Optional[str] = Field( 2067 None, 2068 description="The full URL to call to obtain a new access token.", 2069 examples=["https://connect.squareup.com/oauth2/token"], 2070 title="Token Refresh Endpoint", 2071 ) 2072 access_token_name: Optional[str] = Field( 2073 "access_token", 2074 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2075 examples=["access_token"], 2076 title="Access Token Property Name", 2077 ) 2078 access_token_value: Optional[str] = Field( 2079 None, 2080 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2081 examples=["secret_access_token_value"], 2082 title="Access Token Value", 2083 ) 2084 expires_in_name: Optional[str] = Field( 2085 "expires_in", 2086 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2087 examples=["expires_in"], 2088 title="Token Expiry Property Name", 2089 ) 2090 grant_type_name: Optional[str] = Field( 2091 "grant_type", 2092 description="The name of the property to use to refresh the `access_token`.", 2093 examples=["custom_grant_type"], 2094 title="Grant Type Property Name", 2095 ) 2096 grant_type: Optional[str] = Field( 2097 "refresh_token", 2098 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2099 examples=["refresh_token", "client_credentials"], 2100 title="Grant Type", 2101 ) 2102 refresh_request_body: Optional[Dict[str, Any]] = Field( 2103 None, 2104 description="Body of the request sent to get a new access token.", 2105 examples=[ 2106 { 2107 "applicationId": "{{ config['application_id'] }}", 2108 "applicationSecret": "{{ config['application_secret'] }}", 2109 "token": "{{ config['token'] }}", 2110 } 2111 ], 2112 title="Refresh Request Body", 2113 ) 2114 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2115 None, 2116 description="Headers of the request sent to get a new access token.", 2117 examples=[ 2118 { 2119 "Authorization": "<AUTH_TOKEN>", 2120 "Content-Type": "application/x-www-form-urlencoded", 2121 } 2122 ], 2123 title="Refresh Request Headers", 2124 ) 2125 send_refresh_request_as_query_params: Optional[bool] = Field( 2126 False, 2127 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2128 examples=[True], 2129 title="Send Refresh Request As Query Params", 2130 ) 2131 scopes: Optional[List[str]] = Field( 2132 None, 2133 description="List of scopes that should be granted to the access token.", 2134 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2135 title="Scopes", 2136 ) 2137 token_expiry_date: Optional[str] = Field( 2138 None, 2139 description="The access token expiry date.", 2140 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2141 title="Token Expiry Date", 2142 ) 2143 token_expiry_date_format: Optional[str] = Field( 2144 None, 2145 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2146 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2147 title="Token Expiry Date Format", 2148 ) 2149 refresh_token_error_status_codes: Optional[List[int]] = Field( 2150 None, 2151 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2152 examples=[[400, 500]], 2153 title="Refresh Token Error Status Codes", 2154 ) 2155 refresh_token_error_key: Optional[str] = Field( 2156 None, 2157 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2158 examples=["error"], 2159 title="Refresh Token Error Key", 2160 ) 2161 refresh_token_error_values: Optional[List[str]] = Field( 2162 None, 2163 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2164 examples=[["invalid_grant", "invalid_permissions"]], 2165 title="Refresh Token Error Values", 2166 ) 2167 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2168 None, 2169 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2170 title="Refresh Token Updater", 2171 ) 2172 profile_assertion: Optional[JwtAuthenticator] = Field( 2173 None, 2174 description="The authenticator being used to authenticate the client authenticator.", 2175 title="Profile Assertion", 2176 ) 2177 use_profile_assertion: Optional[bool] = Field( 2178 False, 2179 description="Enable using profile assertion as a flow for OAuth authorization.", 2180 title="Use Profile Assertion", 2181 ) 2182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2183 2184 2185class FixedWindowCallRatePolicy(BaseModel): 2186 class Config: 2187 extra = Extra.allow 2188 2189 type: Literal["FixedWindowCallRatePolicy"] 2190 period: str = Field( 2191 ..., description="The time interval for the rate limit window.", title="Period" 2192 ) 2193 call_limit: int = Field( 2194 ..., 2195 description="The maximum number of calls allowed within the period.", 2196 title="Call Limit", 2197 ) 2198 matchers: List[HttpRequestRegexMatcher] = Field( 2199 ..., 2200 description="List of matchers that define which requests this policy applies to.", 2201 title="Matchers", 2202 ) 2203 2204 2205class MovingWindowCallRatePolicy(BaseModel): 2206 class Config: 2207 extra = Extra.allow 2208 2209 type: Literal["MovingWindowCallRatePolicy"] 2210 rates: List[Rate] = Field( 2211 ..., 2212 description="List of rates that define the call limits for different time intervals.", 2213 title="Rates", 2214 ) 2215 matchers: List[HttpRequestRegexMatcher] = Field( 2216 ..., 2217 description="List of matchers that define which requests this policy applies to.", 2218 title="Matchers", 2219 ) 2220 2221 2222class UnlimitedCallRatePolicy(BaseModel): 2223 class Config: 2224 extra = Extra.allow 2225 2226 type: Literal["UnlimitedCallRatePolicy"] 2227 matchers: List[HttpRequestRegexMatcher] = Field( 2228 ..., 2229 description="List of matchers that define which requests this policy applies to.", 2230 title="Matchers", 2231 ) 2232 2233 2234class DefaultErrorHandler(BaseModel): 2235 type: Literal["DefaultErrorHandler"] 2236 backoff_strategies: Optional[ 2237 List[ 2238 Union[ 2239 ConstantBackoffStrategy, 2240 ExponentialBackoffStrategy, 2241 WaitTimeFromHeader, 2242 WaitUntilTimeFromHeader, 2243 CustomBackoffStrategy, 2244 ] 2245 ] 2246 ] = Field( 2247 None, 2248 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2249 title="Backoff Strategies", 2250 ) 2251 max_retries: Optional[Union[int, str]] = Field( 2252 5, 2253 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2254 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2255 title="Max Retry Count", 2256 ) 2257 response_filters: Optional[List[HttpResponseFilter]] = Field( 2258 None, 2259 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2260 title="Response Filters", 2261 ) 2262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2263 2264 2265class DefaultPaginator(BaseModel): 2266 type: Literal["DefaultPaginator"] 2267 pagination_strategy: Union[ 2268 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2269 ] = Field( 2270 ..., 2271 description="Strategy defining how records are paginated.", 2272 title="Pagination Strategy", 2273 ) 2274 page_size_option: Optional[RequestOption] = Field( 2275 None, title="Inject Page Size Into Outgoing HTTP Request" 2276 ) 2277 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2278 None, 2279 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2280 title="Inject Page Token Into Outgoing HTTP Request", 2281 ) 2282 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2283 2284 2285class RecordExpander(BaseModel): 2286 type: Literal["RecordExpander"] 2287 expand_records_from_field: List[str] = Field( 2288 ..., 2289 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2290 examples=[ 2291 ["lines", "data"], 2292 ["items"], 2293 ["nested", "array"], 2294 ["sections", "*", "items"], 2295 ], 2296 title="Expand Records From Field", 2297 ) 2298 remain_original_record: Optional[bool] = Field( 2299 False, 2300 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2301 title="Remain Original Record", 2302 ) 2303 on_no_records: Optional[OnNoRecords] = Field( 2304 OnNoRecords.skip, 2305 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2306 title="On No Records", 2307 ) 2308 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2309 2310 2311class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2312 type: Literal["ApiKey"] 2313 inject_into: RequestOption = Field( 2314 ..., 2315 description="Configure how the API Key will be sent in requests to the source API.", 2316 examples=[ 2317 {"inject_into": "header", "field_name": "Authorization"}, 2318 {"inject_into": "request_parameter", "field_name": "authKey"}, 2319 ], 2320 title="Inject API Key Into Outgoing HTTP Request", 2321 ) 2322 api_token: Optional[str] = Field( 2323 "{{ session_token }}", 2324 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2325 examples=[ 2326 "{{ session_token }}", 2327 "Token {{ session_token }}", 2328 "Bearer {{ session_token }}", 2329 ], 2330 title="API Token Template", 2331 ) 2332 2333 2334class JsonSchemaPropertySelector(BaseModel): 2335 type: Literal["JsonSchemaPropertySelector"] 2336 transformations: Optional[ 2337 List[ 2338 Union[ 2339 AddFields, 2340 RemoveFields, 2341 KeysToLower, 2342 KeysToSnakeCase, 2343 FlattenFields, 2344 DpathFlattenFields, 2345 KeysReplace, 2346 CustomTransformation, 2347 ] 2348 ] 2349 ] = Field( 2350 None, 2351 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2352 title="Transformations", 2353 ) 2354 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2355 2356 2357class ListPartitionRouter(BaseModel): 2358 type: Literal["ListPartitionRouter"] 2359 cursor_field: str = Field( 2360 ..., 2361 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2362 examples=["section", "{{ config['section_key'] }}"], 2363 title="Current Partition Value Identifier", 2364 ) 2365 values: Union[str, List[str]] = Field( 2366 ..., 2367 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2368 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2369 title="Partition Values", 2370 ) 2371 request_option: Optional[RequestOption] = Field( 2372 None, 2373 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2374 title="Inject Partition Value Into Outgoing HTTP Request", 2375 ) 2376 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2377 2378 2379class PaginationReset(BaseModel): 2380 type: Literal["PaginationReset"] 2381 action: Action1 2382 limits: Optional[PaginationResetLimits] = None 2383 2384 2385class GzipDecoder(BaseModel): 2386 type: Literal["GzipDecoder"] 2387 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2388 2389 2390class RequestBodyGraphQL(BaseModel): 2391 type: Literal["RequestBodyGraphQL"] 2392 value: RequestBodyGraphQlQuery 2393 2394 2395class DpathValidator(BaseModel): 2396 type: Literal["DpathValidator"] 2397 field_path: List[str] = Field( 2398 ..., 2399 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2400 examples=[ 2401 ["data"], 2402 ["data", "records"], 2403 ["data", "{{ parameters.name }}"], 2404 ["data", "*", "record"], 2405 ], 2406 title="Field Path", 2407 ) 2408 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2409 ..., 2410 description="The condition that the specified config value will be evaluated against", 2411 title="Validation Strategy", 2412 ) 2413 2414 2415class PredicateValidator(BaseModel): 2416 type: Literal["PredicateValidator"] 2417 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2418 ..., 2419 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2420 examples=[ 2421 "test-value", 2422 "{{ config['api_version'] }}", 2423 "{{ config['tenant_id'] }}", 2424 123, 2425 ], 2426 title="Value", 2427 ) 2428 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2429 ..., 2430 description="The validation strategy to apply to the value.", 2431 title="Validation Strategy", 2432 ) 2433 2434 2435class ConfigAddFields(BaseModel): 2436 type: Literal["ConfigAddFields"] 2437 fields: List[AddedFieldDefinition] = Field( 2438 ..., 2439 description="A list of transformations (path and corresponding value) that will be added to the config.", 2440 title="Fields", 2441 ) 2442 condition: Optional[str] = Field( 2443 "", 2444 description="Fields will be added if expression is evaluated to True.", 2445 examples=[ 2446 "{{ config['environemnt'] == 'sandbox' }}", 2447 "{{ property is integer }}", 2448 "{{ property|length > 5 }}", 2449 "{{ property == 'some_string_to_match' }}", 2450 ], 2451 ) 2452 2453 2454class CompositeErrorHandler(BaseModel): 2455 type: Literal["CompositeErrorHandler"] 2456 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2457 Field( 2458 ..., 2459 description="List of error handlers to iterate on to determine how to handle a failed response.", 2460 title="Error Handlers", 2461 ) 2462 ) 2463 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2464 2465 2466class HTTPAPIBudget(BaseModel): 2467 class Config: 2468 extra = Extra.allow 2469 2470 type: Literal["HTTPAPIBudget"] 2471 policies: List[ 2472 Union[ 2473 FixedWindowCallRatePolicy, 2474 MovingWindowCallRatePolicy, 2475 UnlimitedCallRatePolicy, 2476 ] 2477 ] = Field( 2478 ..., 2479 description="List of call rate policies that define how many calls are allowed.", 2480 title="Policies", 2481 ) 2482 ratelimit_reset_header: Optional[str] = Field( 2483 "ratelimit-reset", 2484 description="The HTTP response header name that indicates when the rate limit resets.", 2485 title="Rate Limit Reset Header", 2486 ) 2487 ratelimit_remaining_header: Optional[str] = Field( 2488 "ratelimit-remaining", 2489 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2490 title="Rate Limit Remaining Header", 2491 ) 2492 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2493 [429], 2494 description="List of HTTP status codes that indicate a rate limit has been hit.", 2495 title="Status Codes for Rate Limit Hit", 2496 ) 2497 2498 2499class DpathExtractor(BaseModel): 2500 type: Literal["DpathExtractor"] 2501 field_path: List[str] = Field( 2502 ..., 2503 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2504 examples=[ 2505 ["data"], 2506 ["data", "records"], 2507 ["data", "{{ parameters.name }}"], 2508 ["data", "*", "record"], 2509 ], 2510 title="Field Path", 2511 ) 2512 record_expander: Optional[RecordExpander] = Field( 2513 None, 2514 description="Optional component to expand records by extracting items from nested array fields.", 2515 title="Record Expander", 2516 ) 2517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2518 2519 2520class ZipfileDecoder(BaseModel): 2521 class Config: 2522 extra = Extra.allow 2523 2524 type: Literal["ZipfileDecoder"] 2525 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2526 ..., 2527 description="Parser to parse the decompressed data from the zipfile(s).", 2528 title="Parser", 2529 ) 2530 2531 2532class RecordSelector(BaseModel): 2533 type: Literal["RecordSelector"] 2534 extractor: Union[DpathExtractor, CustomRecordExtractor] 2535 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2536 None, 2537 description="Responsible for filtering records to be emitted by the Source.", 2538 title="Record Filter", 2539 ) 2540 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2541 None, 2542 description="Responsible for normalization according to the schema.", 2543 title="Schema Normalization", 2544 ) 2545 transform_before_filtering: Optional[bool] = Field( 2546 None, 2547 description="If true, transformation will be applied before record filtering.", 2548 title="Transform Before Filtering", 2549 ) 2550 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2551 2552 2553class ConfigMigration(BaseModel): 2554 type: Literal["ConfigMigration"] 2555 description: Optional[str] = Field( 2556 None, description="The description/purpose of the config migration." 2557 ) 2558 transformations: List[ 2559 Union[ 2560 ConfigRemapField, 2561 ConfigAddFields, 2562 ConfigRemoveFields, 2563 CustomConfigTransformation, 2564 ] 2565 ] = Field( 2566 ..., 2567 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2568 title="Transformations", 2569 ) 2570 2571 2572class ConfigNormalizationRules(BaseModel): 2573 class Config: 2574 extra = Extra.forbid 2575 2576 type: Literal["ConfigNormalizationRules"] 2577 config_migrations: Optional[List[ConfigMigration]] = Field( 2578 [], 2579 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2580 title="Config Migrations", 2581 ) 2582 transformations: Optional[ 2583 List[ 2584 Union[ 2585 ConfigRemapField, 2586 ConfigAddFields, 2587 ConfigRemoveFields, 2588 CustomConfigTransformation, 2589 ] 2590 ] 2591 ] = Field( 2592 [], 2593 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2594 title="Transformations", 2595 ) 2596 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2597 [], 2598 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2599 title="Validations", 2600 ) 2601 2602 2603class Spec(BaseModel): 2604 type: Literal["Spec"] 2605 connection_specification: Dict[str, Any] = Field( 2606 ..., 2607 description="A connection specification describing how a the connector can be configured.", 2608 title="Connection Specification", 2609 ) 2610 documentation_url: Optional[str] = Field( 2611 None, 2612 description="URL of the connector's documentation page.", 2613 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2614 title="Documentation URL", 2615 ) 2616 advanced_auth: Optional[AuthFlow] = Field( 2617 None, 2618 description="Advanced specification for configuring the authentication flow.", 2619 title="Advanced Auth", 2620 ) 2621 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2622 None, title="Config Normalization Rules" 2623 ) 2624 2625 2626class DeclarativeSource1(BaseModel): 2627 class Config: 2628 extra = Extra.forbid 2629 2630 type: Literal["DeclarativeSource"] 2631 check: Union[CheckStream, CheckDynamicStream] 2632 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2633 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2634 version: str = Field( 2635 ..., 2636 description="The version of the Airbyte CDK used to build and test the source.", 2637 ) 2638 schemas: Optional[Schemas] = None 2639 definitions: Optional[Dict[str, Any]] = None 2640 spec: Optional[Spec] = None 2641 concurrency_level: Optional[ConcurrencyLevel] = None 2642 api_budget: Optional[HTTPAPIBudget] = None 2643 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2644 None, 2645 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2646 title="Stream Groups", 2647 ) 2648 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2649 None, 2650 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2651 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2652 title="Maximum Concurrent Asynchronous Jobs", 2653 ) 2654 metadata: Optional[Dict[str, Any]] = Field( 2655 None, 2656 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2657 ) 2658 description: Optional[str] = Field( 2659 None, 2660 description="A description of the connector. It will be presented on the Source documentation page.", 2661 ) 2662 2663 2664class DeclarativeSource2(BaseModel): 2665 class Config: 2666 extra = Extra.forbid 2667 2668 type: Literal["DeclarativeSource"] 2669 check: Union[CheckStream, CheckDynamicStream] 2670 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2671 None 2672 ) 2673 dynamic_streams: List[DynamicDeclarativeStream] 2674 version: str = Field( 2675 ..., 2676 description="The version of the Airbyte CDK used to build and test the source.", 2677 ) 2678 schemas: Optional[Schemas] = None 2679 definitions: Optional[Dict[str, Any]] = None 2680 spec: Optional[Spec] = None 2681 concurrency_level: Optional[ConcurrencyLevel] = None 2682 api_budget: Optional[HTTPAPIBudget] = None 2683 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2684 None, 2685 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2686 title="Stream Groups", 2687 ) 2688 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2689 None, 2690 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2691 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2692 title="Maximum Concurrent Asynchronous Jobs", 2693 ) 2694 metadata: Optional[Dict[str, Any]] = Field( 2695 None, 2696 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2697 ) 2698 description: Optional[str] = Field( 2699 None, 2700 description="A description of the connector. It will be presented on the Source documentation page.", 2701 ) 2702 2703 2704class DeclarativeSource(BaseModel): 2705 class Config: 2706 extra = Extra.forbid 2707 2708 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2709 ..., 2710 description="An API source that extracts data according to its declarative components.", 2711 title="DeclarativeSource", 2712 ) 2713 2714 2715class SelectiveAuthenticator(BaseModel): 2716 class Config: 2717 extra = Extra.allow 2718 2719 type: Literal["SelectiveAuthenticator"] 2720 authenticator_selection_path: List[str] = Field( 2721 ..., 2722 description="Path of the field in config with selected authenticator name", 2723 examples=[["auth"], ["auth", "type"]], 2724 title="Authenticator Selection Path", 2725 ) 2726 authenticators: Dict[ 2727 str, 2728 Union[ 2729 ApiKeyAuthenticator, 2730 BasicHttpAuthenticator, 2731 BearerAuthenticator, 2732 OAuthAuthenticator, 2733 JwtAuthenticator, 2734 SessionTokenAuthenticator, 2735 LegacySessionTokenAuthenticator, 2736 CustomAuthenticator, 2737 NoAuth, 2738 RateLimitedMultipleTokenAuthenticator, 2739 ], 2740 ] = Field( 2741 ..., 2742 description="Authenticators to select from.", 2743 examples=[ 2744 { 2745 "authenticators": { 2746 "token": "#/definitions/ApiKeyAuthenticator", 2747 "oauth": "#/definitions/OAuthAuthenticator", 2748 "jwt": "#/definitions/JwtAuthenticator", 2749 } 2750 } 2751 ], 2752 title="Authenticators", 2753 ) 2754 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2755 2756 2757class ConditionalStreams(BaseModel): 2758 type: Literal["ConditionalStreams"] 2759 condition: str = Field( 2760 ..., 2761 description="Condition that will be evaluated to determine if a set of streams should be available.", 2762 examples=["{{ config['is_sandbox'] }}"], 2763 title="Condition", 2764 ) 2765 streams: List[DeclarativeStream] = Field( 2766 ..., 2767 description="Streams that will be used during an operation based on the condition.", 2768 title="Streams", 2769 ) 2770 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2771 2772 2773class FileUploader(BaseModel): 2774 type: Literal["FileUploader"] 2775 requester: Union[HttpRequester, CustomRequester] = Field( 2776 ..., 2777 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2778 ) 2779 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2780 ..., 2781 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2782 ) 2783 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2784 None, 2785 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2786 ) 2787 filename_extractor: Optional[str] = Field( 2788 None, 2789 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2790 examples=[ 2791 "{{ record.id }}/{{ record.file_name }}/", 2792 "{{ record.id }}_{{ record.file_name }}/", 2793 ], 2794 ) 2795 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2796 2797 2798class DeclarativeStream(BaseModel): 2799 class Config: 2800 extra = Extra.allow 2801 2802 type: Literal["DeclarativeStream"] 2803 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2804 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2805 ..., 2806 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2807 title="Retriever", 2808 ) 2809 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2810 None, 2811 description="Component used to fetch data incrementally based on a time field in the data.", 2812 title="Incremental Sync", 2813 ) 2814 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2815 schema_loader: Optional[ 2816 Union[ 2817 InlineSchemaLoader, 2818 DynamicSchemaLoader, 2819 JsonFileSchemaLoader, 2820 List[ 2821 Union[ 2822 InlineSchemaLoader, 2823 DynamicSchemaLoader, 2824 JsonFileSchemaLoader, 2825 CustomSchemaLoader, 2826 ] 2827 ], 2828 CustomSchemaLoader, 2829 ] 2830 ] = Field( 2831 None, 2832 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2833 title="Schema Loader", 2834 ) 2835 transformations: Optional[ 2836 List[ 2837 Union[ 2838 AddFields, 2839 RemoveFields, 2840 KeysToLower, 2841 KeysToSnakeCase, 2842 FlattenFields, 2843 DpathFlattenFields, 2844 KeysReplace, 2845 CustomTransformation, 2846 ] 2847 ] 2848 ] = Field( 2849 None, 2850 description="A list of transformations to be applied to each output record.", 2851 title="Transformations", 2852 ) 2853 state_migrations: Optional[ 2854 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2855 ] = Field( 2856 [], 2857 description="Array of state migrations to be applied on the input state", 2858 title="State Migrations", 2859 ) 2860 file_uploader: Optional[FileUploader] = Field( 2861 None, 2862 description="(experimental) Describes how to fetch a file", 2863 title="File Uploader", 2864 ) 2865 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2866 2867 2868class SessionTokenAuthenticator(BaseModel): 2869 type: Literal["SessionTokenAuthenticator"] 2870 login_requester: HttpRequester = Field( 2871 ..., 2872 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2873 examples=[ 2874 { 2875 "type": "HttpRequester", 2876 "url_base": "https://my_api.com", 2877 "path": "/login", 2878 "authenticator": { 2879 "type": "BasicHttpAuthenticator", 2880 "username": "{{ config.username }}", 2881 "password": "{{ config.password }}", 2882 }, 2883 } 2884 ], 2885 title="Login Requester", 2886 ) 2887 session_token_path: List[str] = Field( 2888 ..., 2889 description="The path in the response body returned from the login requester to the session token.", 2890 examples=[["access_token"], ["result", "token"]], 2891 title="Session Token Path", 2892 ) 2893 expiration_duration: Optional[str] = Field( 2894 None, 2895 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2896 examples=["PT1H", "P1D"], 2897 title="Expiration Duration", 2898 ) 2899 request_authentication: Union[ 2900 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2901 ] = Field( 2902 ..., 2903 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2904 title="Data Request Authentication", 2905 ) 2906 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2907 None, description="Component used to decode the response.", title="Decoder" 2908 ) 2909 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2910 2911 2912class HttpRequester(BaseModelWithDeprecations): 2913 type: Literal["HttpRequester"] 2914 url_base: Optional[str] = Field( 2915 None, 2916 deprecated=True, 2917 deprecation_message="Use `url` field instead.", 2918 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2919 examples=[ 2920 "https://connect.squareup.com/v2", 2921 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2922 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2923 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2924 ], 2925 title="API Base URL", 2926 ) 2927 url: Optional[str] = Field( 2928 None, 2929 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2930 examples=[ 2931 "https://connect.squareup.com/v2", 2932 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2933 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2934 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2935 ], 2936 title="API Endpoint URL", 2937 ) 2938 path: Optional[str] = Field( 2939 None, 2940 deprecated=True, 2941 deprecation_message="Use `url` field instead.", 2942 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2943 examples=[ 2944 "/products", 2945 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2946 "/trades/{{ config['symbol_id'] }}/history", 2947 ], 2948 title="URL Path", 2949 ) 2950 http_method: Optional[HttpMethod] = Field( 2951 HttpMethod.GET, 2952 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2953 examples=["GET", "POST"], 2954 title="HTTP Method", 2955 ) 2956 authenticator: Optional[ 2957 Union[ 2958 ApiKeyAuthenticator, 2959 BasicHttpAuthenticator, 2960 BearerAuthenticator, 2961 OAuthAuthenticator, 2962 JwtAuthenticator, 2963 SessionTokenAuthenticator, 2964 SelectiveAuthenticator, 2965 CustomAuthenticator, 2966 NoAuth, 2967 LegacySessionTokenAuthenticator, 2968 RateLimitedMultipleTokenAuthenticator, 2969 ] 2970 ] = Field( 2971 None, 2972 description="Authentication method to use for requests sent to the API.", 2973 title="Authenticator", 2974 ) 2975 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2976 None, 2977 deprecated=True, 2978 deprecation_message="Use `query_properties` field instead.", 2979 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2980 title="Fetch Properties from Endpoint", 2981 ) 2982 query_properties: Optional[QueryProperties] = Field( 2983 None, 2984 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2985 title="Query Properties", 2986 ) 2987 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2988 None, 2989 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2990 examples=[ 2991 {"unit": "day"}, 2992 { 2993 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2994 }, 2995 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2996 {"sort_by[asc]": "updated_at"}, 2997 ], 2998 title="Query Parameters", 2999 ) 3000 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3001 None, 3002 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3003 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3004 title="Request Headers", 3005 ) 3006 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3007 None, 3008 deprecated=True, 3009 deprecation_message="Use `request_body` field instead.", 3010 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3011 examples=[ 3012 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3013 ], 3014 title="Request Body Payload (Non-JSON)", 3015 ) 3016 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3017 None, 3018 deprecated=True, 3019 deprecation_message="Use `request_body` field instead.", 3020 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3021 examples=[ 3022 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3023 {"key": "{{ config['value'] }}"}, 3024 {"sort": {"field": "updated_at", "order": "ascending"}}, 3025 ], 3026 title="Request Body JSON Payload", 3027 ) 3028 request_body: Optional[ 3029 Union[ 3030 RequestBodyPlainText, 3031 RequestBodyUrlEncodedForm, 3032 RequestBodyJsonObject, 3033 RequestBodyGraphQL, 3034 ] 3035 ] = Field( 3036 None, 3037 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3038 title="Request Body", 3039 ) 3040 error_handler: Optional[ 3041 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3042 ] = Field( 3043 None, 3044 description="Error handler component that defines how to handle errors.", 3045 title="Error Handler", 3046 ) 3047 use_cache: Optional[bool] = Field( 3048 False, 3049 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3050 title="Use Cache", 3051 ) 3052 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3053 3054 3055class DynamicSchemaLoader(BaseModel): 3056 type: Literal["DynamicSchemaLoader"] 3057 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3058 ..., 3059 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3060 title="Retriever", 3061 ) 3062 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3063 None, 3064 description="Responsible for filtering fields to be added to json schema.", 3065 title="Schema Filter", 3066 ) 3067 schema_transformations: Optional[ 3068 List[ 3069 Union[ 3070 AddFields, 3071 RemoveFields, 3072 KeysToLower, 3073 KeysToSnakeCase, 3074 FlattenFields, 3075 DpathFlattenFields, 3076 KeysReplace, 3077 CustomTransformation, 3078 ] 3079 ] 3080 ] = Field( 3081 None, 3082 description="A list of transformations to be applied to the schema.", 3083 title="Schema Transformations", 3084 ) 3085 schema_type_identifier: SchemaTypeIdentifier 3086 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3087 3088 3089class ParentStreamConfig(BaseModel): 3090 type: Literal["ParentStreamConfig"] 3091 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3092 ..., description="Reference to the parent stream.", title="Parent Stream" 3093 ) 3094 parent_key: str = Field( 3095 ..., 3096 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3097 examples=["id", "{{ config['parent_record_id'] }}"], 3098 title="Parent Key", 3099 ) 3100 partition_field: str = Field( 3101 ..., 3102 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3103 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3104 title="Current Parent Key Value Identifier", 3105 ) 3106 request_option: Optional[RequestOption] = Field( 3107 None, 3108 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3109 title="Request Option", 3110 ) 3111 incremental_dependency: Optional[bool] = Field( 3112 False, 3113 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3114 title="Incremental Dependency", 3115 ) 3116 lazy_read_pointer: Optional[List[str]] = Field( 3117 [], 3118 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3119 title="Lazy Read Pointer", 3120 ) 3121 extra_fields: Optional[List[List[str]]] = Field( 3122 None, 3123 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3124 title="Extra Fields", 3125 ) 3126 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3127 3128 3129class PropertiesFromEndpoint(BaseModel): 3130 type: Literal["PropertiesFromEndpoint"] 3131 property_field_path: List[str] = Field( 3132 ..., 3133 description="Describes the path to the field that should be extracted", 3134 examples=[["name"]], 3135 ) 3136 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3137 ..., 3138 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3139 ) 3140 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3141 3142 3143class QueryProperties(BaseModel): 3144 type: Literal["QueryProperties"] 3145 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3146 ..., 3147 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3148 title="Property List", 3149 ) 3150 always_include_properties: Optional[List[str]] = Field( 3151 None, 3152 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3153 title="Always Include Properties", 3154 ) 3155 property_chunking: Optional[PropertyChunking] = Field( 3156 None, 3157 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3158 title="Property Chunking", 3159 ) 3160 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3161 None, 3162 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3163 title="Property Selector", 3164 ) 3165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3166 3167 3168class StateDelegatingStream(BaseModel): 3169 type: Literal["StateDelegatingStream"] 3170 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3171 full_refresh_stream: DeclarativeStream = Field( 3172 ..., 3173 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3174 title="Full Refresh Stream", 3175 ) 3176 incremental_stream: DeclarativeStream = Field( 3177 ..., 3178 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3179 title="Incremental Stream", 3180 ) 3181 api_retention_period: Optional[str] = Field( 3182 None, 3183 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3184 examples=["P30D", "P90D", "P1Y"], 3185 title="API Retention Period", 3186 ) 3187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3188 3189 3190class SimpleRetriever(BaseModel): 3191 type: Literal["SimpleRetriever"] 3192 requester: Union[HttpRequester, CustomRequester] = Field( 3193 ..., 3194 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3195 ) 3196 decoder: Optional[ 3197 Union[ 3198 JsonDecoder, 3199 JsonItemsDecoder, 3200 XmlDecoder, 3201 CsvDecoder, 3202 JsonlDecoder, 3203 GzipDecoder, 3204 IterableDecoder, 3205 ZipfileDecoder, 3206 CustomDecoder, 3207 ] 3208 ] = Field( 3209 None, 3210 description="Component decoding the response so records can be extracted.", 3211 title="HTTP Response Format", 3212 ) 3213 record_selector: RecordSelector = Field( 3214 ..., 3215 description="Component that describes how to extract records from a HTTP response.", 3216 ) 3217 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3218 None, 3219 description="Paginator component that describes how to navigate through the API's pages.", 3220 ) 3221 pagination_reset: Optional[PaginationReset] = Field( 3222 None, 3223 description="Describes what triggers pagination reset and how to handle it.", 3224 ) 3225 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3226 False, 3227 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3228 ) 3229 partition_router: Optional[ 3230 Union[ 3231 SubstreamPartitionRouter, 3232 ListPartitionRouter, 3233 GroupingPartitionRouter, 3234 UnionPartitionRouter, 3235 CustomPartitionRouter, 3236 List[ 3237 Union[ 3238 SubstreamPartitionRouter, 3239 ListPartitionRouter, 3240 GroupingPartitionRouter, 3241 UnionPartitionRouter, 3242 CustomPartitionRouter, 3243 ] 3244 ], 3245 ] 3246 ] = Field( 3247 None, 3248 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3249 title="Partition Router", 3250 ) 3251 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3252 3253 3254class AsyncRetriever(BaseModel): 3255 type: Literal["AsyncRetriever"] 3256 record_selector: RecordSelector = Field( 3257 ..., 3258 description="Component that describes how to extract records from a HTTP response.", 3259 ) 3260 status_mapping: AsyncJobStatusMap = Field( 3261 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3262 ) 3263 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3264 ..., description="Responsible for fetching the actual status of the async job." 3265 ) 3266 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3267 None, 3268 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3269 ) 3270 download_extractor: Optional[ 3271 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3272 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3273 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3274 ..., 3275 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3276 ) 3277 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3278 ..., 3279 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3280 ) 3281 polling_job_timeout: Optional[Union[int, str]] = Field( 3282 None, 3283 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3284 ) 3285 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3286 None, 3287 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3288 ge=1, 3289 ) 3290 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3291 None, 3292 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3293 ) 3294 download_requester: Union[HttpRequester, CustomRequester] = Field( 3295 ..., 3296 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3297 ) 3298 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3299 None, 3300 description="Paginator component that describes how to navigate through the API's pages during download.", 3301 ) 3302 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3303 None, 3304 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3305 ) 3306 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3307 None, 3308 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3309 ) 3310 partition_router: Optional[ 3311 Union[ 3312 ListPartitionRouter, 3313 SubstreamPartitionRouter, 3314 GroupingPartitionRouter, 3315 UnionPartitionRouter, 3316 CustomPartitionRouter, 3317 List[ 3318 Union[ 3319 ListPartitionRouter, 3320 SubstreamPartitionRouter, 3321 GroupingPartitionRouter, 3322 UnionPartitionRouter, 3323 CustomPartitionRouter, 3324 ] 3325 ], 3326 ] 3327 ] = Field( 3328 [], 3329 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3330 title="Partition Router", 3331 ) 3332 decoder: Optional[ 3333 Union[ 3334 CsvDecoder, 3335 GzipDecoder, 3336 JsonDecoder, 3337 JsonItemsDecoder, 3338 JsonlDecoder, 3339 IterableDecoder, 3340 XmlDecoder, 3341 ZipfileDecoder, 3342 CustomDecoder, 3343 ] 3344 ] = Field( 3345 None, 3346 description="Component decoding the response so records can be extracted.", 3347 title="HTTP Response Format", 3348 ) 3349 download_decoder: Optional[ 3350 Union[ 3351 CsvDecoder, 3352 GzipDecoder, 3353 JsonDecoder, 3354 JsonItemsDecoder, 3355 JsonlDecoder, 3356 IterableDecoder, 3357 XmlDecoder, 3358 ZipfileDecoder, 3359 CustomDecoder, 3360 ] 3361 ] = Field( 3362 None, 3363 description="Component decoding the download response so records can be extracted.", 3364 title="Download HTTP Response Format", 3365 ) 3366 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3367 3368 3369class BlockSimultaneousSyncsAction(BaseModel): 3370 type: Literal["BlockSimultaneousSyncsAction"] 3371 3372 3373class StreamGroup(BaseModel): 3374 streams: List[str] = Field( 3375 ..., 3376 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3377 title="Streams", 3378 ) 3379 action: BlockSimultaneousSyncsAction = Field( 3380 ..., 3381 description="The action to apply to streams in this group.", 3382 title="Action", 3383 ) 3384 3385 3386class SubstreamPartitionRouter(BaseModel): 3387 type: Literal["SubstreamPartitionRouter"] 3388 parent_stream_configs: List[ParentStreamConfig] = Field( 3389 ..., 3390 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3391 title="Parent Stream Configs", 3392 ) 3393 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3394 3395 3396class GroupingPartitionRouter(BaseModel): 3397 type: Literal["GroupingPartitionRouter"] 3398 group_size: int = Field( 3399 ..., 3400 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3401 examples=[10, 50], 3402 title="Group Size", 3403 ) 3404 underlying_partition_router: Union[ 3405 ListPartitionRouter, 3406 SubstreamPartitionRouter, 3407 "UnionPartitionRouter", 3408 CustomPartitionRouter, 3409 ] = Field( 3410 ..., 3411 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3412 title="Underlying Partition Router", 3413 ) 3414 deduplicate: Optional[bool] = Field( 3415 True, 3416 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3417 title="Deduplicate Partitions", 3418 ) 3419 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3420 3421 3422class UnionPartitionRouter(BaseModel): 3423 type: Literal["UnionPartitionRouter"] 3424 partition_field: str = Field( 3425 ..., 3426 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3427 examples=["repository", "{{ config['partition_field'] }}"], 3428 title="Partition Field", 3429 ) 3430 partition_routers: List[ 3431 Union[ 3432 ListPartitionRouter, 3433 SubstreamPartitionRouter, 3434 UnionPartitionRouter, 3435 CustomPartitionRouter, 3436 ] 3437 ] = Field( 3438 ..., 3439 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3440 title="Partition Routers", 3441 ) 3442 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3443 3444 3445class HttpComponentsResolver(BaseModel): 3446 type: Literal["HttpComponentsResolver"] 3447 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3448 ..., 3449 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3450 title="Retriever", 3451 ) 3452 components_mapping: List[ComponentMappingDefinition] 3453 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3454 3455 3456class DynamicDeclarativeStream(BaseModel): 3457 type: Literal["DynamicDeclarativeStream"] 3458 name: Optional[str] = Field( 3459 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3460 ) 3461 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3462 ..., description="Reference to the stream template.", title="Stream Template" 3463 ) 3464 components_resolver: Union[ 3465 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3466 ] = Field( 3467 ..., 3468 description="Component resolve and populates stream templates with components values.", 3469 title="Components Resolver", 3470 ) 3471 use_parent_parameters: Optional[bool] = Field( 3472 True, 3473 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3474 title="Use Parent Parameters", 3475 ) 3476 3477 3478ComplexFieldType.update_forward_refs() 3479GzipDecoder.update_forward_refs() 3480CompositeErrorHandler.update_forward_refs() 3481DeclarativeSource1.update_forward_refs() 3482DeclarativeSource2.update_forward_refs() 3483SelectiveAuthenticator.update_forward_refs() 3484ConditionalStreams.update_forward_refs() 3485FileUploader.update_forward_refs() 3486DeclarativeStream.update_forward_refs() 3487SessionTokenAuthenticator.update_forward_refs() 3488HttpRequester.update_forward_refs() 3489DynamicSchemaLoader.update_forward_refs() 3490ParentStreamConfig.update_forward_refs() 3491PropertiesFromEndpoint.update_forward_refs() 3492SimpleRetriever.update_forward_refs() 3493AsyncRetriever.update_forward_refs() 3494GroupingPartitionRouter.update_forward_refs() 3495UnionPartitionRouter.update_forward_refs()
class
AuthFlowType(enum.Enum):
An enumeration.
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
An enumeration.
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 ) 81 config_overrides: Optional[Dict[str, Any]] = Field( 82 None, 83 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 84 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 85 title="Config Overrides", 86 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
89class ConcurrencyLevel(BaseModel): 90 type: Optional[Literal["ConcurrencyLevel"]] = None 91 default_concurrency: Union[int, str] = Field( 92 ..., 93 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 94 examples=[10, "{{ config['num_workers'] or 10 }}"], 95 title="Default Concurrency", 96 ) 97 max_concurrency: Optional[int] = Field( 98 None, 99 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 100 examples=[20, 100], 101 title="Max Concurrency", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
106class ConstantBackoffStrategy(BaseModel): 107 type: Literal["ConstantBackoffStrategy"] 108 backoff_time_in_seconds: Union[float, str] = Field( 109 ..., 110 description="Backoff time in seconds.", 111 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 112 title="Backoff Time", 113 ) 114 jitter_range_in_seconds: Optional[float] = Field( 115 None, 116 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 117 examples=[15], 118 ge=0, 119 title="Jitter Range", 120 ) 121 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
124class CursorPagination(BaseModel): 125 type: Literal["CursorPagination"] 126 cursor_value: str = Field( 127 ..., 128 description="Value of the cursor defining the next page to fetch.", 129 examples=[ 130 "{{ headers.link.next.cursor }}", 131 "{{ last_record['key'] }}", 132 "{{ response['nextPage'] }}", 133 ], 134 title="Cursor Value", 135 ) 136 page_size: Optional[Union[int, str]] = Field( 137 None, 138 description="The number of records to include in each pages.", 139 examples=[100, "{{ config['page_size'] }}"], 140 title="Page Size", 141 ) 142 stop_condition: Optional[str] = Field( 143 None, 144 description="Template string evaluating when to stop paginating.", 145 examples=[ 146 "{{ response.data.has_more is false }}", 147 "{{ 'next' not in headers['link'] }}", 148 ], 149 title="Stop Condition", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
154class CustomAuthenticator(BaseModel): 155 class Config: 156 extra = Extra.allow 157 158 type: Literal["CustomAuthenticator"] 159 class_name: str = Field( 160 ..., 161 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 162 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 163 title="Class Name", 164 ) 165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
168class CustomBackoffStrategy(BaseModel): 169 class Config: 170 extra = Extra.allow 171 172 type: Literal["CustomBackoffStrategy"] 173 class_name: str = Field( 174 ..., 175 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 176 examples=["source_railz.components.MyCustomBackoffStrategy"], 177 title="Class Name", 178 ) 179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
182class CustomErrorHandler(BaseModel): 183 class Config: 184 extra = Extra.allow 185 186 type: Literal["CustomErrorHandler"] 187 class_name: str = Field( 188 ..., 189 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 190 examples=["source_railz.components.MyCustomErrorHandler"], 191 title="Class Name", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
196class CustomPaginationStrategy(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomPaginationStrategy"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomPaginationStrategy"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
210class CustomRecordExtractor(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordExtractor"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomRecordExtractor"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
224class CustomRecordFilter(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRecordFilter"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomCustomRecordFilter"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
238class CustomRequester(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRequester"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRecordExtractor"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
252class CustomRetriever(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomRetriever"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomRetriever"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
266class CustomPartitionRouter(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomPartitionRouter"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomPartitionRouter"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
280class CustomSchemaLoader(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaLoader"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 288 examples=["source_railz.components.MyCustomSchemaLoader"], 289 title="Class Name", 290 ) 291 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
294class CustomSchemaNormalization(BaseModel): 295 class Config: 296 extra = Extra.allow 297 298 type: Literal["CustomSchemaNormalization"] 299 class_name: str = Field( 300 ..., 301 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 302 examples=[ 303 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 304 ], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
310class CustomStateMigration(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomStateMigration"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomStateMigration"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
324class CustomTransformation(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomTransformation"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 332 examples=["source_railz.components.MyCustomTransformation"], 333 title="Class Name", 334 ) 335 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
355class Algorithm(Enum): 356 HS256 = "HS256" 357 HS384 = "HS384" 358 HS512 = "HS512" 359 ES256 = "ES256" 360 ES256K = "ES256K" 361 ES384 = "ES384" 362 ES512 = "ES512" 363 RS256 = "RS256" 364 RS384 = "RS384" 365 RS512 = "RS512" 366 PS256 = "PS256" 367 PS384 = "PS384" 368 PS512 = "PS512" 369 EdDSA = "EdDSA"
An enumeration.
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
372class JwtHeaders(BaseModel): 373 class Config: 374 extra = Extra.forbid 375 376 kid: Optional[str] = Field( 377 None, 378 description="Private key ID for user account.", 379 examples=["{{ config['kid'] }}"], 380 title="Key Identifier", 381 ) 382 typ: Optional[str] = Field( 383 "JWT", 384 description="The media type of the complete JWT.", 385 examples=["JWT"], 386 title="Type", 387 ) 388 cty: Optional[str] = Field( 389 None, 390 description="Content type of JWT header.", 391 examples=["JWT"], 392 title="Content Type", 393 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
396class JwtPayload(BaseModel): 397 class Config: 398 extra = Extra.forbid 399 400 iss: Optional[str] = Field( 401 None, 402 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 403 examples=["{{ config['iss'] }}"], 404 title="Issuer", 405 ) 406 sub: Optional[str] = Field( 407 None, 408 description="The subject of the JWT. Commonly defined by the API.", 409 title="Subject", 410 ) 411 aud: Optional[str] = Field( 412 None, 413 description="The recipient that the JWT is intended for. Commonly defined by the API.", 414 examples=["appstoreconnect-v1"], 415 title="Audience", 416 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
419class RefreshTokenUpdater(BaseModel): 420 refresh_token_name: Optional[str] = Field( 421 "refresh_token", 422 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 423 examples=["refresh_token"], 424 title="Refresh Token Property Name", 425 ) 426 access_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "access_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "access_token"], ["access_token"]], 430 title="Config Path To Access Token", 431 ) 432 refresh_token_config_path: Optional[List[str]] = Field( 433 ["credentials", "refresh_token"], 434 description="Config path to the access token. Make sure the field actually exists in the config.", 435 examples=[["credentials", "refresh_token"], ["refresh_token"]], 436 title="Config Path To Refresh Token", 437 ) 438 token_expiry_date_config_path: Optional[List[str]] = Field( 439 ["credentials", "token_expiry_date"], 440 description="Config path to the expiry date. Make sure actually exists in the config.", 441 examples=[["credentials", "token_expiry_date"]], 442 title="Config Path To Expiry Date", 443 ) 444 refresh_token_error_status_codes: Optional[List[int]] = Field( 445 [], 446 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 447 examples=[[400, 500]], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 449 ) 450 refresh_token_error_key: Optional[str] = Field( 451 "", 452 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 453 examples=["error"], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 455 ) 456 refresh_token_error_values: Optional[List[str]] = Field( 457 [], 458 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 459 examples=[["invalid_grant", "invalid_permissions"]], 460 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 461 )
class
Rate(pydantic.v1.main.BaseModel):
464class Rate(BaseModel): 465 class Config: 466 extra = Extra.allow 467 468 limit: Union[int, str] = Field( 469 ..., 470 description="The maximum number of calls allowed within the interval.", 471 title="Limit", 472 ) 473 interval: str = Field( 474 ..., 475 description="The time interval for the rate limit.", 476 examples=["PT1H", "P1D"], 477 title="Interval", 478 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
481class HttpRequestRegexMatcher(BaseModel): 482 class Config: 483 extra = Extra.allow 484 485 method: Optional[str] = Field( 486 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 487 ) 488 url_base: Optional[str] = Field( 489 None, 490 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 491 title="URL Base", 492 ) 493 url_path_pattern: Optional[str] = Field( 494 None, 495 description="A regular expression pattern to match the URL path.", 496 title="URL Path Pattern", 497 ) 498 params: Optional[Dict[str, Any]] = Field( 499 None, description="The query parameters to match.", title="Parameters" 500 ) 501 headers: Optional[Dict[str, Any]] = Field( 502 None, description="The headers to match.", title="Headers" 503 ) 504 weight: Optional[Union[int, str]] = Field( 505 None, 506 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 507 title="Weight", 508 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
511class ResponseToFileExtractor(BaseModel): 512 type: Literal["ResponseToFileExtractor"] 513 preserve_na_values: Optional[bool] = Field( 514 False, 515 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 516 title="Preserve NA Values", 517 ) 518 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
An enumeration.
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
526class ExponentialBackoffStrategy(BaseModel): 527 type: Literal["ExponentialBackoffStrategy"] 528 factor: Optional[Union[float, str]] = Field( 529 5, 530 description="Multiplicative constant applied on each retry.", 531 examples=[5, 5.5, "10"], 532 title="Factor", 533 ) 534 jitter_range_in_seconds: Optional[float] = Field( 535 None, 536 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 537 examples=[2], 538 ge=0, 539 title="Jitter Range", 540 ) 541 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
544class GroupByKeyMergeStrategy(BaseModel): 545 type: Literal["GroupByKeyMergeStrategy"] 546 key: Union[str, List[str]] = Field( 547 ..., 548 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 549 examples=["id", ["parent_id", "end_date"]], 550 title="Key", 551 ) 552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
An enumeration.
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
QuotaStatusSource(pydantic.v1.main.BaseModel):
564class QuotaStatusSource(BaseModel): 565 type: Literal["QuotaStatusSource"] 566 url: str = Field( 567 ..., 568 description="The full URL of the quota status endpoint.", 569 examples=[ 570 "https://api.github.com/rate_limit", 571 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 572 ], 573 title="URL", 574 ) 575 http_method: Optional[HttpMethod] = Field( 576 HttpMethod.GET, 577 description="The HTTP method used to fetch the quota status.", 578 title="HTTP Method", 579 ) 580 request_headers: Optional[Dict[str, str]] = Field( 581 None, 582 description="Additional headers to send with the quota status request.", 583 title="Request Headers", 584 ) 585 unavailable_status_codes: Optional[List[int]] = Field( 586 None, 587 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 588 examples=[[404]], 589 title="Unavailable Status Codes", 590 unique_items=True, 591 ) 592 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
http_method: Optional[HttpMethod]
class
TokenQuota(pydantic.v1.main.BaseModel):
595class TokenQuota(BaseModel): 596 type: Literal["TokenQuota"] 597 name: str = Field( 598 ..., 599 description="Name of the quota pool.", 600 examples=["rest", "graphql"], 601 title="Name", 602 ) 603 remaining_path: List[str] = Field( 604 ..., 605 description="Path to the remaining call count for this pool in the quota status response.", 606 examples=[["resources", "core", "remaining"]], 607 title="Remaining Path", 608 ) 609 reset_path: List[str] = Field( 610 ..., 611 description="Path to the quota reset timestamp for this pool in the quota status response.", 612 examples=[["resources", "core", "reset"]], 613 title="Reset Path", 614 ) 615 limit_path: Optional[List[str]] = Field( 616 None, 617 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 618 examples=[["resources", "core", "limit"]], 619 title="Limit Path", 620 ) 621 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 622 None, 623 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 624 title="Matchers", 625 ) 626 remaining_header: Optional[str] = Field( 627 None, 628 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 629 examples=["X-RateLimit-Remaining"], 630 title="Remaining Header", 631 ) 632 reset_header: Optional[str] = Field( 633 None, 634 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 635 examples=["X-RateLimit-Reset"], 636 title="Reset Header", 637 ) 638 limit_header: Optional[str] = Field( 639 None, 640 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 641 examples=["X-RateLimit-Limit"], 642 title="Limit Header", 643 ) 644 exhaustion_status_codes: Optional[List[int]] = Field( 645 None, 646 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 647 examples=[[429]], 648 title="Exhaustion Status Codes", 649 ) 650 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
matchers: Optional[List[HttpRequestRegexMatcher]]
class
RateLimitedMultipleTokenAuthenticator(pydantic.v1.main.BaseModel):
653class RateLimitedMultipleTokenAuthenticator(BaseModel): 654 type: Literal["RateLimitedMultipleTokenAuthenticator"] 655 tokens: Union[str, List[str]] = Field( 656 ..., 657 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 658 examples=[ 659 "{{ config['credentials']['personal_access_token'] }}", 660 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 661 ], 662 title="Tokens", 663 ) 664 token_delimiter: Optional[str] = Field( 665 ",", 666 description="Delimiter used to split a single token string into multiple tokens.", 667 title="Token Delimiter", 668 ) 669 auth_method: Optional[str] = Field( 670 "Bearer", 671 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 672 examples=["Bearer", "token"], 673 title="Auth Method", 674 ) 675 header: Optional[str] = Field( 676 "Authorization", 677 description="The name of the HTTP header in which to inject the token.", 678 title="Header Name", 679 ) 680 quota_status_source: QuotaStatusSource = Field( 681 ..., 682 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 683 title="Quota Status Source", 684 ) 685 quotas: List[TokenQuota] = Field( 686 ..., 687 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 688 min_items=1, 689 title="Quota Pools", 690 ) 691 max_wait_time: Optional[str] = Field( 692 "PT2H", 693 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 694 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 695 title="Maximum Wait Time", 696 ) 697 budget_reserve_fraction: Optional[float] = Field( 698 0.1, 699 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 700 title="Budget Reserve Fraction", 701 ) 702 budget_min_reserve: Optional[int] = Field( 703 50, 704 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 705 title="Budget Minimum Reserve", 706 ) 707 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
quota_status_source: QuotaStatusSource
quotas: List[TokenQuota]
class
Action(enum.Enum):
710class Action(Enum): 711 SUCCESS = "SUCCESS" 712 FAIL = "FAIL" 713 RETRY = "RETRY" 714 IGNORE = "IGNORE" 715 RESET_PAGINATION = "RESET_PAGINATION" 716 RATE_LIMITED = "RATE_LIMITED" 717 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
An enumeration.
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
720class FailureType(Enum): 721 system_error = "system_error" 722 config_error = "config_error" 723 transient_error = "transient_error"
An enumeration.
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
726class HttpResponseFilter(BaseModel): 727 type: Literal["HttpResponseFilter"] 728 action: Optional[Action] = Field( 729 None, 730 description="Action to execute if a response matches the filter.", 731 examples=[ 732 "SUCCESS", 733 "FAIL", 734 "RETRY", 735 "IGNORE", 736 "RESET_PAGINATION", 737 "RATE_LIMITED", 738 "REFRESH_TOKEN_THEN_RETRY", 739 ], 740 title="Action", 741 ) 742 failure_type: Optional[FailureType] = Field( 743 None, 744 description="Failure type of traced exception if a response matches the filter.", 745 examples=["system_error", "config_error", "transient_error"], 746 title="Failure Type", 747 ) 748 error_message: Optional[str] = Field( 749 None, 750 description="Error Message to display if the response matches the filter.", 751 title="Error Message", 752 ) 753 error_message_contains: Optional[str] = Field( 754 None, 755 description="Match the response if its error message contains the substring.", 756 example=["This API operation is not enabled for this site"], 757 title="Error Message Substring", 758 ) 759 http_codes: Optional[List[int]] = Field( 760 None, 761 description="Match the response if its HTTP code is included in this list.", 762 examples=[[420, 429], [500]], 763 title="HTTP Codes", 764 unique_items=True, 765 ) 766 predicate: Optional[str] = Field( 767 None, 768 description="Match the response if the predicate evaluates to true.", 769 examples=[ 770 "{{ 'Too much requests' in response }}", 771 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 772 ], 773 title="Predicate", 774 ) 775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
778class ComplexFieldType(BaseModel): 779 field_type: str 780 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
783class TypesMap(BaseModel): 784 target_type: Union[str, List[str], ComplexFieldType] 785 current_type: Union[str, List[str]] 786 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
789class SchemaTypeIdentifier(BaseModel): 790 type: Optional[Literal["SchemaTypeIdentifier"]] = None 791 schema_pointer: Optional[List[str]] = Field( 792 [], 793 description="List of nested fields defining the schema field path to extract. Defaults to [].", 794 title="Schema Path", 795 ) 796 key_pointer: List[str] = Field( 797 ..., 798 description="List of potentially nested fields describing the full path of the field key to extract.", 799 title="Key Path", 800 ) 801 type_pointer: Optional[List[str]] = Field( 802 None, 803 description="List of potentially nested fields describing the full path of the field type to extract.", 804 title="Type Path", 805 ) 806 types_mapping: Optional[List[TypesMap]] = None 807 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
810class InlineSchemaLoader(BaseModel): 811 type: Literal["InlineSchemaLoader"] 812 schema_: Optional[Dict[str, Any]] = Field( 813 None, 814 alias="schema", 815 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 816 title="Schema", 817 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
820class JsonFileSchemaLoader(BaseModel): 821 type: Literal["JsonFileSchemaLoader"] 822 file_path: Optional[str] = Field( 823 None, 824 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 825 example=["./schemas/users.json"], 826 title="File Path", 827 ) 828 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
835class JsonItemsDecoder(BaseModel): 836 type: Literal["JsonItemsDecoder"] 837 items_path: str = Field( 838 ..., 839 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 840 title="Items Path", 841 ) 842 encoding: Optional[str] = Field( 843 "utf-8", 844 description="The character encoding of the JSON data. Defaults to UTF-8.", 845 title="Encoding", 846 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
863class FlattenFields(BaseModel): 864 type: Literal["FlattenFields"] 865 flatten_lists: Optional[bool] = Field( 866 True, 867 description="Whether to flatten lists or leave it as is. Default is True.", 868 title="Flatten Lists", 869 ) 870 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
873class KeyTransformation(BaseModel): 874 type: Literal["KeyTransformation"] 875 prefix: Optional[str] = Field( 876 None, 877 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 878 examples=["flattened_"], 879 title="Key Prefix", 880 ) 881 suffix: Optional[str] = Field( 882 None, 883 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 884 examples=["_flattened"], 885 title="Key Suffix", 886 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
889class DpathFlattenFields(BaseModel): 890 type: Literal["DpathFlattenFields"] 891 field_path: List[str] = Field( 892 ..., 893 description="A path to field that needs to be flattened.", 894 examples=[["data"], ["data", "*", "field"]], 895 title="Field Path", 896 ) 897 delete_origin_value: Optional[bool] = Field( 898 None, 899 description="Whether to delete the origin value or keep it. Default is False.", 900 title="Delete Origin Value", 901 ) 902 replace_record: Optional[bool] = Field( 903 None, 904 description="Whether to replace the origin record or not. Default is False.", 905 title="Replace Origin Record", 906 ) 907 key_transformation: Optional[KeyTransformation] = Field( 908 None, 909 description="Transformation for object keys. If not provided, original key will be used.", 910 title="Key transformation", 911 ) 912 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
915class KeysReplace(BaseModel): 916 type: Literal["KeysReplace"] 917 old: str = Field( 918 ..., 919 description="Old value to replace.", 920 examples=[ 921 " ", 922 "{{ record.id }}", 923 "{{ config['id'] }}", 924 "{{ stream_slice['id'] }}", 925 ], 926 title="Old value", 927 ) 928 new: str = Field( 929 ..., 930 description="New value to set.", 931 examples=[ 932 "_", 933 "{{ record.id }}", 934 "{{ config['id'] }}", 935 "{{ stream_slice['id'] }}", 936 ], 937 title="New value", 938 ) 939 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
950class CustomDecoder(BaseModel): 951 class Config: 952 extra = Extra.allow 953 954 type: Literal["CustomDecoder"] 955 class_name: str = Field( 956 ..., 957 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 958 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 959 title="Class Name", 960 ) 961 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
964class MinMaxDatetime(BaseModel): 965 type: Literal["MinMaxDatetime"] 966 datetime: str = Field( 967 ..., 968 description="Datetime value.", 969 examples=[ 970 "2021-01-01", 971 "2021-01-01T00:00:00Z", 972 "{{ config['start_time'] }}", 973 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 974 ], 975 title="Datetime", 976 ) 977 datetime_format: Optional[str] = Field( 978 "", 979 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 980 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 981 title="Datetime Format", 982 ) 983 max_datetime: Optional[str] = Field( 984 None, 985 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 986 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 987 title="Max Datetime", 988 ) 989 min_datetime: Optional[str] = Field( 990 None, 991 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 992 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 993 title="Min Datetime", 994 ) 995 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1025class OauthConnectorInputSpecification(BaseModel): 1026 class Config: 1027 extra = Extra.allow 1028 1029 consent_url: str = Field( 1030 ..., 1031 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1032 examples=[ 1033 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1034 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1035 ], 1036 title="Consent URL", 1037 ) 1038 scope: Optional[str] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1041 examples=["user:read user:read_orders workspaces:read"], 1042 title="Scopes", 1043 ) 1044 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1045 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1046 # The CDK schema defines the manifest contract; the platform reads these fields 1047 # during the OAuth consent flow to build the authorization URL. 1048 scopes: Optional[List[OAuthScope]] = Field( 1049 None, 1050 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1051 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1052 title="Scopes", 1053 ) 1054 optional_scopes: Optional[List[OAuthScope]] = Field( 1055 None, 1056 description="Optional OAuth scope objects that may or may not be granted.", 1057 examples=[[{"scope": "admin:read"}]], 1058 title="Optional Scopes", 1059 ) 1060 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1061 ScopesJoinStrategy.space, 1062 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1063 title="Scopes Join Strategy", 1064 ) 1065 access_token_url: str = Field( 1066 ..., 1067 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1068 examples=[ 1069 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1070 ], 1071 title="Access Token URL", 1072 ) 1073 access_token_headers: Optional[Dict[str, Any]] = Field( 1074 None, 1075 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1076 examples=[ 1077 { 1078 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1079 } 1080 ], 1081 title="Access Token Headers", 1082 ) 1083 access_token_params: Optional[Dict[str, Any]] = Field( 1084 None, 1085 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1086 examples=[ 1087 { 1088 "{{ auth_code_key }}": "{{ auth_code_value }}", 1089 "{{ client_id_key }}": "{{ client_id_value }}", 1090 "{{ client_secret_key }}": "{{ client_secret_value }}", 1091 } 1092 ], 1093 title="Access Token Query Params (Json Encoded)", 1094 ) 1095 extract_output: Optional[List[str]] = Field( 1096 None, 1097 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1098 examples=[["access_token", "refresh_token", "other_field"]], 1099 title="Extract Output", 1100 ) 1101 state: Optional[State] = Field( 1102 None, 1103 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1104 examples=[{"min": 7, "max": 128}], 1105 title="Configurable State Query Param", 1106 ) 1107 client_id_key: Optional[str] = Field( 1108 None, 1109 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1110 examples=["my_custom_client_id_key_name"], 1111 title="Client ID Key Override", 1112 ) 1113 client_secret_key: Optional[str] = Field( 1114 None, 1115 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1116 examples=["my_custom_client_secret_key_name"], 1117 title="Client Secret Key Override", 1118 ) 1119 scope_key: Optional[str] = Field( 1120 None, 1121 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1122 examples=["my_custom_scope_key_key_name"], 1123 title="Scopes Key Override", 1124 ) 1125 state_key: Optional[str] = Field( 1126 None, 1127 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1128 examples=["my_custom_state_key_key_name"], 1129 title="State Key Override", 1130 ) 1131 auth_code_key: Optional[str] = Field( 1132 None, 1133 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1134 examples=["my_custom_auth_code_key_name"], 1135 title="Auth Code Key Override", 1136 ) 1137 redirect_uri_key: Optional[str] = Field( 1138 None, 1139 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1140 examples=["my_custom_redirect_uri_key_name"], 1141 title="Redirect URI Key Override", 1142 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1145class OAuthConfigSpecification(BaseModel): 1146 class Config: 1147 extra = Extra.allow 1148 1149 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1150 None, 1151 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1152 examples=[ 1153 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1154 { 1155 "app_id": { 1156 "type": "string", 1157 "path_in_connector_config": ["info", "app_id"], 1158 } 1159 }, 1160 ], 1161 title="OAuth user input", 1162 ) 1163 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1164 None, 1165 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1166 title="DeclarativeOAuth Connector Specification", 1167 ) 1168 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1169 None, 1170 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1171 examples=[ 1172 { 1173 "refresh_token": { 1174 "type": "string,", 1175 "path_in_connector_config": ["credentials", "refresh_token"], 1176 } 1177 } 1178 ], 1179 title="OAuth output specification", 1180 ) 1181 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1182 None, 1183 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1184 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1185 title="OAuth input specification", 1186 ) 1187 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1188 None, 1189 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1190 examples=[ 1191 { 1192 "client_id": { 1193 "type": "string,", 1194 "path_in_connector_config": ["credentials", "client_id"], 1195 }, 1196 "client_secret": { 1197 "type": "string,", 1198 "path_in_connector_config": ["credentials", "client_secret"], 1199 }, 1200 } 1201 ], 1202 title="OAuth server output specification", 1203 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1206class OffsetIncrement(BaseModel): 1207 type: Literal["OffsetIncrement"] 1208 page_size: Optional[Union[int, str]] = Field( 1209 None, 1210 description="The number of records to include in each pages.", 1211 examples=[100, "{{ config['page_size'] }}"], 1212 title="Limit", 1213 ) 1214 inject_on_first_request: Optional[bool] = Field( 1215 False, 1216 description="Using the `offset` with value `0` during the first request", 1217 title="Inject Offset on First Request", 1218 ) 1219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1222class PageIncrement(BaseModel): 1223 type: Literal["PageIncrement"] 1224 page_size: Optional[Union[int, str]] = Field( 1225 None, 1226 description="The number of records to include in each pages.", 1227 examples=[100, "100", "{{ config['page_size'] }}"], 1228 title="Page Size", 1229 ) 1230 start_from_page: Optional[int] = Field( 1231 0, 1232 description="Index of the first page to request.", 1233 examples=[0, 1], 1234 title="Start From Page", 1235 ) 1236 inject_on_first_request: Optional[bool] = Field( 1237 False, 1238 description="Using the `page number` with value defined by `start_from_page` during the first request", 1239 title="Inject Page Number on First Request", 1240 ) 1241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1244class PrimaryKey(BaseModel): 1245 __root__: Union[str, List[str], List[List[str]]] = Field( 1246 ..., 1247 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1248 examples=["id", ["code", "type"]], 1249 title="Primary Key", 1250 )
class
PropertyLimitType(enum.Enum):
1253class PropertyLimitType(Enum): 1254 characters = "characters" 1255 property_count = "property_count"
An enumeration.
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1258class PropertyChunking(BaseModel): 1259 type: Literal["PropertyChunking"] 1260 property_limit_type: PropertyLimitType = Field( 1261 ..., 1262 description="The type used to determine the maximum number of properties per chunk", 1263 title="Property Limit Type", 1264 ) 1265 property_limit: Optional[int] = Field( 1266 None, 1267 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1268 title="Property Limit", 1269 ) 1270 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1271 None, 1272 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1273 title="Record Merge Strategy", 1274 ) 1275 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1278class RecordFilter(BaseModel): 1279 type: Literal["RecordFilter"] 1280 condition: Optional[str] = Field( 1281 "", 1282 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1283 examples=[ 1284 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1285 "{{ record.status in ['active', 'expired'] }}", 1286 ], 1287 title="Condition", 1288 ) 1289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
An enumeration.
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1297class RemoveFields(BaseModel): 1298 type: Literal["RemoveFields"] 1299 condition: Optional[str] = Field( 1300 "", 1301 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1302 examples=[ 1303 "{{ property|string == '' }}", 1304 "{{ property is integer }}", 1305 "{{ property|length > 5 }}", 1306 "{{ property == 'some_string_to_match' }}", 1307 ], 1308 ) 1309 field_pointers: List[List[str]] = Field( 1310 ..., 1311 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1312 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1313 title="Field Paths", 1314 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1321class InjectInto(Enum): 1322 request_parameter = "request_parameter" 1323 header = "header" 1324 body_data = "body_data" 1325 body_json = "body_json"
An enumeration.
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1328class RequestOption(BaseModel): 1329 type: Literal["RequestOption"] 1330 inject_into: InjectInto = Field( 1331 ..., 1332 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1333 examples=["request_parameter", "header", "body_data", "body_json"], 1334 title="Inject Into", 1335 ) 1336 field_name: Optional[str] = Field( 1337 None, 1338 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1339 examples=["segment_id"], 1340 title="Field Name", 1341 ) 1342 field_path: Optional[List[str]] = Field( 1343 None, 1344 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1345 examples=[["data", "viewer", "id"]], 1346 title="Field Path", 1347 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1357class LegacySessionTokenAuthenticator(BaseModel): 1358 type: Literal["LegacySessionTokenAuthenticator"] 1359 header: str = Field( 1360 ..., 1361 description="The name of the session token header that will be injected in the request", 1362 examples=["X-Session"], 1363 title="Session Request Header", 1364 ) 1365 login_url: str = Field( 1366 ..., 1367 description="Path of the login URL (do not include the base URL)", 1368 examples=["session"], 1369 title="Login Path", 1370 ) 1371 session_token: Optional[str] = Field( 1372 None, 1373 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1374 example=["{{ config['session_token'] }}"], 1375 title="Session Token", 1376 ) 1377 session_token_response_key: str = Field( 1378 ..., 1379 description="Name of the key of the session token to be extracted from the response", 1380 examples=["id"], 1381 title="Response Token Response Key", 1382 ) 1383 username: Optional[str] = Field( 1384 None, 1385 description="Username used to authenticate and obtain a session token", 1386 examples=[" {{ config['username'] }}"], 1387 title="Username", 1388 ) 1389 password: Optional[str] = Field( 1390 "", 1391 description="Password used to authenticate and obtain a session token", 1392 examples=["{{ config['password'] }}", ""], 1393 title="Password", 1394 ) 1395 validate_session_url: str = Field( 1396 ..., 1397 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1398 examples=["user/current"], 1399 title="Validate Session Path", 1400 ) 1401 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
An enumeration.
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1430class ValueType(Enum): 1431 string = "string" 1432 number = "number" 1433 integer = "integer" 1434 boolean = "boolean"
An enumeration.
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1437class WaitTimeFromHeader(BaseModel): 1438 type: Literal["WaitTimeFromHeader"] 1439 header: str = Field( 1440 ..., 1441 description="The name of the response header defining how long to wait before retrying.", 1442 examples=["Retry-After"], 1443 title="Response Header Name", 1444 ) 1445 regex: Optional[str] = Field( 1446 None, 1447 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1448 examples=["([-+]?\\d+)"], 1449 title="Extraction Regex", 1450 ) 1451 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1452 None, 1453 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1454 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1455 title="Max Waiting Time in Seconds", 1456 ) 1457 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1460class WaitUntilTimeFromHeader(BaseModel): 1461 type: Literal["WaitUntilTimeFromHeader"] 1462 header: str = Field( 1463 ..., 1464 description="The name of the response header defining how long to wait before retrying.", 1465 examples=["wait_time"], 1466 title="Response Header", 1467 ) 1468 min_wait: Optional[Union[float, str]] = Field( 1469 None, 1470 description="Minimum time to wait before retrying.", 1471 examples=[10, "60"], 1472 title="Minimum Wait Time", 1473 ) 1474 regex: Optional[str] = Field( 1475 None, 1476 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1477 examples=["([-+]?\\d+)"], 1478 title="Extraction Regex", 1479 ) 1480 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1481 None, 1482 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1483 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1484 title="Max Waiting Time in Seconds", 1485 ) 1486 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1489class ComponentMappingDefinition(BaseModel): 1490 type: Literal["ComponentMappingDefinition"] 1491 field_path: List[str] = Field( 1492 ..., 1493 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1494 examples=[ 1495 ["name"], 1496 ["retriever", "requester", "url"], 1497 ["retriever", "requester", "{{ components_values.field }}"], 1498 ["*", "**", "name"], 1499 ], 1500 title="Field Path", 1501 ) 1502 value: str = Field( 1503 ..., 1504 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1505 examples=[ 1506 "{{ components_values['updates'] }}", 1507 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1508 "{{ config['segment_id'] }}", 1509 "{{ stream_slice['parent_id'] }}", 1510 "{{ stream_slice['extra_fields']['name'] }}", 1511 ], 1512 title="Value", 1513 ) 1514 value_type: Optional[ValueType] = Field( 1515 None, 1516 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1517 title="Value Type", 1518 ) 1519 create_or_update: Optional[bool] = Field( 1520 False, 1521 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1522 title="Create or Update", 1523 ) 1524 condition: Optional[str] = Field( 1525 None, 1526 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1527 examples=[ 1528 "{{ components_values.get('cursor_field', None) }}", 1529 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1530 ], 1531 title="Condition", 1532 ) 1533 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1536class StreamConfig(BaseModel): 1537 type: Literal["StreamConfig"] 1538 configs_pointer: List[str] = Field( 1539 ..., 1540 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1541 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1542 title="Configs Pointer", 1543 ) 1544 default_values: Optional[List[Dict[str, Any]]] = Field( 1545 None, 1546 description="A list of default values, each matching the structure expected from the parsed component value.", 1547 title="Default Values", 1548 ) 1549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1552class ConfigComponentsResolver(BaseModel): 1553 type: Literal["ConfigComponentsResolver"] 1554 stream_config: Union[List[StreamConfig], StreamConfig] 1555 components_mapping: List[ComponentMappingDefinition] 1556 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1559class StreamParametersDefinition(BaseModel): 1560 type: Literal["StreamParametersDefinition"] 1561 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1562 ..., 1563 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1564 examples=[ 1565 [ 1566 { 1567 "name": "test stream", 1568 "$parameters": {"entity": "test entity"}, 1569 "primary_key": "test key", 1570 } 1571 ] 1572 ], 1573 title="Stream Parameters", 1574 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1577class ParametrizedComponentsResolver(BaseModel): 1578 type: Literal["ParametrizedComponentsResolver"] 1579 stream_parameters: StreamParametersDefinition 1580 components_mapping: List[ComponentMappingDefinition] 1581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1606class ValidateAdheresToSchema(BaseModel): 1607 type: Literal["ValidateAdheresToSchema"] 1608 base_schema: Union[str, Dict[str, Any]] = Field( 1609 ..., 1610 description="The base JSON schema against which the user-provided schema will be validated.", 1611 examples=[ 1612 "{{ config['report_validation_schema'] }}", 1613 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1614 { 1615 "$schema": "http://json-schema.org/draft-07/schema#", 1616 "title": "Person", 1617 "type": "object", 1618 "properties": { 1619 "name": {"type": "string", "description": "The person's name"}, 1620 "age": { 1621 "type": "integer", 1622 "minimum": 0, 1623 "description": "The person's age", 1624 }, 1625 }, 1626 "required": ["name", "age"], 1627 }, 1628 ], 1629 title="Base JSON Schema", 1630 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1633class CustomValidationStrategy(BaseModel): 1634 class Config: 1635 extra = Extra.allow 1636 1637 type: Literal["CustomValidationStrategy"] 1638 class_name: str = Field( 1639 ..., 1640 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1641 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1642 title="Class Name", 1643 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1646class ConfigRemapField(BaseModel): 1647 type: Literal["ConfigRemapField"] 1648 map: Union[Dict[str, Any], str] = Field( 1649 ..., 1650 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1651 examples=[ 1652 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1653 "{{ config['status_mapping'] }}", 1654 ], 1655 title="Value Mapping", 1656 ) 1657 field_path: List[str] = Field( 1658 ..., 1659 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1660 examples=[ 1661 ["status"], 1662 ["data", "status"], 1663 ["data", "{{ config.name }}", "status"], 1664 ["data", "*", "status"], 1665 ], 1666 title="Field Path", 1667 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1670class ConfigRemoveFields(BaseModel): 1671 type: Literal["ConfigRemoveFields"] 1672 field_pointers: List[List[str]] = Field( 1673 ..., 1674 description="A list of field pointers to be removed from the config.", 1675 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1676 title="Field Pointers", 1677 ) 1678 condition: Optional[str] = Field( 1679 "", 1680 description="Fields will be removed if expression is evaluated to True.", 1681 examples=[ 1682 "{{ config['environemnt'] == 'sandbox' }}", 1683 "{{ property is integer }}", 1684 "{{ property|length > 5 }}", 1685 "{{ property == 'some_string_to_match' }}", 1686 ], 1687 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1690class CustomConfigTransformation(BaseModel): 1691 type: Literal["CustomConfigTransformation"] 1692 class_name: str = Field( 1693 ..., 1694 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1695 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1696 ) 1697 parameters: Optional[Dict[str, Any]] = Field( 1698 None, 1699 alias="$parameters", 1700 description="Additional parameters to be passed to the custom config transformation.", 1701 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1704class AddedFieldDefinition(BaseModel): 1705 type: Literal["AddedFieldDefinition"] 1706 path: List[str] = Field( 1707 ..., 1708 description="List of strings defining the path where to add the value on the record.", 1709 examples=[["segment_id"], ["metadata", "segment_id"]], 1710 title="Path", 1711 ) 1712 value: str = Field( 1713 ..., 1714 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1715 examples=[ 1716 "{{ record['updates'] }}", 1717 "{{ record['MetaData']['LastUpdatedTime'] }}", 1718 "{{ stream_partition['segment_id'] }}", 1719 ], 1720 title="Value", 1721 ) 1722 value_type: Optional[ValueType] = Field( 1723 None, 1724 description="Type of the value. If not specified, the type will be inferred from the value.", 1725 title="Value Type", 1726 ) 1727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1730class AddFields(BaseModel): 1731 type: Literal["AddFields"] 1732 fields: List[AddedFieldDefinition] = Field( 1733 ..., 1734 description="List of transformations (path and corresponding value) that will be added to the record.", 1735 title="Fields", 1736 ) 1737 condition: Optional[str] = Field( 1738 "", 1739 description="Fields will be added if expression is evaluated to True.", 1740 examples=[ 1741 "{{ property|string == '' }}", 1742 "{{ property is integer }}", 1743 "{{ property|length > 5 }}", 1744 "{{ property == 'some_string_to_match' }}", 1745 ], 1746 ) 1747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1750class ApiKeyAuthenticator(BaseModel): 1751 type: Literal["ApiKeyAuthenticator"] 1752 api_token: Optional[str] = Field( 1753 None, 1754 description="The API key to inject in the request. Fill it in the user inputs.", 1755 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1756 title="API Key", 1757 ) 1758 header: Optional[str] = Field( 1759 None, 1760 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1761 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1762 title="Header Name", 1763 ) 1764 inject_into: Optional[RequestOption] = Field( 1765 None, 1766 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1767 examples=[ 1768 {"inject_into": "header", "field_name": "Authorization"}, 1769 {"inject_into": "request_parameter", "field_name": "authKey"}, 1770 ], 1771 title="Inject API Key Into Outgoing HTTP Request", 1772 ) 1773 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1776class AuthFlow(BaseModel): 1777 auth_flow_type: Optional[AuthFlowType] = Field( 1778 None, description="The type of auth to use", title="Auth flow type" 1779 ) 1780 predicate_key: Optional[List[str]] = Field( 1781 None, 1782 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1783 examples=[["credentials", "auth_type"]], 1784 title="Predicate key", 1785 ) 1786 predicate_value: Optional[str] = Field( 1787 None, 1788 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1789 examples=["Oauth"], 1790 title="Predicate value", 1791 ) 1792 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1795class CheckStream(BaseModel): 1796 type: Literal["CheckStream"] 1797 stream_names: Optional[List[str]] = Field( 1798 None, 1799 description="Names of the streams to try reading from when running a check operation.", 1800 examples=[["users"], ["users", "contacts"]], 1801 title="Stream Names", 1802 ) 1803 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1804 config_overrides: Optional[Dict[str, Any]] = Field( 1805 None, 1806 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1807 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1808 title="Config Overrides", 1809 )
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1812class IncrementingCountCursor(BaseModel): 1813 type: Literal["IncrementingCountCursor"] 1814 cursor_field: str = Field( 1815 ..., 1816 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1817 examples=["created_at", "{{ config['record_cursor'] }}"], 1818 title="Cursor Field", 1819 ) 1820 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1821 None, 1822 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1823 title="Allow Catalog Defined Cursor Field", 1824 ) 1825 start_value: Optional[Union[str, int]] = Field( 1826 None, 1827 description="The value that determines the earliest record that should be synced.", 1828 examples=[0, "{{ config['start_value'] }}"], 1829 title="Start Value", 1830 ) 1831 start_value_option: Optional[RequestOption] = Field( 1832 None, 1833 description="Optionally configures how the start value will be sent in requests to the source API.", 1834 title="Inject Start Value Into Outgoing HTTP Request", 1835 ) 1836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1839class DatetimeBasedCursor(BaseModel): 1840 type: Literal["DatetimeBasedCursor"] 1841 clamping: Optional[Clamping] = Field( 1842 None, 1843 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1844 title="Date Range Clamping", 1845 ) 1846 cursor_field: str = Field( 1847 ..., 1848 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1849 examples=["created_at", "{{ config['record_cursor'] }}"], 1850 title="Cursor Field", 1851 ) 1852 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1853 None, 1854 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1855 title="Allow Catalog Defined Cursor Field", 1856 ) 1857 cursor_datetime_formats: Optional[List[str]] = Field( 1858 None, 1859 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1860 examples=[ 1861 "%Y-%m-%d", 1862 "%Y-%m-%d %H:%M:%S", 1863 "%Y-%m-%dT%H:%M:%S", 1864 "%Y-%m-%dT%H:%M:%SZ", 1865 "%Y-%m-%dT%H:%M:%S%z", 1866 "%Y-%m-%dT%H:%M:%S.%fZ", 1867 "%Y-%m-%dT%H:%M:%S.%f%z", 1868 "%Y-%m-%d %H:%M:%S.%f+00:00", 1869 "%s", 1870 "%ms", 1871 ], 1872 title="Cursor Datetime Formats", 1873 ) 1874 start_datetime: Union[MinMaxDatetime, str] = Field( 1875 ..., 1876 description="The datetime that determines the earliest record that should be synced.", 1877 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1878 title="Start Datetime", 1879 ) 1880 start_time_option: Optional[RequestOption] = Field( 1881 None, 1882 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1883 title="Inject Start Time Into Outgoing HTTP Request", 1884 ) 1885 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1886 None, 1887 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1888 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1889 title="End Datetime", 1890 ) 1891 end_time_option: Optional[RequestOption] = Field( 1892 None, 1893 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1894 title="Inject End Time Into Outgoing HTTP Request", 1895 ) 1896 datetime_format: str = Field( 1897 ..., 1898 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1899 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1900 title="Outgoing Datetime Format", 1901 ) 1902 cursor_granularity: Optional[str] = Field( 1903 None, 1904 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1905 examples=["PT1S"], 1906 title="Cursor Granularity", 1907 ) 1908 is_data_feed: Optional[bool] = Field( 1909 None, 1910 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1911 title="Data Feed API", 1912 ) 1913 is_client_side_incremental: Optional[bool] = Field( 1914 None, 1915 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1916 title="Client-side Incremental Filtering", 1917 ) 1918 is_compare_strictly: Optional[bool] = Field( 1919 False, 1920 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1921 title="Strict Start-End Time Comparison", 1922 ) 1923 global_substream_cursor: Optional[bool] = Field( 1924 False, 1925 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1926 title="Global Substream Cursor", 1927 ) 1928 lookback_window: Optional[str] = Field( 1929 None, 1930 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1931 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1932 title="Lookback Window", 1933 ) 1934 partition_field_end: Optional[str] = Field( 1935 None, 1936 description="Name of the partition start time field.", 1937 examples=["ending_time"], 1938 title="Partition Field End", 1939 ) 1940 partition_field_start: Optional[str] = Field( 1941 None, 1942 description="Name of the partition end time field.", 1943 examples=["starting_time"], 1944 title="Partition Field Start", 1945 ) 1946 step: Optional[str] = Field( 1947 None, 1948 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1949 examples=["P1W", "{{ config['step_increment'] }}"], 1950 title="Step", 1951 ) 1952 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1955class JwtAuthenticator(BaseModel): 1956 type: Literal["JwtAuthenticator"] 1957 secret_key: str = Field( 1958 ..., 1959 description="Secret used to sign the JSON web token.", 1960 examples=["{{ config['secret_key'] }}"], 1961 title="Secret Key", 1962 ) 1963 base64_encode_secret_key: Optional[bool] = Field( 1964 False, 1965 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1966 title="Base64-encode Secret Key", 1967 ) 1968 algorithm: Algorithm = Field( 1969 ..., 1970 description="Algorithm used to sign the JSON web token.", 1971 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1972 title="Algorithm", 1973 ) 1974 token_duration: Optional[int] = Field( 1975 1200, 1976 description="The amount of time in seconds a JWT token can be valid after being issued.", 1977 examples=[1200, 3600], 1978 title="Token Duration", 1979 ) 1980 header_prefix: Optional[str] = Field( 1981 None, 1982 description="The prefix to be used within the Authentication header.", 1983 examples=["Bearer", "Basic"], 1984 title="Header Prefix", 1985 ) 1986 jwt_headers: Optional[JwtHeaders] = Field( 1987 None, 1988 description="JWT headers used when signing JSON web token.", 1989 title="JWT Headers", 1990 ) 1991 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1992 None, 1993 description="Additional headers to be included with the JWT headers object.", 1994 title="Additional JWT Headers", 1995 ) 1996 jwt_payload: Optional[JwtPayload] = Field( 1997 None, 1998 description="JWT Payload used when signing JSON web token.", 1999 title="JWT Payload", 2000 ) 2001 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2002 None, 2003 description="Additional properties to be added to the JWT payload.", 2004 title="Additional JWT Payload Properties", 2005 ) 2006 passphrase: Optional[str] = Field( 2007 None, 2008 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2009 examples=["{{ config['passphrase'] }}"], 2010 title="Passphrase", 2011 ) 2012 request_option: Optional[RequestOption] = Field( 2013 None, 2014 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2015 title="Request Option", 2016 ) 2017 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
2020class OAuthAuthenticator(BaseModel): 2021 type: Literal["OAuthAuthenticator"] 2022 client_id_name: Optional[str] = Field( 2023 "client_id", 2024 description="The name of the property to use to refresh the `access_token`.", 2025 examples=["custom_app_id"], 2026 title="Client ID Property Name", 2027 ) 2028 client_id: Optional[str] = Field( 2029 None, 2030 description="The OAuth client ID. Fill it in the user inputs.", 2031 examples=[ 2032 "{{ config['client_id'] }}", 2033 "{{ config['credentials']['client_id }}", 2034 ], 2035 title="Client ID", 2036 ) 2037 client_secret_name: Optional[str] = Field( 2038 "client_secret", 2039 description="The name of the property to use to refresh the `access_token`.", 2040 examples=["custom_app_secret"], 2041 title="Client Secret Property Name", 2042 ) 2043 client_secret: Optional[str] = Field( 2044 None, 2045 description="The OAuth client secret. Fill it in the user inputs.", 2046 examples=[ 2047 "{{ config['client_secret'] }}", 2048 "{{ config['credentials']['client_secret }}", 2049 ], 2050 title="Client Secret", 2051 ) 2052 refresh_token_name: Optional[str] = Field( 2053 "refresh_token", 2054 description="The name of the property to use to refresh the `access_token`.", 2055 examples=["custom_app_refresh_value"], 2056 title="Refresh Token Property Name", 2057 ) 2058 refresh_token: Optional[str] = Field( 2059 None, 2060 description="Credential artifact used to get a new access token.", 2061 examples=[ 2062 "{{ config['refresh_token'] }}", 2063 "{{ config['credentials]['refresh_token'] }}", 2064 ], 2065 title="Refresh Token", 2066 ) 2067 token_refresh_endpoint: Optional[str] = Field( 2068 None, 2069 description="The full URL to call to obtain a new access token.", 2070 examples=["https://connect.squareup.com/oauth2/token"], 2071 title="Token Refresh Endpoint", 2072 ) 2073 access_token_name: Optional[str] = Field( 2074 "access_token", 2075 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2076 examples=["access_token"], 2077 title="Access Token Property Name", 2078 ) 2079 access_token_value: Optional[str] = Field( 2080 None, 2081 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2082 examples=["secret_access_token_value"], 2083 title="Access Token Value", 2084 ) 2085 expires_in_name: Optional[str] = Field( 2086 "expires_in", 2087 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2088 examples=["expires_in"], 2089 title="Token Expiry Property Name", 2090 ) 2091 grant_type_name: Optional[str] = Field( 2092 "grant_type", 2093 description="The name of the property to use to refresh the `access_token`.", 2094 examples=["custom_grant_type"], 2095 title="Grant Type Property Name", 2096 ) 2097 grant_type: Optional[str] = Field( 2098 "refresh_token", 2099 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2100 examples=["refresh_token", "client_credentials"], 2101 title="Grant Type", 2102 ) 2103 refresh_request_body: Optional[Dict[str, Any]] = Field( 2104 None, 2105 description="Body of the request sent to get a new access token.", 2106 examples=[ 2107 { 2108 "applicationId": "{{ config['application_id'] }}", 2109 "applicationSecret": "{{ config['application_secret'] }}", 2110 "token": "{{ config['token'] }}", 2111 } 2112 ], 2113 title="Refresh Request Body", 2114 ) 2115 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2116 None, 2117 description="Headers of the request sent to get a new access token.", 2118 examples=[ 2119 { 2120 "Authorization": "<AUTH_TOKEN>", 2121 "Content-Type": "application/x-www-form-urlencoded", 2122 } 2123 ], 2124 title="Refresh Request Headers", 2125 ) 2126 send_refresh_request_as_query_params: Optional[bool] = Field( 2127 False, 2128 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2129 examples=[True], 2130 title="Send Refresh Request As Query Params", 2131 ) 2132 scopes: Optional[List[str]] = Field( 2133 None, 2134 description="List of scopes that should be granted to the access token.", 2135 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2136 title="Scopes", 2137 ) 2138 token_expiry_date: Optional[str] = Field( 2139 None, 2140 description="The access token expiry date.", 2141 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2142 title="Token Expiry Date", 2143 ) 2144 token_expiry_date_format: Optional[str] = Field( 2145 None, 2146 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2147 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2148 title="Token Expiry Date Format", 2149 ) 2150 refresh_token_error_status_codes: Optional[List[int]] = Field( 2151 None, 2152 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2153 examples=[[400, 500]], 2154 title="Refresh Token Error Status Codes", 2155 ) 2156 refresh_token_error_key: Optional[str] = Field( 2157 None, 2158 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2159 examples=["error"], 2160 title="Refresh Token Error Key", 2161 ) 2162 refresh_token_error_values: Optional[List[str]] = Field( 2163 None, 2164 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2165 examples=[["invalid_grant", "invalid_permissions"]], 2166 title="Refresh Token Error Values", 2167 ) 2168 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2169 None, 2170 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2171 title="Refresh Token Updater", 2172 ) 2173 profile_assertion: Optional[JwtAuthenticator] = Field( 2174 None, 2175 description="The authenticator being used to authenticate the client authenticator.", 2176 title="Profile Assertion", 2177 ) 2178 use_profile_assertion: Optional[bool] = Field( 2179 False, 2180 description="Enable using profile assertion as a flow for OAuth authorization.", 2181 title="Use Profile Assertion", 2182 ) 2183 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2186class FixedWindowCallRatePolicy(BaseModel): 2187 class Config: 2188 extra = Extra.allow 2189 2190 type: Literal["FixedWindowCallRatePolicy"] 2191 period: str = Field( 2192 ..., description="The time interval for the rate limit window.", title="Period" 2193 ) 2194 call_limit: int = Field( 2195 ..., 2196 description="The maximum number of calls allowed within the period.", 2197 title="Call Limit", 2198 ) 2199 matchers: List[HttpRequestRegexMatcher] = Field( 2200 ..., 2201 description="List of matchers that define which requests this policy applies to.", 2202 title="Matchers", 2203 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2206class MovingWindowCallRatePolicy(BaseModel): 2207 class Config: 2208 extra = Extra.allow 2209 2210 type: Literal["MovingWindowCallRatePolicy"] 2211 rates: List[Rate] = Field( 2212 ..., 2213 description="List of rates that define the call limits for different time intervals.", 2214 title="Rates", 2215 ) 2216 matchers: List[HttpRequestRegexMatcher] = Field( 2217 ..., 2218 description="List of matchers that define which requests this policy applies to.", 2219 title="Matchers", 2220 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2223class UnlimitedCallRatePolicy(BaseModel): 2224 class Config: 2225 extra = Extra.allow 2226 2227 type: Literal["UnlimitedCallRatePolicy"] 2228 matchers: List[HttpRequestRegexMatcher] = Field( 2229 ..., 2230 description="List of matchers that define which requests this policy applies to.", 2231 title="Matchers", 2232 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2235class DefaultErrorHandler(BaseModel): 2236 type: Literal["DefaultErrorHandler"] 2237 backoff_strategies: Optional[ 2238 List[ 2239 Union[ 2240 ConstantBackoffStrategy, 2241 ExponentialBackoffStrategy, 2242 WaitTimeFromHeader, 2243 WaitUntilTimeFromHeader, 2244 CustomBackoffStrategy, 2245 ] 2246 ] 2247 ] = Field( 2248 None, 2249 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2250 title="Backoff Strategies", 2251 ) 2252 max_retries: Optional[Union[int, str]] = Field( 2253 5, 2254 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2255 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2256 title="Max Retry Count", 2257 ) 2258 response_filters: Optional[List[HttpResponseFilter]] = Field( 2259 None, 2260 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2261 title="Response Filters", 2262 ) 2263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2266class DefaultPaginator(BaseModel): 2267 type: Literal["DefaultPaginator"] 2268 pagination_strategy: Union[ 2269 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2270 ] = Field( 2271 ..., 2272 description="Strategy defining how records are paginated.", 2273 title="Pagination Strategy", 2274 ) 2275 page_size_option: Optional[RequestOption] = Field( 2276 None, title="Inject Page Size Into Outgoing HTTP Request" 2277 ) 2278 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2279 None, 2280 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2281 title="Inject Page Token Into Outgoing HTTP Request", 2282 ) 2283 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2286class RecordExpander(BaseModel): 2287 type: Literal["RecordExpander"] 2288 expand_records_from_field: List[str] = Field( 2289 ..., 2290 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2291 examples=[ 2292 ["lines", "data"], 2293 ["items"], 2294 ["nested", "array"], 2295 ["sections", "*", "items"], 2296 ], 2297 title="Expand Records From Field", 2298 ) 2299 remain_original_record: Optional[bool] = Field( 2300 False, 2301 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2302 title="Remain Original Record", 2303 ) 2304 on_no_records: Optional[OnNoRecords] = Field( 2305 OnNoRecords.skip, 2306 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2307 title="On No Records", 2308 ) 2309 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2312class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2313 type: Literal["ApiKey"] 2314 inject_into: RequestOption = Field( 2315 ..., 2316 description="Configure how the API Key will be sent in requests to the source API.", 2317 examples=[ 2318 {"inject_into": "header", "field_name": "Authorization"}, 2319 {"inject_into": "request_parameter", "field_name": "authKey"}, 2320 ], 2321 title="Inject API Key Into Outgoing HTTP Request", 2322 ) 2323 api_token: Optional[str] = Field( 2324 "{{ session_token }}", 2325 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2326 examples=[ 2327 "{{ session_token }}", 2328 "Token {{ session_token }}", 2329 "Bearer {{ session_token }}", 2330 ], 2331 title="API Token Template", 2332 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2335class JsonSchemaPropertySelector(BaseModel): 2336 type: Literal["JsonSchemaPropertySelector"] 2337 transformations: Optional[ 2338 List[ 2339 Union[ 2340 AddFields, 2341 RemoveFields, 2342 KeysToLower, 2343 KeysToSnakeCase, 2344 FlattenFields, 2345 DpathFlattenFields, 2346 KeysReplace, 2347 CustomTransformation, 2348 ] 2349 ] 2350 ] = Field( 2351 None, 2352 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2353 title="Transformations", 2354 ) 2355 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2358class ListPartitionRouter(BaseModel): 2359 type: Literal["ListPartitionRouter"] 2360 cursor_field: str = Field( 2361 ..., 2362 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2363 examples=["section", "{{ config['section_key'] }}"], 2364 title="Current Partition Value Identifier", 2365 ) 2366 values: Union[str, List[str]] = Field( 2367 ..., 2368 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2369 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2370 title="Partition Values", 2371 ) 2372 request_option: Optional[RequestOption] = Field( 2373 None, 2374 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2375 title="Inject Partition Value Into Outgoing HTTP Request", 2376 ) 2377 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2380class PaginationReset(BaseModel): 2381 type: Literal["PaginationReset"] 2382 action: Action1 2383 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2386class GzipDecoder(BaseModel): 2387 type: Literal["GzipDecoder"] 2388 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2391class RequestBodyGraphQL(BaseModel): 2392 type: Literal["RequestBodyGraphQL"] 2393 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2396class DpathValidator(BaseModel): 2397 type: Literal["DpathValidator"] 2398 field_path: List[str] = Field( 2399 ..., 2400 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2401 examples=[ 2402 ["data"], 2403 ["data", "records"], 2404 ["data", "{{ parameters.name }}"], 2405 ["data", "*", "record"], 2406 ], 2407 title="Field Path", 2408 ) 2409 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2410 ..., 2411 description="The condition that the specified config value will be evaluated against", 2412 title="Validation Strategy", 2413 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2416class PredicateValidator(BaseModel): 2417 type: Literal["PredicateValidator"] 2418 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2419 ..., 2420 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2421 examples=[ 2422 "test-value", 2423 "{{ config['api_version'] }}", 2424 "{{ config['tenant_id'] }}", 2425 123, 2426 ], 2427 title="Value", 2428 ) 2429 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2430 ..., 2431 description="The validation strategy to apply to the value.", 2432 title="Validation Strategy", 2433 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2436class ConfigAddFields(BaseModel): 2437 type: Literal["ConfigAddFields"] 2438 fields: List[AddedFieldDefinition] = Field( 2439 ..., 2440 description="A list of transformations (path and corresponding value) that will be added to the config.", 2441 title="Fields", 2442 ) 2443 condition: Optional[str] = Field( 2444 "", 2445 description="Fields will be added if expression is evaluated to True.", 2446 examples=[ 2447 "{{ config['environemnt'] == 'sandbox' }}", 2448 "{{ property is integer }}", 2449 "{{ property|length > 5 }}", 2450 "{{ property == 'some_string_to_match' }}", 2451 ], 2452 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2455class CompositeErrorHandler(BaseModel): 2456 type: Literal["CompositeErrorHandler"] 2457 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2458 Field( 2459 ..., 2460 description="List of error handlers to iterate on to determine how to handle a failed response.", 2461 title="Error Handlers", 2462 ) 2463 ) 2464 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2467class HTTPAPIBudget(BaseModel): 2468 class Config: 2469 extra = Extra.allow 2470 2471 type: Literal["HTTPAPIBudget"] 2472 policies: List[ 2473 Union[ 2474 FixedWindowCallRatePolicy, 2475 MovingWindowCallRatePolicy, 2476 UnlimitedCallRatePolicy, 2477 ] 2478 ] = Field( 2479 ..., 2480 description="List of call rate policies that define how many calls are allowed.", 2481 title="Policies", 2482 ) 2483 ratelimit_reset_header: Optional[str] = Field( 2484 "ratelimit-reset", 2485 description="The HTTP response header name that indicates when the rate limit resets.", 2486 title="Rate Limit Reset Header", 2487 ) 2488 ratelimit_remaining_header: Optional[str] = Field( 2489 "ratelimit-remaining", 2490 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2491 title="Rate Limit Remaining Header", 2492 ) 2493 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2494 [429], 2495 description="List of HTTP status codes that indicate a rate limit has been hit.", 2496 title="Status Codes for Rate Limit Hit", 2497 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2500class DpathExtractor(BaseModel): 2501 type: Literal["DpathExtractor"] 2502 field_path: List[str] = Field( 2503 ..., 2504 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2505 examples=[ 2506 ["data"], 2507 ["data", "records"], 2508 ["data", "{{ parameters.name }}"], 2509 ["data", "*", "record"], 2510 ], 2511 title="Field Path", 2512 ) 2513 record_expander: Optional[RecordExpander] = Field( 2514 None, 2515 description="Optional component to expand records by extracting items from nested array fields.", 2516 title="Record Expander", 2517 ) 2518 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2521class ZipfileDecoder(BaseModel): 2522 class Config: 2523 extra = Extra.allow 2524 2525 type: Literal["ZipfileDecoder"] 2526 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2527 ..., 2528 description="Parser to parse the decompressed data from the zipfile(s).", 2529 title="Parser", 2530 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2533class RecordSelector(BaseModel): 2534 type: Literal["RecordSelector"] 2535 extractor: Union[DpathExtractor, CustomRecordExtractor] 2536 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2537 None, 2538 description="Responsible for filtering records to be emitted by the Source.", 2539 title="Record Filter", 2540 ) 2541 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2542 None, 2543 description="Responsible for normalization according to the schema.", 2544 title="Schema Normalization", 2545 ) 2546 transform_before_filtering: Optional[bool] = Field( 2547 None, 2548 description="If true, transformation will be applied before record filtering.", 2549 title="Transform Before Filtering", 2550 ) 2551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2554class ConfigMigration(BaseModel): 2555 type: Literal["ConfigMigration"] 2556 description: Optional[str] = Field( 2557 None, description="The description/purpose of the config migration." 2558 ) 2559 transformations: List[ 2560 Union[ 2561 ConfigRemapField, 2562 ConfigAddFields, 2563 ConfigRemoveFields, 2564 CustomConfigTransformation, 2565 ] 2566 ] = Field( 2567 ..., 2568 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2569 title="Transformations", 2570 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2573class ConfigNormalizationRules(BaseModel): 2574 class Config: 2575 extra = Extra.forbid 2576 2577 type: Literal["ConfigNormalizationRules"] 2578 config_migrations: Optional[List[ConfigMigration]] = Field( 2579 [], 2580 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2581 title="Config Migrations", 2582 ) 2583 transformations: Optional[ 2584 List[ 2585 Union[ 2586 ConfigRemapField, 2587 ConfigAddFields, 2588 ConfigRemoveFields, 2589 CustomConfigTransformation, 2590 ] 2591 ] 2592 ] = Field( 2593 [], 2594 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2595 title="Transformations", 2596 ) 2597 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2598 [], 2599 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2600 title="Validations", 2601 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2604class Spec(BaseModel): 2605 type: Literal["Spec"] 2606 connection_specification: Dict[str, Any] = Field( 2607 ..., 2608 description="A connection specification describing how a the connector can be configured.", 2609 title="Connection Specification", 2610 ) 2611 documentation_url: Optional[str] = Field( 2612 None, 2613 description="URL of the connector's documentation page.", 2614 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2615 title="Documentation URL", 2616 ) 2617 advanced_auth: Optional[AuthFlow] = Field( 2618 None, 2619 description="Advanced specification for configuring the authentication flow.", 2620 title="Advanced Auth", 2621 ) 2622 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2623 None, title="Config Normalization Rules" 2624 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2627class DeclarativeSource1(BaseModel): 2628 class Config: 2629 extra = Extra.forbid 2630 2631 type: Literal["DeclarativeSource"] 2632 check: Union[CheckStream, CheckDynamicStream] 2633 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2634 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2635 version: str = Field( 2636 ..., 2637 description="The version of the Airbyte CDK used to build and test the source.", 2638 ) 2639 schemas: Optional[Schemas] = None 2640 definitions: Optional[Dict[str, Any]] = None 2641 spec: Optional[Spec] = None 2642 concurrency_level: Optional[ConcurrencyLevel] = None 2643 api_budget: Optional[HTTPAPIBudget] = None 2644 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2645 None, 2646 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2647 title="Stream Groups", 2648 ) 2649 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2650 None, 2651 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2652 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2653 title="Maximum Concurrent Asynchronous Jobs", 2654 ) 2655 metadata: Optional[Dict[str, Any]] = Field( 2656 None, 2657 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2658 ) 2659 description: Optional[str] = Field( 2660 None, 2661 description="A description of the connector. It will be presented on the Source documentation page.", 2662 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2665class DeclarativeSource2(BaseModel): 2666 class Config: 2667 extra = Extra.forbid 2668 2669 type: Literal["DeclarativeSource"] 2670 check: Union[CheckStream, CheckDynamicStream] 2671 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2672 None 2673 ) 2674 dynamic_streams: List[DynamicDeclarativeStream] 2675 version: str = Field( 2676 ..., 2677 description="The version of the Airbyte CDK used to build and test the source.", 2678 ) 2679 schemas: Optional[Schemas] = None 2680 definitions: Optional[Dict[str, Any]] = None 2681 spec: Optional[Spec] = None 2682 concurrency_level: Optional[ConcurrencyLevel] = None 2683 api_budget: Optional[HTTPAPIBudget] = None 2684 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2685 None, 2686 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2687 title="Stream Groups", 2688 ) 2689 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2690 None, 2691 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2692 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2693 title="Maximum Concurrent Asynchronous Jobs", 2694 ) 2695 metadata: Optional[Dict[str, Any]] = Field( 2696 None, 2697 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2698 ) 2699 description: Optional[str] = Field( 2700 None, 2701 description="A description of the connector. It will be presented on the Source documentation page.", 2702 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2705class DeclarativeSource(BaseModel): 2706 class Config: 2707 extra = Extra.forbid 2708 2709 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2710 ..., 2711 description="An API source that extracts data according to its declarative components.", 2712 title="DeclarativeSource", 2713 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2716class SelectiveAuthenticator(BaseModel): 2717 class Config: 2718 extra = Extra.allow 2719 2720 type: Literal["SelectiveAuthenticator"] 2721 authenticator_selection_path: List[str] = Field( 2722 ..., 2723 description="Path of the field in config with selected authenticator name", 2724 examples=[["auth"], ["auth", "type"]], 2725 title="Authenticator Selection Path", 2726 ) 2727 authenticators: Dict[ 2728 str, 2729 Union[ 2730 ApiKeyAuthenticator, 2731 BasicHttpAuthenticator, 2732 BearerAuthenticator, 2733 OAuthAuthenticator, 2734 JwtAuthenticator, 2735 SessionTokenAuthenticator, 2736 LegacySessionTokenAuthenticator, 2737 CustomAuthenticator, 2738 NoAuth, 2739 RateLimitedMultipleTokenAuthenticator, 2740 ], 2741 ] = Field( 2742 ..., 2743 description="Authenticators to select from.", 2744 examples=[ 2745 { 2746 "authenticators": { 2747 "token": "#/definitions/ApiKeyAuthenticator", 2748 "oauth": "#/definitions/OAuthAuthenticator", 2749 "jwt": "#/definitions/JwtAuthenticator", 2750 } 2751 } 2752 ], 2753 title="Authenticators", 2754 ) 2755 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2758class ConditionalStreams(BaseModel): 2759 type: Literal["ConditionalStreams"] 2760 condition: str = Field( 2761 ..., 2762 description="Condition that will be evaluated to determine if a set of streams should be available.", 2763 examples=["{{ config['is_sandbox'] }}"], 2764 title="Condition", 2765 ) 2766 streams: List[DeclarativeStream] = Field( 2767 ..., 2768 description="Streams that will be used during an operation based on the condition.", 2769 title="Streams", 2770 ) 2771 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2774class FileUploader(BaseModel): 2775 type: Literal["FileUploader"] 2776 requester: Union[HttpRequester, CustomRequester] = Field( 2777 ..., 2778 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2779 ) 2780 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2781 ..., 2782 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2783 ) 2784 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2785 None, 2786 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2787 ) 2788 filename_extractor: Optional[str] = Field( 2789 None, 2790 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2791 examples=[ 2792 "{{ record.id }}/{{ record.file_name }}/", 2793 "{{ record.id }}_{{ record.file_name }}/", 2794 ], 2795 ) 2796 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2799class DeclarativeStream(BaseModel): 2800 class Config: 2801 extra = Extra.allow 2802 2803 type: Literal["DeclarativeStream"] 2804 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2805 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2806 ..., 2807 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2808 title="Retriever", 2809 ) 2810 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2811 None, 2812 description="Component used to fetch data incrementally based on a time field in the data.", 2813 title="Incremental Sync", 2814 ) 2815 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2816 schema_loader: Optional[ 2817 Union[ 2818 InlineSchemaLoader, 2819 DynamicSchemaLoader, 2820 JsonFileSchemaLoader, 2821 List[ 2822 Union[ 2823 InlineSchemaLoader, 2824 DynamicSchemaLoader, 2825 JsonFileSchemaLoader, 2826 CustomSchemaLoader, 2827 ] 2828 ], 2829 CustomSchemaLoader, 2830 ] 2831 ] = Field( 2832 None, 2833 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2834 title="Schema Loader", 2835 ) 2836 transformations: Optional[ 2837 List[ 2838 Union[ 2839 AddFields, 2840 RemoveFields, 2841 KeysToLower, 2842 KeysToSnakeCase, 2843 FlattenFields, 2844 DpathFlattenFields, 2845 KeysReplace, 2846 CustomTransformation, 2847 ] 2848 ] 2849 ] = Field( 2850 None, 2851 description="A list of transformations to be applied to each output record.", 2852 title="Transformations", 2853 ) 2854 state_migrations: Optional[ 2855 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2856 ] = Field( 2857 [], 2858 description="Array of state migrations to be applied on the input state", 2859 title="State Migrations", 2860 ) 2861 file_uploader: Optional[FileUploader] = Field( 2862 None, 2863 description="(experimental) Describes how to fetch a file", 2864 title="File Uploader", 2865 ) 2866 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2869class SessionTokenAuthenticator(BaseModel): 2870 type: Literal["SessionTokenAuthenticator"] 2871 login_requester: HttpRequester = Field( 2872 ..., 2873 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2874 examples=[ 2875 { 2876 "type": "HttpRequester", 2877 "url_base": "https://my_api.com", 2878 "path": "/login", 2879 "authenticator": { 2880 "type": "BasicHttpAuthenticator", 2881 "username": "{{ config.username }}", 2882 "password": "{{ config.password }}", 2883 }, 2884 } 2885 ], 2886 title="Login Requester", 2887 ) 2888 session_token_path: List[str] = Field( 2889 ..., 2890 description="The path in the response body returned from the login requester to the session token.", 2891 examples=[["access_token"], ["result", "token"]], 2892 title="Session Token Path", 2893 ) 2894 expiration_duration: Optional[str] = Field( 2895 None, 2896 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2897 examples=["PT1H", "P1D"], 2898 title="Expiration Duration", 2899 ) 2900 request_authentication: Union[ 2901 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2902 ] = Field( 2903 ..., 2904 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2905 title="Data Request Authentication", 2906 ) 2907 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2908 None, description="Component used to decode the response.", title="Decoder" 2909 ) 2910 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2913class HttpRequester(BaseModelWithDeprecations): 2914 type: Literal["HttpRequester"] 2915 url_base: Optional[str] = Field( 2916 None, 2917 deprecated=True, 2918 deprecation_message="Use `url` field instead.", 2919 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2920 examples=[ 2921 "https://connect.squareup.com/v2", 2922 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2923 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2924 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2925 ], 2926 title="API Base URL", 2927 ) 2928 url: Optional[str] = Field( 2929 None, 2930 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2931 examples=[ 2932 "https://connect.squareup.com/v2", 2933 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2934 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2935 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2936 ], 2937 title="API Endpoint URL", 2938 ) 2939 path: Optional[str] = Field( 2940 None, 2941 deprecated=True, 2942 deprecation_message="Use `url` field instead.", 2943 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2944 examples=[ 2945 "/products", 2946 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2947 "/trades/{{ config['symbol_id'] }}/history", 2948 ], 2949 title="URL Path", 2950 ) 2951 http_method: Optional[HttpMethod] = Field( 2952 HttpMethod.GET, 2953 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2954 examples=["GET", "POST"], 2955 title="HTTP Method", 2956 ) 2957 authenticator: Optional[ 2958 Union[ 2959 ApiKeyAuthenticator, 2960 BasicHttpAuthenticator, 2961 BearerAuthenticator, 2962 OAuthAuthenticator, 2963 JwtAuthenticator, 2964 SessionTokenAuthenticator, 2965 SelectiveAuthenticator, 2966 CustomAuthenticator, 2967 NoAuth, 2968 LegacySessionTokenAuthenticator, 2969 RateLimitedMultipleTokenAuthenticator, 2970 ] 2971 ] = Field( 2972 None, 2973 description="Authentication method to use for requests sent to the API.", 2974 title="Authenticator", 2975 ) 2976 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2977 None, 2978 deprecated=True, 2979 deprecation_message="Use `query_properties` field instead.", 2980 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2981 title="Fetch Properties from Endpoint", 2982 ) 2983 query_properties: Optional[QueryProperties] = Field( 2984 None, 2985 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2986 title="Query Properties", 2987 ) 2988 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2989 None, 2990 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2991 examples=[ 2992 {"unit": "day"}, 2993 { 2994 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2995 }, 2996 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2997 {"sort_by[asc]": "updated_at"}, 2998 ], 2999 title="Query Parameters", 3000 ) 3001 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3002 None, 3003 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3004 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3005 title="Request Headers", 3006 ) 3007 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3008 None, 3009 deprecated=True, 3010 deprecation_message="Use `request_body` field instead.", 3011 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3012 examples=[ 3013 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3014 ], 3015 title="Request Body Payload (Non-JSON)", 3016 ) 3017 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3018 None, 3019 deprecated=True, 3020 deprecation_message="Use `request_body` field instead.", 3021 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3022 examples=[ 3023 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3024 {"key": "{{ config['value'] }}"}, 3025 {"sort": {"field": "updated_at", "order": "ascending"}}, 3026 ], 3027 title="Request Body JSON Payload", 3028 ) 3029 request_body: Optional[ 3030 Union[ 3031 RequestBodyPlainText, 3032 RequestBodyUrlEncodedForm, 3033 RequestBodyJsonObject, 3034 RequestBodyGraphQL, 3035 ] 3036 ] = Field( 3037 None, 3038 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3039 title="Request Body", 3040 ) 3041 error_handler: Optional[ 3042 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3043 ] = Field( 3044 None, 3045 description="Error handler component that defines how to handle errors.", 3046 title="Error Handler", 3047 ) 3048 use_cache: Optional[bool] = Field( 3049 False, 3050 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3051 title="Use Cache", 3052 ) 3053 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, RateLimitedMultipleTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
3056class DynamicSchemaLoader(BaseModel): 3057 type: Literal["DynamicSchemaLoader"] 3058 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3059 ..., 3060 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3061 title="Retriever", 3062 ) 3063 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3064 None, 3065 description="Responsible for filtering fields to be added to json schema.", 3066 title="Schema Filter", 3067 ) 3068 schema_transformations: Optional[ 3069 List[ 3070 Union[ 3071 AddFields, 3072 RemoveFields, 3073 KeysToLower, 3074 KeysToSnakeCase, 3075 FlattenFields, 3076 DpathFlattenFields, 3077 KeysReplace, 3078 CustomTransformation, 3079 ] 3080 ] 3081 ] = Field( 3082 None, 3083 description="A list of transformations to be applied to the schema.", 3084 title="Schema Transformations", 3085 ) 3086 schema_type_identifier: SchemaTypeIdentifier 3087 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
3090class ParentStreamConfig(BaseModel): 3091 type: Literal["ParentStreamConfig"] 3092 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3093 ..., description="Reference to the parent stream.", title="Parent Stream" 3094 ) 3095 parent_key: str = Field( 3096 ..., 3097 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3098 examples=["id", "{{ config['parent_record_id'] }}"], 3099 title="Parent Key", 3100 ) 3101 partition_field: str = Field( 3102 ..., 3103 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3104 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3105 title="Current Parent Key Value Identifier", 3106 ) 3107 request_option: Optional[RequestOption] = Field( 3108 None, 3109 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3110 title="Request Option", 3111 ) 3112 incremental_dependency: Optional[bool] = Field( 3113 False, 3114 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3115 title="Incremental Dependency", 3116 ) 3117 lazy_read_pointer: Optional[List[str]] = Field( 3118 [], 3119 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3120 title="Lazy Read Pointer", 3121 ) 3122 extra_fields: Optional[List[List[str]]] = Field( 3123 None, 3124 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3125 title="Extra Fields", 3126 ) 3127 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
3130class PropertiesFromEndpoint(BaseModel): 3131 type: Literal["PropertiesFromEndpoint"] 3132 property_field_path: List[str] = Field( 3133 ..., 3134 description="Describes the path to the field that should be extracted", 3135 examples=[["name"]], 3136 ) 3137 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3138 ..., 3139 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3140 ) 3141 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
3144class QueryProperties(BaseModel): 3145 type: Literal["QueryProperties"] 3146 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3147 ..., 3148 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3149 title="Property List", 3150 ) 3151 always_include_properties: Optional[List[str]] = Field( 3152 None, 3153 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3154 title="Always Include Properties", 3155 ) 3156 property_chunking: Optional[PropertyChunking] = Field( 3157 None, 3158 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3159 title="Property Chunking", 3160 ) 3161 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3162 None, 3163 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3164 title="Property Selector", 3165 ) 3166 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3169class StateDelegatingStream(BaseModel): 3170 type: Literal["StateDelegatingStream"] 3171 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3172 full_refresh_stream: DeclarativeStream = Field( 3173 ..., 3174 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3175 title="Full Refresh Stream", 3176 ) 3177 incremental_stream: DeclarativeStream = Field( 3178 ..., 3179 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3180 title="Incremental Stream", 3181 ) 3182 api_retention_period: Optional[str] = Field( 3183 None, 3184 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3185 examples=["P30D", "P90D", "P1Y"], 3186 title="API Retention Period", 3187 ) 3188 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3191class SimpleRetriever(BaseModel): 3192 type: Literal["SimpleRetriever"] 3193 requester: Union[HttpRequester, CustomRequester] = Field( 3194 ..., 3195 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3196 ) 3197 decoder: Optional[ 3198 Union[ 3199 JsonDecoder, 3200 JsonItemsDecoder, 3201 XmlDecoder, 3202 CsvDecoder, 3203 JsonlDecoder, 3204 GzipDecoder, 3205 IterableDecoder, 3206 ZipfileDecoder, 3207 CustomDecoder, 3208 ] 3209 ] = Field( 3210 None, 3211 description="Component decoding the response so records can be extracted.", 3212 title="HTTP Response Format", 3213 ) 3214 record_selector: RecordSelector = Field( 3215 ..., 3216 description="Component that describes how to extract records from a HTTP response.", 3217 ) 3218 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3219 None, 3220 description="Paginator component that describes how to navigate through the API's pages.", 3221 ) 3222 pagination_reset: Optional[PaginationReset] = Field( 3223 None, 3224 description="Describes what triggers pagination reset and how to handle it.", 3225 ) 3226 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3227 False, 3228 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3229 ) 3230 partition_router: Optional[ 3231 Union[ 3232 SubstreamPartitionRouter, 3233 ListPartitionRouter, 3234 GroupingPartitionRouter, 3235 UnionPartitionRouter, 3236 CustomPartitionRouter, 3237 List[ 3238 Union[ 3239 SubstreamPartitionRouter, 3240 ListPartitionRouter, 3241 GroupingPartitionRouter, 3242 UnionPartitionRouter, 3243 CustomPartitionRouter, 3244 ] 3245 ], 3246 ] 3247 ] = Field( 3248 None, 3249 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3250 title="Partition Router", 3251 ) 3252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3255class AsyncRetriever(BaseModel): 3256 type: Literal["AsyncRetriever"] 3257 record_selector: RecordSelector = Field( 3258 ..., 3259 description="Component that describes how to extract records from a HTTP response.", 3260 ) 3261 status_mapping: AsyncJobStatusMap = Field( 3262 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3263 ) 3264 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3265 ..., description="Responsible for fetching the actual status of the async job." 3266 ) 3267 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3268 None, 3269 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3270 ) 3271 download_extractor: Optional[ 3272 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3273 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3274 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3275 ..., 3276 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3277 ) 3278 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3279 ..., 3280 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3281 ) 3282 polling_job_timeout: Optional[Union[int, str]] = Field( 3283 None, 3284 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3285 ) 3286 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3287 None, 3288 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3289 ge=1, 3290 ) 3291 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3292 None, 3293 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3294 ) 3295 download_requester: Union[HttpRequester, CustomRequester] = Field( 3296 ..., 3297 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3298 ) 3299 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3300 None, 3301 description="Paginator component that describes how to navigate through the API's pages during download.", 3302 ) 3303 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3304 None, 3305 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3306 ) 3307 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3308 None, 3309 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3310 ) 3311 partition_router: Optional[ 3312 Union[ 3313 ListPartitionRouter, 3314 SubstreamPartitionRouter, 3315 GroupingPartitionRouter, 3316 UnionPartitionRouter, 3317 CustomPartitionRouter, 3318 List[ 3319 Union[ 3320 ListPartitionRouter, 3321 SubstreamPartitionRouter, 3322 GroupingPartitionRouter, 3323 UnionPartitionRouter, 3324 CustomPartitionRouter, 3325 ] 3326 ], 3327 ] 3328 ] = Field( 3329 [], 3330 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3331 title="Partition Router", 3332 ) 3333 decoder: Optional[ 3334 Union[ 3335 CsvDecoder, 3336 GzipDecoder, 3337 JsonDecoder, 3338 JsonItemsDecoder, 3339 JsonlDecoder, 3340 IterableDecoder, 3341 XmlDecoder, 3342 ZipfileDecoder, 3343 CustomDecoder, 3344 ] 3345 ] = Field( 3346 None, 3347 description="Component decoding the response so records can be extracted.", 3348 title="HTTP Response Format", 3349 ) 3350 download_decoder: Optional[ 3351 Union[ 3352 CsvDecoder, 3353 GzipDecoder, 3354 JsonDecoder, 3355 JsonItemsDecoder, 3356 JsonlDecoder, 3357 IterableDecoder, 3358 XmlDecoder, 3359 ZipfileDecoder, 3360 CustomDecoder, 3361 ] 3362 ] = Field( 3363 None, 3364 description="Component decoding the download response so records can be extracted.", 3365 title="Download HTTP Response Format", 3366 ) 3367 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3374class StreamGroup(BaseModel): 3375 streams: List[str] = Field( 3376 ..., 3377 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3378 title="Streams", 3379 ) 3380 action: BlockSimultaneousSyncsAction = Field( 3381 ..., 3382 description="The action to apply to streams in this group.", 3383 title="Action", 3384 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3387class SubstreamPartitionRouter(BaseModel): 3388 type: Literal["SubstreamPartitionRouter"] 3389 parent_stream_configs: List[ParentStreamConfig] = Field( 3390 ..., 3391 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3392 title="Parent Stream Configs", 3393 ) 3394 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3397class GroupingPartitionRouter(BaseModel): 3398 type: Literal["GroupingPartitionRouter"] 3399 group_size: int = Field( 3400 ..., 3401 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3402 examples=[10, 50], 3403 title="Group Size", 3404 ) 3405 underlying_partition_router: Union[ 3406 ListPartitionRouter, 3407 SubstreamPartitionRouter, 3408 "UnionPartitionRouter", 3409 CustomPartitionRouter, 3410 ] = Field( 3411 ..., 3412 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3413 title="Underlying Partition Router", 3414 ) 3415 deduplicate: Optional[bool] = Field( 3416 True, 3417 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3418 title="Deduplicate Partitions", 3419 ) 3420 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]
class
UnionPartitionRouter(pydantic.v1.main.BaseModel):
3423class UnionPartitionRouter(BaseModel): 3424 type: Literal["UnionPartitionRouter"] 3425 partition_field: str = Field( 3426 ..., 3427 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3428 examples=["repository", "{{ config['partition_field'] }}"], 3429 title="Partition Field", 3430 ) 3431 partition_routers: List[ 3432 Union[ 3433 ListPartitionRouter, 3434 SubstreamPartitionRouter, 3435 UnionPartitionRouter, 3436 CustomPartitionRouter, 3437 ] 3438 ] = Field( 3439 ..., 3440 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3441 title="Partition Routers", 3442 ) 3443 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
partition_routers: List[Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3446class HttpComponentsResolver(BaseModel): 3447 type: Literal["HttpComponentsResolver"] 3448 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3449 ..., 3450 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3451 title="Retriever", 3452 ) 3453 components_mapping: List[ComponentMappingDefinition] 3454 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3457class DynamicDeclarativeStream(BaseModel): 3458 type: Literal["DynamicDeclarativeStream"] 3459 name: Optional[str] = Field( 3460 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3461 ) 3462 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3463 ..., description="Reference to the stream template.", title="Stream Template" 3464 ) 3465 components_resolver: Union[ 3466 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3467 ] = Field( 3468 ..., 3469 description="Component resolve and populates stream templates with components values.", 3470 title="Components Resolver", 3471 ) 3472 use_parent_parameters: Optional[bool] = Field( 3473 True, 3474 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3475 title="Use Parent Parameters", 3476 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]