airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 config_overrides: Optional[Dict[str, Any]] = Field( 81 None, 82 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 83 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 84 title="Config Overrides", 85 ) 86 87 88class ConcurrencyLevel(BaseModel): 89 type: Optional[Literal["ConcurrencyLevel"]] = None 90 default_concurrency: Union[int, str] = Field( 91 ..., 92 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 93 examples=[10, "{{ config['num_workers'] or 10 }}"], 94 title="Default Concurrency", 95 ) 96 max_concurrency: Optional[int] = Field( 97 None, 98 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 99 examples=[20, 100], 100 title="Max Concurrency", 101 ) 102 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 103 104 105class ConstantBackoffStrategy(BaseModel): 106 type: Literal["ConstantBackoffStrategy"] 107 backoff_time_in_seconds: Union[float, str] = Field( 108 ..., 109 description="Backoff time in seconds.", 110 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 111 title="Backoff Time", 112 ) 113 jitter_range_in_seconds: Optional[float] = Field( 114 None, 115 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 116 examples=[15], 117 ge=0, 118 title="Jitter Range", 119 ) 120 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 121 122 123class CursorPagination(BaseModel): 124 type: Literal["CursorPagination"] 125 cursor_value: str = Field( 126 ..., 127 description="Value of the cursor defining the next page to fetch.", 128 examples=[ 129 "{{ headers.link.next.cursor }}", 130 "{{ last_record['key'] }}", 131 "{{ response['nextPage'] }}", 132 ], 133 title="Cursor Value", 134 ) 135 page_size: Optional[Union[int, str]] = Field( 136 None, 137 description="The number of records to include in each pages.", 138 examples=[100, "{{ config['page_size'] }}"], 139 title="Page Size", 140 ) 141 stop_condition: Optional[str] = Field( 142 None, 143 description="Template string evaluating when to stop paginating.", 144 examples=[ 145 "{{ response.data.has_more is false }}", 146 "{{ 'next' not in headers['link'] }}", 147 ], 148 title="Stop Condition", 149 ) 150 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 151 152 153class CustomAuthenticator(BaseModel): 154 class Config: 155 extra = Extra.allow 156 157 type: Literal["CustomAuthenticator"] 158 class_name: str = Field( 159 ..., 160 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 161 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 162 title="Class Name", 163 ) 164 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 165 166 167class CustomBackoffStrategy(BaseModel): 168 class Config: 169 extra = Extra.allow 170 171 type: Literal["CustomBackoffStrategy"] 172 class_name: str = Field( 173 ..., 174 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 175 examples=["source_railz.components.MyCustomBackoffStrategy"], 176 title="Class Name", 177 ) 178 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 179 180 181class CustomErrorHandler(BaseModel): 182 class Config: 183 extra = Extra.allow 184 185 type: Literal["CustomErrorHandler"] 186 class_name: str = Field( 187 ..., 188 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 189 examples=["source_railz.components.MyCustomErrorHandler"], 190 title="Class Name", 191 ) 192 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 193 194 195class CustomPaginationStrategy(BaseModel): 196 class Config: 197 extra = Extra.allow 198 199 type: Literal["CustomPaginationStrategy"] 200 class_name: str = Field( 201 ..., 202 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 203 examples=["source_railz.components.MyCustomPaginationStrategy"], 204 title="Class Name", 205 ) 206 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 207 208 209class CustomRecordExtractor(BaseModel): 210 class Config: 211 extra = Extra.allow 212 213 type: Literal["CustomRecordExtractor"] 214 class_name: str = Field( 215 ..., 216 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 217 examples=["source_railz.components.MyCustomRecordExtractor"], 218 title="Class Name", 219 ) 220 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 221 222 223class CustomRecordFilter(BaseModel): 224 class Config: 225 extra = Extra.allow 226 227 type: Literal["CustomRecordFilter"] 228 class_name: str = Field( 229 ..., 230 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 231 examples=["source_railz.components.MyCustomCustomRecordFilter"], 232 title="Class Name", 233 ) 234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 235 236 237class CustomRequester(BaseModel): 238 class Config: 239 extra = Extra.allow 240 241 type: Literal["CustomRequester"] 242 class_name: str = Field( 243 ..., 244 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 245 examples=["source_railz.components.MyCustomRecordExtractor"], 246 title="Class Name", 247 ) 248 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 249 250 251class CustomRetriever(BaseModel): 252 class Config: 253 extra = Extra.allow 254 255 type: Literal["CustomRetriever"] 256 class_name: str = Field( 257 ..., 258 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 259 examples=["source_railz.components.MyCustomRetriever"], 260 title="Class Name", 261 ) 262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 263 264 265class CustomPartitionRouter(BaseModel): 266 class Config: 267 extra = Extra.allow 268 269 type: Literal["CustomPartitionRouter"] 270 class_name: str = Field( 271 ..., 272 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 273 examples=["source_railz.components.MyCustomPartitionRouter"], 274 title="Class Name", 275 ) 276 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 277 278 279class CustomSchemaLoader(BaseModel): 280 class Config: 281 extra = Extra.allow 282 283 type: Literal["CustomSchemaLoader"] 284 class_name: str = Field( 285 ..., 286 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 287 examples=["source_railz.components.MyCustomSchemaLoader"], 288 title="Class Name", 289 ) 290 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 291 292 293class CustomSchemaNormalization(BaseModel): 294 class Config: 295 extra = Extra.allow 296 297 type: Literal["CustomSchemaNormalization"] 298 class_name: str = Field( 299 ..., 300 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 301 examples=[ 302 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 303 ], 304 title="Class Name", 305 ) 306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 307 308 309class CustomStateMigration(BaseModel): 310 class Config: 311 extra = Extra.allow 312 313 type: Literal["CustomStateMigration"] 314 class_name: str = Field( 315 ..., 316 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 317 examples=["source_railz.components.MyCustomStateMigration"], 318 title="Class Name", 319 ) 320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 321 322 323class CustomTransformation(BaseModel): 324 class Config: 325 extra = Extra.allow 326 327 type: Literal["CustomTransformation"] 328 class_name: str = Field( 329 ..., 330 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 331 examples=["source_railz.components.MyCustomTransformation"], 332 title="Class Name", 333 ) 334 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 335 336 337class LegacyToPerPartitionStateMigration(BaseModel): 338 class Config: 339 extra = Extra.allow 340 341 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 342 343 344class Clamping(BaseModel): 345 target: str = Field( 346 ..., 347 description="The period of time that datetime windows will be clamped by", 348 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 349 title="Target", 350 ) 351 target_details: Optional[Dict[str, Any]] = None 352 353 354class Algorithm(Enum): 355 HS256 = "HS256" 356 HS384 = "HS384" 357 HS512 = "HS512" 358 ES256 = "ES256" 359 ES256K = "ES256K" 360 ES384 = "ES384" 361 ES512 = "ES512" 362 RS256 = "RS256" 363 RS384 = "RS384" 364 RS512 = "RS512" 365 PS256 = "PS256" 366 PS384 = "PS384" 367 PS512 = "PS512" 368 EdDSA = "EdDSA" 369 370 371class JwtHeaders(BaseModel): 372 class Config: 373 extra = Extra.forbid 374 375 kid: Optional[str] = Field( 376 None, 377 description="Private key ID for user account.", 378 examples=["{{ config['kid'] }}"], 379 title="Key Identifier", 380 ) 381 typ: Optional[str] = Field( 382 "JWT", 383 description="The media type of the complete JWT.", 384 examples=["JWT"], 385 title="Type", 386 ) 387 cty: Optional[str] = Field( 388 None, 389 description="Content type of JWT header.", 390 examples=["JWT"], 391 title="Content Type", 392 ) 393 394 395class JwtPayload(BaseModel): 396 class Config: 397 extra = Extra.forbid 398 399 iss: Optional[str] = Field( 400 None, 401 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 402 examples=["{{ config['iss'] }}"], 403 title="Issuer", 404 ) 405 sub: Optional[str] = Field( 406 None, 407 description="The subject of the JWT. Commonly defined by the API.", 408 title="Subject", 409 ) 410 aud: Optional[str] = Field( 411 None, 412 description="The recipient that the JWT is intended for. Commonly defined by the API.", 413 examples=["appstoreconnect-v1"], 414 title="Audience", 415 ) 416 417 418class RefreshTokenUpdater(BaseModel): 419 refresh_token_name: Optional[str] = Field( 420 "refresh_token", 421 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 422 examples=["refresh_token"], 423 title="Refresh Token Property Name", 424 ) 425 access_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "access_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "access_token"], ["access_token"]], 429 title="Config Path To Access Token", 430 ) 431 refresh_token_config_path: Optional[List[str]] = Field( 432 ["credentials", "refresh_token"], 433 description="Config path to the access token. Make sure the field actually exists in the config.", 434 examples=[["credentials", "refresh_token"], ["refresh_token"]], 435 title="Config Path To Refresh Token", 436 ) 437 token_expiry_date_config_path: Optional[List[str]] = Field( 438 ["credentials", "token_expiry_date"], 439 description="Config path to the expiry date. Make sure actually exists in the config.", 440 examples=[["credentials", "token_expiry_date"]], 441 title="Config Path To Expiry Date", 442 ) 443 refresh_token_error_status_codes: Optional[List[int]] = Field( 444 [], 445 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 446 examples=[[400, 500]], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 448 ) 449 refresh_token_error_key: Optional[str] = Field( 450 "", 451 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 452 examples=["error"], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 454 ) 455 refresh_token_error_values: Optional[List[str]] = Field( 456 [], 457 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 458 examples=[["invalid_grant", "invalid_permissions"]], 459 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 460 ) 461 462 463class Rate(BaseModel): 464 class Config: 465 extra = Extra.allow 466 467 limit: Union[int, str] = Field( 468 ..., 469 description="The maximum number of calls allowed within the interval.", 470 title="Limit", 471 ) 472 interval: str = Field( 473 ..., 474 description="The time interval for the rate limit.", 475 examples=["PT1H", "P1D"], 476 title="Interval", 477 ) 478 479 480class HttpRequestRegexMatcher(BaseModel): 481 class Config: 482 extra = Extra.allow 483 484 method: Optional[str] = Field( 485 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 486 ) 487 url_base: Optional[str] = Field( 488 None, 489 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 490 title="URL Base", 491 ) 492 url_path_pattern: Optional[str] = Field( 493 None, 494 description="A regular expression pattern to match the URL path.", 495 title="URL Path Pattern", 496 ) 497 params: Optional[Dict[str, Any]] = Field( 498 None, description="The query parameters to match.", title="Parameters" 499 ) 500 headers: Optional[Dict[str, Any]] = Field( 501 None, description="The headers to match.", title="Headers" 502 ) 503 weight: Optional[Union[int, str]] = Field( 504 None, 505 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 506 title="Weight", 507 ) 508 509 510class ResponseToFileExtractor(BaseModel): 511 type: Literal["ResponseToFileExtractor"] 512 preserve_na_values: Optional[bool] = Field( 513 False, 514 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 515 title="Preserve NA Values", 516 ) 517 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 518 519 520class OnNoRecords(Enum): 521 skip = "skip" 522 emit_parent = "emit_parent" 523 524 525class ExponentialBackoffStrategy(BaseModel): 526 type: Literal["ExponentialBackoffStrategy"] 527 factor: Optional[Union[float, str]] = Field( 528 5, 529 description="Multiplicative constant applied on each retry.", 530 examples=[5, 5.5, "10"], 531 title="Factor", 532 ) 533 jitter_range_in_seconds: Optional[float] = Field( 534 None, 535 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 536 examples=[2], 537 ge=0, 538 title="Jitter Range", 539 ) 540 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 541 542 543class GroupByKeyMergeStrategy(BaseModel): 544 type: Literal["GroupByKeyMergeStrategy"] 545 key: Union[str, List[str]] = Field( 546 ..., 547 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 548 examples=["id", ["parent_id", "end_date"]], 549 title="Key", 550 ) 551 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 552 553 554class SessionTokenRequestBearerAuthenticator(BaseModel): 555 type: Literal["Bearer"] 556 557 558class HttpMethod(Enum): 559 GET = "GET" 560 POST = "POST" 561 562 563class QuotaStatusSource(BaseModel): 564 type: Literal["QuotaStatusSource"] 565 url: str = Field( 566 ..., 567 description="The full URL of the quota status endpoint.", 568 examples=[ 569 "https://api.github.com/rate_limit", 570 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 571 ], 572 title="URL", 573 ) 574 http_method: Optional[HttpMethod] = Field( 575 HttpMethod.GET, 576 description="The HTTP method used to fetch the quota status.", 577 title="HTTP Method", 578 ) 579 request_headers: Optional[Dict[str, str]] = Field( 580 None, 581 description="Additional headers to send with the quota status request.", 582 title="Request Headers", 583 ) 584 unavailable_status_codes: Optional[List[int]] = Field( 585 None, 586 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 587 examples=[[404]], 588 title="Unavailable Status Codes", 589 unique_items=True, 590 ) 591 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 592 593 594class TokenQuota(BaseModel): 595 type: Literal["TokenQuota"] 596 name: str = Field( 597 ..., 598 description="Name of the quota pool.", 599 examples=["rest", "graphql"], 600 title="Name", 601 ) 602 remaining_path: List[str] = Field( 603 ..., 604 description="Path to the remaining call count for this pool in the quota status response.", 605 examples=[["resources", "core", "remaining"]], 606 title="Remaining Path", 607 ) 608 reset_path: List[str] = Field( 609 ..., 610 description="Path to the quota reset timestamp for this pool in the quota status response.", 611 examples=[["resources", "core", "reset"]], 612 title="Reset Path", 613 ) 614 limit_path: Optional[List[str]] = Field( 615 None, 616 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 617 examples=[["resources", "core", "limit"]], 618 title="Limit Path", 619 ) 620 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 621 None, 622 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 623 title="Matchers", 624 ) 625 remaining_header: Optional[str] = Field( 626 None, 627 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 628 examples=["X-RateLimit-Remaining"], 629 title="Remaining Header", 630 ) 631 reset_header: Optional[str] = Field( 632 None, 633 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 634 examples=["X-RateLimit-Reset"], 635 title="Reset Header", 636 ) 637 limit_header: Optional[str] = Field( 638 None, 639 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 640 examples=["X-RateLimit-Limit"], 641 title="Limit Header", 642 ) 643 exhaustion_status_codes: Optional[List[int]] = Field( 644 None, 645 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 646 examples=[[429]], 647 title="Exhaustion Status Codes", 648 ) 649 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 650 651 652class RateLimitedMultipleTokenAuthenticator(BaseModel): 653 type: Literal["RateLimitedMultipleTokenAuthenticator"] 654 tokens: Union[str, List[str]] = Field( 655 ..., 656 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 657 examples=[ 658 "{{ config['credentials']['personal_access_token'] }}", 659 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 660 ], 661 title="Tokens", 662 ) 663 token_delimiter: Optional[str] = Field( 664 ",", 665 description="Delimiter used to split a single token string into multiple tokens.", 666 title="Token Delimiter", 667 ) 668 auth_method: Optional[str] = Field( 669 "Bearer", 670 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 671 examples=["Bearer", "token"], 672 title="Auth Method", 673 ) 674 header: Optional[str] = Field( 675 "Authorization", 676 description="The name of the HTTP header in which to inject the token.", 677 title="Header Name", 678 ) 679 quota_status_source: QuotaStatusSource = Field( 680 ..., 681 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 682 title="Quota Status Source", 683 ) 684 quotas: List[TokenQuota] = Field( 685 ..., 686 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 687 min_items=1, 688 title="Quota Pools", 689 ) 690 max_wait_time: Optional[str] = Field( 691 "PT2H", 692 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 693 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 694 title="Maximum Wait Time", 695 ) 696 budget_reserve_fraction: Optional[float] = Field( 697 0.1, 698 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 699 title="Budget Reserve Fraction", 700 ) 701 budget_min_reserve: Optional[int] = Field( 702 50, 703 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 704 title="Budget Minimum Reserve", 705 ) 706 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 707 708 709class Action(Enum): 710 SUCCESS = "SUCCESS" 711 FAIL = "FAIL" 712 RETRY = "RETRY" 713 IGNORE = "IGNORE" 714 RESET_PAGINATION = "RESET_PAGINATION" 715 RATE_LIMITED = "RATE_LIMITED" 716 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 717 718 719class FailureType(Enum): 720 system_error = "system_error" 721 config_error = "config_error" 722 transient_error = "transient_error" 723 724 725class HttpResponseFilter(BaseModel): 726 type: Literal["HttpResponseFilter"] 727 action: Optional[Action] = Field( 728 None, 729 description="Action to execute if a response matches the filter.", 730 examples=[ 731 "SUCCESS", 732 "FAIL", 733 "RETRY", 734 "IGNORE", 735 "RESET_PAGINATION", 736 "RATE_LIMITED", 737 "REFRESH_TOKEN_THEN_RETRY", 738 ], 739 title="Action", 740 ) 741 failure_type: Optional[FailureType] = Field( 742 None, 743 description="Failure type of traced exception if a response matches the filter.", 744 examples=["system_error", "config_error", "transient_error"], 745 title="Failure Type", 746 ) 747 error_message: Optional[str] = Field( 748 None, 749 description="Error Message to display if the response matches the filter.", 750 title="Error Message", 751 ) 752 error_message_contains: Optional[str] = Field( 753 None, 754 description="Match the response if its error message contains the substring.", 755 example=["This API operation is not enabled for this site"], 756 title="Error Message Substring", 757 ) 758 http_codes: Optional[List[int]] = Field( 759 None, 760 description="Match the response if its HTTP code is included in this list.", 761 examples=[[420, 429], [500]], 762 title="HTTP Codes", 763 unique_items=True, 764 ) 765 predicate: Optional[str] = Field( 766 None, 767 description="Match the response if the predicate evaluates to true.", 768 examples=[ 769 "{{ 'Too much requests' in response }}", 770 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 771 ], 772 title="Predicate", 773 ) 774 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 775 776 777class ComplexFieldType(BaseModel): 778 field_type: str 779 items: Optional[Union[str, ComplexFieldType]] = None 780 781 782class TypesMap(BaseModel): 783 target_type: Union[str, List[str], ComplexFieldType] 784 current_type: Union[str, List[str]] 785 condition: Optional[str] = None 786 787 788class SchemaTypeIdentifier(BaseModel): 789 type: Optional[Literal["SchemaTypeIdentifier"]] = None 790 schema_pointer: Optional[List[str]] = Field( 791 [], 792 description="List of nested fields defining the schema field path to extract. Defaults to [].", 793 title="Schema Path", 794 ) 795 key_pointer: List[str] = Field( 796 ..., 797 description="List of potentially nested fields describing the full path of the field key to extract.", 798 title="Key Path", 799 ) 800 type_pointer: Optional[List[str]] = Field( 801 None, 802 description="List of potentially nested fields describing the full path of the field type to extract.", 803 title="Type Path", 804 ) 805 types_mapping: Optional[List[TypesMap]] = None 806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 807 808 809class InlineSchemaLoader(BaseModel): 810 type: Literal["InlineSchemaLoader"] 811 schema_: Optional[Dict[str, Any]] = Field( 812 None, 813 alias="schema", 814 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 815 title="Schema", 816 ) 817 818 819class JsonFileSchemaLoader(BaseModel): 820 type: Literal["JsonFileSchemaLoader"] 821 file_path: Optional[str] = Field( 822 None, 823 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 824 example=["./schemas/users.json"], 825 title="File Path", 826 ) 827 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 828 829 830class JsonDecoder(BaseModel): 831 type: Literal["JsonDecoder"] 832 833 834class JsonItemsDecoder(BaseModel): 835 type: Literal["JsonItemsDecoder"] 836 items_path: str = Field( 837 ..., 838 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 839 title="Items Path", 840 ) 841 encoding: Optional[str] = Field( 842 "utf-8", 843 description="The character encoding of the JSON data. Defaults to UTF-8.", 844 title="Encoding", 845 ) 846 847 848class JsonlDecoder(BaseModel): 849 type: Literal["JsonlDecoder"] 850 851 852class KeysToLower(BaseModel): 853 type: Literal["KeysToLower"] 854 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 855 856 857class KeysToSnakeCase(BaseModel): 858 type: Literal["KeysToSnakeCase"] 859 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 860 861 862class FlattenFields(BaseModel): 863 type: Literal["FlattenFields"] 864 flatten_lists: Optional[bool] = Field( 865 True, 866 description="Whether to flatten lists or leave it as is. Default is True.", 867 title="Flatten Lists", 868 ) 869 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 870 871 872class KeyTransformation(BaseModel): 873 type: Literal["KeyTransformation"] 874 prefix: Optional[str] = Field( 875 None, 876 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 877 examples=["flattened_"], 878 title="Key Prefix", 879 ) 880 suffix: Optional[str] = Field( 881 None, 882 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 883 examples=["_flattened"], 884 title="Key Suffix", 885 ) 886 887 888class DpathFlattenFields(BaseModel): 889 type: Literal["DpathFlattenFields"] 890 field_path: List[str] = Field( 891 ..., 892 description="A path to field that needs to be flattened.", 893 examples=[["data"], ["data", "*", "field"]], 894 title="Field Path", 895 ) 896 delete_origin_value: Optional[bool] = Field( 897 None, 898 description="Whether to delete the origin value or keep it. Default is False.", 899 title="Delete Origin Value", 900 ) 901 replace_record: Optional[bool] = Field( 902 None, 903 description="Whether to replace the origin record or not. Default is False.", 904 title="Replace Origin Record", 905 ) 906 key_transformation: Optional[KeyTransformation] = Field( 907 None, 908 description="Transformation for object keys. If not provided, original key will be used.", 909 title="Key transformation", 910 ) 911 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 912 913 914class KeysReplace(BaseModel): 915 type: Literal["KeysReplace"] 916 old: str = Field( 917 ..., 918 description="Old value to replace.", 919 examples=[ 920 " ", 921 "{{ record.id }}", 922 "{{ config['id'] }}", 923 "{{ stream_slice['id'] }}", 924 ], 925 title="Old value", 926 ) 927 new: str = Field( 928 ..., 929 description="New value to set.", 930 examples=[ 931 "_", 932 "{{ record.id }}", 933 "{{ config['id'] }}", 934 "{{ stream_slice['id'] }}", 935 ], 936 title="New value", 937 ) 938 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 939 940 941class IterableDecoder(BaseModel): 942 type: Literal["IterableDecoder"] 943 944 945class XmlDecoder(BaseModel): 946 type: Literal["XmlDecoder"] 947 948 949class CustomDecoder(BaseModel): 950 class Config: 951 extra = Extra.allow 952 953 type: Literal["CustomDecoder"] 954 class_name: str = Field( 955 ..., 956 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 957 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 958 title="Class Name", 959 ) 960 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 961 962 963class MinMaxDatetime(BaseModel): 964 type: Literal["MinMaxDatetime"] 965 datetime: str = Field( 966 ..., 967 description="Datetime value.", 968 examples=[ 969 "2021-01-01", 970 "2021-01-01T00:00:00Z", 971 "{{ config['start_time'] }}", 972 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 973 ], 974 title="Datetime", 975 ) 976 datetime_format: Optional[str] = Field( 977 "", 978 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 979 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 980 title="Datetime Format", 981 ) 982 max_datetime: Optional[str] = Field( 983 None, 984 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 985 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 986 title="Max Datetime", 987 ) 988 min_datetime: Optional[str] = Field( 989 None, 990 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 991 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 992 title="Min Datetime", 993 ) 994 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 995 996 997class NoAuth(BaseModel): 998 type: Literal["NoAuth"] 999 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1000 1001 1002class NoPagination(BaseModel): 1003 type: Literal["NoPagination"] 1004 1005 1006class State(BaseModel): 1007 class Config: 1008 extra = Extra.allow 1009 1010 min: int 1011 max: int 1012 1013 1014class OAuthScope(BaseModel): 1015 class Config: 1016 extra = Extra.allow 1017 1018 scope: str = Field( 1019 ..., 1020 description="The OAuth scope string to request from the provider.", 1021 ) 1022 1023 1024class OauthConnectorInputSpecification(BaseModel): 1025 class Config: 1026 extra = Extra.allow 1027 1028 consent_url: str = Field( 1029 ..., 1030 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1031 examples=[ 1032 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1033 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1034 ], 1035 title="Consent URL", 1036 ) 1037 scope: Optional[str] = Field( 1038 None, 1039 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1040 examples=["user:read user:read_orders workspaces:read"], 1041 title="Scopes", 1042 ) 1043 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1044 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1045 # The CDK schema defines the manifest contract; the platform reads these fields 1046 # during the OAuth consent flow to build the authorization URL. 1047 scopes: Optional[List[OAuthScope]] = Field( 1048 None, 1049 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1050 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1051 title="Scopes", 1052 ) 1053 optional_scopes: Optional[List[OAuthScope]] = Field( 1054 None, 1055 description="Optional OAuth scope objects that may or may not be granted.", 1056 examples=[[{"scope": "admin:read"}]], 1057 title="Optional Scopes", 1058 ) 1059 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1060 ScopesJoinStrategy.space, 1061 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1062 title="Scopes Join Strategy", 1063 ) 1064 access_token_url: str = Field( 1065 ..., 1066 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1067 examples=[ 1068 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1069 ], 1070 title="Access Token URL", 1071 ) 1072 access_token_headers: Optional[Dict[str, Any]] = Field( 1073 None, 1074 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1075 examples=[ 1076 { 1077 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1078 } 1079 ], 1080 title="Access Token Headers", 1081 ) 1082 access_token_params: Optional[Dict[str, Any]] = Field( 1083 None, 1084 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1085 examples=[ 1086 { 1087 "{{ auth_code_key }}": "{{ auth_code_value }}", 1088 "{{ client_id_key }}": "{{ client_id_value }}", 1089 "{{ client_secret_key }}": "{{ client_secret_value }}", 1090 } 1091 ], 1092 title="Access Token Query Params (Json Encoded)", 1093 ) 1094 extract_output: Optional[List[str]] = Field( 1095 None, 1096 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1097 examples=[["access_token", "refresh_token", "other_field"]], 1098 title="Extract Output", 1099 ) 1100 state: Optional[State] = Field( 1101 None, 1102 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1103 examples=[{"min": 7, "max": 128}], 1104 title="Configurable State Query Param", 1105 ) 1106 client_id_key: Optional[str] = Field( 1107 None, 1108 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1109 examples=["my_custom_client_id_key_name"], 1110 title="Client ID Key Override", 1111 ) 1112 client_secret_key: Optional[str] = Field( 1113 None, 1114 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1115 examples=["my_custom_client_secret_key_name"], 1116 title="Client Secret Key Override", 1117 ) 1118 scope_key: Optional[str] = Field( 1119 None, 1120 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1121 examples=["my_custom_scope_key_key_name"], 1122 title="Scopes Key Override", 1123 ) 1124 state_key: Optional[str] = Field( 1125 None, 1126 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1127 examples=["my_custom_state_key_key_name"], 1128 title="State Key Override", 1129 ) 1130 auth_code_key: Optional[str] = Field( 1131 None, 1132 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1133 examples=["my_custom_auth_code_key_name"], 1134 title="Auth Code Key Override", 1135 ) 1136 redirect_uri_key: Optional[str] = Field( 1137 None, 1138 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1139 examples=["my_custom_redirect_uri_key_name"], 1140 title="Redirect URI Key Override", 1141 ) 1142 1143 1144class OAuthConfigSpecification(BaseModel): 1145 class Config: 1146 extra = Extra.allow 1147 1148 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1149 None, 1150 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1151 examples=[ 1152 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1153 { 1154 "app_id": { 1155 "type": "string", 1156 "path_in_connector_config": ["info", "app_id"], 1157 } 1158 }, 1159 ], 1160 title="OAuth user input", 1161 ) 1162 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1163 None, 1164 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1165 title="DeclarativeOAuth Connector Specification", 1166 ) 1167 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1168 None, 1169 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1170 examples=[ 1171 { 1172 "refresh_token": { 1173 "type": "string,", 1174 "path_in_connector_config": ["credentials", "refresh_token"], 1175 } 1176 } 1177 ], 1178 title="OAuth output specification", 1179 ) 1180 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1181 None, 1182 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1183 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1184 title="OAuth input specification", 1185 ) 1186 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1187 None, 1188 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1189 examples=[ 1190 { 1191 "client_id": { 1192 "type": "string,", 1193 "path_in_connector_config": ["credentials", "client_id"], 1194 }, 1195 "client_secret": { 1196 "type": "string,", 1197 "path_in_connector_config": ["credentials", "client_secret"], 1198 }, 1199 } 1200 ], 1201 title="OAuth server output specification", 1202 ) 1203 1204 1205class OffsetIncrement(BaseModel): 1206 type: Literal["OffsetIncrement"] 1207 page_size: Optional[Union[int, str]] = Field( 1208 None, 1209 description="The number of records to include in each pages.", 1210 examples=[100, "{{ config['page_size'] }}"], 1211 title="Limit", 1212 ) 1213 inject_on_first_request: Optional[bool] = Field( 1214 False, 1215 description="Using the `offset` with value `0` during the first request", 1216 title="Inject Offset on First Request", 1217 ) 1218 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1219 1220 1221class PageIncrement(BaseModel): 1222 type: Literal["PageIncrement"] 1223 page_size: Optional[Union[int, str]] = Field( 1224 None, 1225 description="The number of records to include in each pages.", 1226 examples=[100, "100", "{{ config['page_size'] }}"], 1227 title="Page Size", 1228 ) 1229 start_from_page: Optional[int] = Field( 1230 0, 1231 description="Index of the first page to request.", 1232 examples=[0, 1], 1233 title="Start From Page", 1234 ) 1235 inject_on_first_request: Optional[bool] = Field( 1236 False, 1237 description="Using the `page number` with value defined by `start_from_page` during the first request", 1238 title="Inject Page Number on First Request", 1239 ) 1240 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1241 1242 1243class PrimaryKey(BaseModel): 1244 __root__: Union[str, List[str], List[List[str]]] = Field( 1245 ..., 1246 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1247 examples=["id", ["code", "type"]], 1248 title="Primary Key", 1249 ) 1250 1251 1252class PropertyLimitType(Enum): 1253 characters = "characters" 1254 property_count = "property_count" 1255 1256 1257class PropertyChunking(BaseModel): 1258 type: Literal["PropertyChunking"] 1259 property_limit_type: PropertyLimitType = Field( 1260 ..., 1261 description="The type used to determine the maximum number of properties per chunk", 1262 title="Property Limit Type", 1263 ) 1264 property_limit: Optional[int] = Field( 1265 None, 1266 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1267 title="Property Limit", 1268 ) 1269 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1270 None, 1271 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1272 title="Record Merge Strategy", 1273 ) 1274 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1275 1276 1277class RecordFilter(BaseModel): 1278 type: Literal["RecordFilter"] 1279 condition: Optional[str] = Field( 1280 "", 1281 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1282 examples=[ 1283 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1284 "{{ record.status in ['active', 'expired'] }}", 1285 ], 1286 title="Condition", 1287 ) 1288 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1289 1290 1291class SchemaNormalization(Enum): 1292 Default = "Default" 1293 None_ = "None" 1294 1295 1296class RemoveFields(BaseModel): 1297 type: Literal["RemoveFields"] 1298 condition: Optional[str] = Field( 1299 "", 1300 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1301 examples=[ 1302 "{{ property|string == '' }}", 1303 "{{ property is integer }}", 1304 "{{ property|length > 5 }}", 1305 "{{ property == 'some_string_to_match' }}", 1306 ], 1307 ) 1308 field_pointers: List[List[str]] = Field( 1309 ..., 1310 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1311 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1312 title="Field Paths", 1313 ) 1314 1315 1316class RequestPath(BaseModel): 1317 type: Literal["RequestPath"] 1318 1319 1320class InjectInto(Enum): 1321 request_parameter = "request_parameter" 1322 header = "header" 1323 body_data = "body_data" 1324 body_json = "body_json" 1325 1326 1327class RequestOption(BaseModel): 1328 type: Literal["RequestOption"] 1329 inject_into: InjectInto = Field( 1330 ..., 1331 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1332 examples=["request_parameter", "header", "body_data", "body_json"], 1333 title="Inject Into", 1334 ) 1335 field_name: Optional[str] = Field( 1336 None, 1337 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1338 examples=["segment_id"], 1339 title="Field Name", 1340 ) 1341 field_path: Optional[List[str]] = Field( 1342 None, 1343 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1344 examples=[["data", "viewer", "id"]], 1345 title="Field Path", 1346 ) 1347 1348 1349class Schemas(BaseModel): 1350 pass 1351 1352 class Config: 1353 extra = Extra.allow 1354 1355 1356class LegacySessionTokenAuthenticator(BaseModel): 1357 type: Literal["LegacySessionTokenAuthenticator"] 1358 header: str = Field( 1359 ..., 1360 description="The name of the session token header that will be injected in the request", 1361 examples=["X-Session"], 1362 title="Session Request Header", 1363 ) 1364 login_url: str = Field( 1365 ..., 1366 description="Path of the login URL (do not include the base URL)", 1367 examples=["session"], 1368 title="Login Path", 1369 ) 1370 session_token: Optional[str] = Field( 1371 None, 1372 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1373 example=["{{ config['session_token'] }}"], 1374 title="Session Token", 1375 ) 1376 session_token_response_key: str = Field( 1377 ..., 1378 description="Name of the key of the session token to be extracted from the response", 1379 examples=["id"], 1380 title="Response Token Response Key", 1381 ) 1382 username: Optional[str] = Field( 1383 None, 1384 description="Username used to authenticate and obtain a session token", 1385 examples=[" {{ config['username'] }}"], 1386 title="Username", 1387 ) 1388 password: Optional[str] = Field( 1389 "", 1390 description="Password used to authenticate and obtain a session token", 1391 examples=["{{ config['password'] }}", ""], 1392 title="Password", 1393 ) 1394 validate_session_url: str = Field( 1395 ..., 1396 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1397 examples=["user/current"], 1398 title="Validate Session Path", 1399 ) 1400 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1401 1402 1403class Action1(Enum): 1404 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1405 RESET = "RESET" 1406 1407 1408class PaginationResetLimits(BaseModel): 1409 type: Literal["PaginationResetLimits"] 1410 number_of_records: Optional[int] = None 1411 1412 1413class CsvDecoder(BaseModel): 1414 type: Literal["CsvDecoder"] 1415 encoding: Optional[str] = "utf-8" 1416 delimiter: Optional[str] = "," 1417 set_values_to_none: Optional[List[str]] = None 1418 1419 1420class AsyncJobStatusMap(BaseModel): 1421 type: Optional[Literal["AsyncJobStatusMap"]] = None 1422 running: List[str] 1423 completed: List[str] 1424 failed: List[str] 1425 timeout: List[str] 1426 skipped: Optional[List[str]] = None 1427 1428 1429class ValueType(Enum): 1430 string = "string" 1431 number = "number" 1432 integer = "integer" 1433 boolean = "boolean" 1434 1435 1436class WaitTimeFromHeader(BaseModel): 1437 type: Literal["WaitTimeFromHeader"] 1438 header: str = Field( 1439 ..., 1440 description="The name of the response header defining how long to wait before retrying.", 1441 examples=["Retry-After"], 1442 title="Response Header Name", 1443 ) 1444 regex: Optional[str] = Field( 1445 None, 1446 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1447 examples=["([-+]?\\d+)"], 1448 title="Extraction Regex", 1449 ) 1450 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1451 None, 1452 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1453 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1454 title="Max Waiting Time in Seconds", 1455 ) 1456 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1457 1458 1459class WaitUntilTimeFromHeader(BaseModel): 1460 type: Literal["WaitUntilTimeFromHeader"] 1461 header: str = Field( 1462 ..., 1463 description="The name of the response header defining how long to wait before retrying.", 1464 examples=["wait_time"], 1465 title="Response Header", 1466 ) 1467 min_wait: Optional[Union[float, str]] = Field( 1468 None, 1469 description="Minimum time to wait before retrying.", 1470 examples=[10, "60"], 1471 title="Minimum Wait Time", 1472 ) 1473 regex: Optional[str] = Field( 1474 None, 1475 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1476 examples=["([-+]?\\d+)"], 1477 title="Extraction Regex", 1478 ) 1479 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1480 None, 1481 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1482 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1483 title="Max Waiting Time in Seconds", 1484 ) 1485 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1486 1487 1488class ComponentMappingDefinition(BaseModel): 1489 type: Literal["ComponentMappingDefinition"] 1490 field_path: List[str] = Field( 1491 ..., 1492 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1493 examples=[ 1494 ["name"], 1495 ["retriever", "requester", "url"], 1496 ["retriever", "requester", "{{ components_values.field }}"], 1497 ["*", "**", "name"], 1498 ], 1499 title="Field Path", 1500 ) 1501 value: str = Field( 1502 ..., 1503 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1504 examples=[ 1505 "{{ components_values['updates'] }}", 1506 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1507 "{{ config['segment_id'] }}", 1508 "{{ stream_slice['parent_id'] }}", 1509 "{{ stream_slice['extra_fields']['name'] }}", 1510 ], 1511 title="Value", 1512 ) 1513 value_type: Optional[ValueType] = Field( 1514 None, 1515 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1516 title="Value Type", 1517 ) 1518 create_or_update: Optional[bool] = Field( 1519 False, 1520 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1521 title="Create or Update", 1522 ) 1523 condition: Optional[str] = Field( 1524 None, 1525 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1526 examples=[ 1527 "{{ components_values.get('cursor_field', None) }}", 1528 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1529 ], 1530 title="Condition", 1531 ) 1532 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1533 1534 1535class StreamConfig(BaseModel): 1536 type: Literal["StreamConfig"] 1537 configs_pointer: List[str] = Field( 1538 ..., 1539 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1540 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1541 title="Configs Pointer", 1542 ) 1543 default_values: Optional[List[Dict[str, Any]]] = Field( 1544 None, 1545 description="A list of default values, each matching the structure expected from the parsed component value.", 1546 title="Default Values", 1547 ) 1548 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1549 1550 1551class ConfigComponentsResolver(BaseModel): 1552 type: Literal["ConfigComponentsResolver"] 1553 stream_config: Union[List[StreamConfig], StreamConfig] 1554 components_mapping: List[ComponentMappingDefinition] 1555 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1556 1557 1558class StreamParametersDefinition(BaseModel): 1559 type: Literal["StreamParametersDefinition"] 1560 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1561 ..., 1562 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1563 examples=[ 1564 [ 1565 { 1566 "name": "test stream", 1567 "$parameters": {"entity": "test entity"}, 1568 "primary_key": "test key", 1569 } 1570 ] 1571 ], 1572 title="Stream Parameters", 1573 ) 1574 1575 1576class ParametrizedComponentsResolver(BaseModel): 1577 type: Literal["ParametrizedComponentsResolver"] 1578 stream_parameters: StreamParametersDefinition 1579 components_mapping: List[ComponentMappingDefinition] 1580 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1581 1582 1583class RequestBodyPlainText(BaseModel): 1584 type: Literal["RequestBodyPlainText"] 1585 value: str 1586 1587 1588class RequestBodyUrlEncodedForm(BaseModel): 1589 type: Literal["RequestBodyUrlEncodedForm"] 1590 value: Dict[str, str] 1591 1592 1593class RequestBodyJsonObject(BaseModel): 1594 type: Literal["RequestBodyJsonObject"] 1595 value: Dict[str, Any] 1596 1597 1598class RequestBodyGraphQlQuery(BaseModel): 1599 class Config: 1600 extra = Extra.allow 1601 1602 query: str = Field(..., description="The GraphQL query to be executed") 1603 1604 1605class ValidateAdheresToSchema(BaseModel): 1606 type: Literal["ValidateAdheresToSchema"] 1607 base_schema: Union[str, Dict[str, Any]] = Field( 1608 ..., 1609 description="The base JSON schema against which the user-provided schema will be validated.", 1610 examples=[ 1611 "{{ config['report_validation_schema'] }}", 1612 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1613 { 1614 "$schema": "http://json-schema.org/draft-07/schema#", 1615 "title": "Person", 1616 "type": "object", 1617 "properties": { 1618 "name": {"type": "string", "description": "The person's name"}, 1619 "age": { 1620 "type": "integer", 1621 "minimum": 0, 1622 "description": "The person's age", 1623 }, 1624 }, 1625 "required": ["name", "age"], 1626 }, 1627 ], 1628 title="Base JSON Schema", 1629 ) 1630 1631 1632class CustomValidationStrategy(BaseModel): 1633 class Config: 1634 extra = Extra.allow 1635 1636 type: Literal["CustomValidationStrategy"] 1637 class_name: str = Field( 1638 ..., 1639 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1640 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1641 title="Class Name", 1642 ) 1643 1644 1645class ConfigRemapField(BaseModel): 1646 type: Literal["ConfigRemapField"] 1647 map: Union[Dict[str, Any], str] = Field( 1648 ..., 1649 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1650 examples=[ 1651 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1652 "{{ config['status_mapping'] }}", 1653 ], 1654 title="Value Mapping", 1655 ) 1656 field_path: List[str] = Field( 1657 ..., 1658 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1659 examples=[ 1660 ["status"], 1661 ["data", "status"], 1662 ["data", "{{ config.name }}", "status"], 1663 ["data", "*", "status"], 1664 ], 1665 title="Field Path", 1666 ) 1667 1668 1669class ConfigRemoveFields(BaseModel): 1670 type: Literal["ConfigRemoveFields"] 1671 field_pointers: List[List[str]] = Field( 1672 ..., 1673 description="A list of field pointers to be removed from the config.", 1674 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1675 title="Field Pointers", 1676 ) 1677 condition: Optional[str] = Field( 1678 "", 1679 description="Fields will be removed if expression is evaluated to True.", 1680 examples=[ 1681 "{{ config['environemnt'] == 'sandbox' }}", 1682 "{{ property is integer }}", 1683 "{{ property|length > 5 }}", 1684 "{{ property == 'some_string_to_match' }}", 1685 ], 1686 ) 1687 1688 1689class CustomConfigTransformation(BaseModel): 1690 type: Literal["CustomConfigTransformation"] 1691 class_name: str = Field( 1692 ..., 1693 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1694 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1695 ) 1696 parameters: Optional[Dict[str, Any]] = Field( 1697 None, 1698 alias="$parameters", 1699 description="Additional parameters to be passed to the custom config transformation.", 1700 ) 1701 1702 1703class AddedFieldDefinition(BaseModel): 1704 type: Literal["AddedFieldDefinition"] 1705 path: List[str] = Field( 1706 ..., 1707 description="List of strings defining the path where to add the value on the record.", 1708 examples=[["segment_id"], ["metadata", "segment_id"]], 1709 title="Path", 1710 ) 1711 value: str = Field( 1712 ..., 1713 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1714 examples=[ 1715 "{{ record['updates'] }}", 1716 "{{ record['MetaData']['LastUpdatedTime'] }}", 1717 "{{ stream_partition['segment_id'] }}", 1718 ], 1719 title="Value", 1720 ) 1721 value_type: Optional[ValueType] = Field( 1722 None, 1723 description="Type of the value. If not specified, the type will be inferred from the value.", 1724 title="Value Type", 1725 ) 1726 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1727 1728 1729class AddFields(BaseModel): 1730 type: Literal["AddFields"] 1731 fields: List[AddedFieldDefinition] = Field( 1732 ..., 1733 description="List of transformations (path and corresponding value) that will be added to the record.", 1734 title="Fields", 1735 ) 1736 condition: Optional[str] = Field( 1737 "", 1738 description="Fields will be added if expression is evaluated to True.", 1739 examples=[ 1740 "{{ property|string == '' }}", 1741 "{{ property is integer }}", 1742 "{{ property|length > 5 }}", 1743 "{{ property == 'some_string_to_match' }}", 1744 ], 1745 ) 1746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1747 1748 1749class ApiKeyAuthenticator(BaseModel): 1750 type: Literal["ApiKeyAuthenticator"] 1751 api_token: Optional[str] = Field( 1752 None, 1753 description="The API key to inject in the request. Fill it in the user inputs.", 1754 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1755 title="API Key", 1756 ) 1757 header: Optional[str] = Field( 1758 None, 1759 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1760 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1761 title="Header Name", 1762 ) 1763 inject_into: Optional[RequestOption] = Field( 1764 None, 1765 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1766 examples=[ 1767 {"inject_into": "header", "field_name": "Authorization"}, 1768 {"inject_into": "request_parameter", "field_name": "authKey"}, 1769 ], 1770 title="Inject API Key Into Outgoing HTTP Request", 1771 ) 1772 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1773 1774 1775class AuthFlow(BaseModel): 1776 auth_flow_type: Optional[AuthFlowType] = Field( 1777 None, description="The type of auth to use", title="Auth flow type" 1778 ) 1779 predicate_key: Optional[List[str]] = Field( 1780 None, 1781 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1782 examples=[["credentials", "auth_type"]], 1783 title="Predicate key", 1784 ) 1785 predicate_value: Optional[str] = Field( 1786 None, 1787 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1788 examples=["Oauth"], 1789 title="Predicate value", 1790 ) 1791 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1792 1793 1794class CheckStream(BaseModel): 1795 type: Literal["CheckStream"] 1796 stream_names: Optional[List[str]] = Field( 1797 None, 1798 description="Names of the streams to try reading from when running a check operation.", 1799 examples=[["users"], ["users", "contacts"]], 1800 title="Stream Names", 1801 ) 1802 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1803 config_overrides: Optional[Dict[str, Any]] = Field( 1804 None, 1805 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1806 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1807 title="Config Overrides", 1808 ) 1809 1810 1811class IncrementingCountCursor(BaseModel): 1812 type: Literal["IncrementingCountCursor"] 1813 cursor_field: str = Field( 1814 ..., 1815 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1816 examples=["created_at", "{{ config['record_cursor'] }}"], 1817 title="Cursor Field", 1818 ) 1819 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1820 None, 1821 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1822 title="Allow Catalog Defined Cursor Field", 1823 ) 1824 start_value: Optional[Union[str, int]] = Field( 1825 None, 1826 description="The value that determines the earliest record that should be synced.", 1827 examples=[0, "{{ config['start_value'] }}"], 1828 title="Start Value", 1829 ) 1830 start_value_option: Optional[RequestOption] = Field( 1831 None, 1832 description="Optionally configures how the start value will be sent in requests to the source API.", 1833 title="Inject Start Value Into Outgoing HTTP Request", 1834 ) 1835 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1836 1837 1838class DatetimeBasedCursor(BaseModel): 1839 type: Literal["DatetimeBasedCursor"] 1840 clamping: Optional[Clamping] = Field( 1841 None, 1842 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1843 title="Date Range Clamping", 1844 ) 1845 cursor_field: str = Field( 1846 ..., 1847 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1848 examples=["created_at", "{{ config['record_cursor'] }}"], 1849 title="Cursor Field", 1850 ) 1851 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1852 None, 1853 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1854 title="Allow Catalog Defined Cursor Field", 1855 ) 1856 cursor_datetime_formats: Optional[List[str]] = Field( 1857 None, 1858 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1859 examples=[ 1860 "%Y-%m-%d", 1861 "%Y-%m-%d %H:%M:%S", 1862 "%Y-%m-%dT%H:%M:%S", 1863 "%Y-%m-%dT%H:%M:%SZ", 1864 "%Y-%m-%dT%H:%M:%S%z", 1865 "%Y-%m-%dT%H:%M:%S.%fZ", 1866 "%Y-%m-%dT%H:%M:%S.%f%z", 1867 "%Y-%m-%d %H:%M:%S.%f+00:00", 1868 "%s", 1869 "%ms", 1870 ], 1871 title="Cursor Datetime Formats", 1872 ) 1873 start_datetime: Union[MinMaxDatetime, str] = Field( 1874 ..., 1875 description="The datetime that determines the earliest record that should be synced.", 1876 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1877 title="Start Datetime", 1878 ) 1879 start_time_option: Optional[RequestOption] = Field( 1880 None, 1881 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1882 title="Inject Start Time Into Outgoing HTTP Request", 1883 ) 1884 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1885 None, 1886 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1887 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1888 title="End Datetime", 1889 ) 1890 end_time_option: Optional[RequestOption] = Field( 1891 None, 1892 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1893 title="Inject End Time Into Outgoing HTTP Request", 1894 ) 1895 datetime_format: str = Field( 1896 ..., 1897 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1898 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1899 title="Outgoing Datetime Format", 1900 ) 1901 cursor_granularity: Optional[str] = Field( 1902 None, 1903 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1904 examples=["PT1S"], 1905 title="Cursor Granularity", 1906 ) 1907 is_data_feed: Optional[bool] = Field( 1908 None, 1909 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1910 title="Data Feed API", 1911 ) 1912 is_client_side_incremental: Optional[bool] = Field( 1913 None, 1914 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1915 title="Client-side Incremental Filtering", 1916 ) 1917 is_compare_strictly: Optional[bool] = Field( 1918 False, 1919 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1920 title="Strict Start-End Time Comparison", 1921 ) 1922 global_substream_cursor: Optional[bool] = Field( 1923 False, 1924 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1925 title="Global Substream Cursor", 1926 ) 1927 lookback_window: Optional[str] = Field( 1928 None, 1929 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1930 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1931 title="Lookback Window", 1932 ) 1933 partition_field_end: Optional[str] = Field( 1934 None, 1935 description="Name of the partition start time field.", 1936 examples=["ending_time"], 1937 title="Partition Field End", 1938 ) 1939 partition_field_start: Optional[str] = Field( 1940 None, 1941 description="Name of the partition end time field.", 1942 examples=["starting_time"], 1943 title="Partition Field Start", 1944 ) 1945 step: Optional[str] = Field( 1946 None, 1947 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1948 examples=["P1W", "{{ config['step_increment'] }}"], 1949 title="Step", 1950 ) 1951 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1952 1953 1954class JwtAuthenticator(BaseModel): 1955 type: Literal["JwtAuthenticator"] 1956 secret_key: str = Field( 1957 ..., 1958 description="Secret used to sign the JSON web token.", 1959 examples=["{{ config['secret_key'] }}"], 1960 title="Secret Key", 1961 ) 1962 base64_encode_secret_key: Optional[bool] = Field( 1963 False, 1964 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1965 title="Base64-encode Secret Key", 1966 ) 1967 algorithm: Algorithm = Field( 1968 ..., 1969 description="Algorithm used to sign the JSON web token.", 1970 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1971 title="Algorithm", 1972 ) 1973 token_duration: Optional[int] = Field( 1974 1200, 1975 description="The amount of time in seconds a JWT token can be valid after being issued.", 1976 examples=[1200, 3600], 1977 title="Token Duration", 1978 ) 1979 header_prefix: Optional[str] = Field( 1980 None, 1981 description="The prefix to be used within the Authentication header.", 1982 examples=["Bearer", "Basic"], 1983 title="Header Prefix", 1984 ) 1985 jwt_headers: Optional[JwtHeaders] = Field( 1986 None, 1987 description="JWT headers used when signing JSON web token.", 1988 title="JWT Headers", 1989 ) 1990 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1991 None, 1992 description="Additional headers to be included with the JWT headers object.", 1993 title="Additional JWT Headers", 1994 ) 1995 jwt_payload: Optional[JwtPayload] = Field( 1996 None, 1997 description="JWT Payload used when signing JSON web token.", 1998 title="JWT Payload", 1999 ) 2000 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2001 None, 2002 description="Additional properties to be added to the JWT payload.", 2003 title="Additional JWT Payload Properties", 2004 ) 2005 passphrase: Optional[str] = Field( 2006 None, 2007 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2008 examples=["{{ config['passphrase'] }}"], 2009 title="Passphrase", 2010 ) 2011 request_option: Optional[RequestOption] = Field( 2012 None, 2013 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2014 title="Request Option", 2015 ) 2016 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2017 2018 2019class OAuthAuthenticator(BaseModel): 2020 type: Literal["OAuthAuthenticator"] 2021 client_id_name: Optional[str] = Field( 2022 "client_id", 2023 description="The name of the property to use to refresh the `access_token`.", 2024 examples=["custom_app_id"], 2025 title="Client ID Property Name", 2026 ) 2027 client_id: Optional[str] = Field( 2028 None, 2029 description="The OAuth client ID. Fill it in the user inputs.", 2030 examples=[ 2031 "{{ config['client_id'] }}", 2032 "{{ config['credentials']['client_id }}", 2033 ], 2034 title="Client ID", 2035 ) 2036 client_secret_name: Optional[str] = Field( 2037 "client_secret", 2038 description="The name of the property to use to refresh the `access_token`.", 2039 examples=["custom_app_secret"], 2040 title="Client Secret Property Name", 2041 ) 2042 client_secret: Optional[str] = Field( 2043 None, 2044 description="The OAuth client secret. Fill it in the user inputs.", 2045 examples=[ 2046 "{{ config['client_secret'] }}", 2047 "{{ config['credentials']['client_secret }}", 2048 ], 2049 title="Client Secret", 2050 ) 2051 refresh_token_name: Optional[str] = Field( 2052 "refresh_token", 2053 description="The name of the property to use to refresh the `access_token`.", 2054 examples=["custom_app_refresh_value"], 2055 title="Refresh Token Property Name", 2056 ) 2057 refresh_token: Optional[str] = Field( 2058 None, 2059 description="Credential artifact used to get a new access token.", 2060 examples=[ 2061 "{{ config['refresh_token'] }}", 2062 "{{ config['credentials]['refresh_token'] }}", 2063 ], 2064 title="Refresh Token", 2065 ) 2066 token_refresh_endpoint: Optional[str] = Field( 2067 None, 2068 description="The full URL to call to obtain a new access token.", 2069 examples=["https://connect.squareup.com/oauth2/token"], 2070 title="Token Refresh Endpoint", 2071 ) 2072 access_token_name: Optional[str] = Field( 2073 "access_token", 2074 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2075 examples=["access_token"], 2076 title="Access Token Property Name", 2077 ) 2078 access_token_value: Optional[str] = Field( 2079 None, 2080 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2081 examples=["secret_access_token_value"], 2082 title="Access Token Value", 2083 ) 2084 expires_in_name: Optional[str] = Field( 2085 "expires_in", 2086 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2087 examples=["expires_in"], 2088 title="Token Expiry Property Name", 2089 ) 2090 grant_type_name: Optional[str] = Field( 2091 "grant_type", 2092 description="The name of the property to use to refresh the `access_token`.", 2093 examples=["custom_grant_type"], 2094 title="Grant Type Property Name", 2095 ) 2096 grant_type: Optional[str] = Field( 2097 "refresh_token", 2098 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2099 examples=["refresh_token", "client_credentials"], 2100 title="Grant Type", 2101 ) 2102 refresh_request_body: Optional[Dict[str, Any]] = Field( 2103 None, 2104 description="Body of the request sent to get a new access token.", 2105 examples=[ 2106 { 2107 "applicationId": "{{ config['application_id'] }}", 2108 "applicationSecret": "{{ config['application_secret'] }}", 2109 "token": "{{ config['token'] }}", 2110 } 2111 ], 2112 title="Refresh Request Body", 2113 ) 2114 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2115 None, 2116 description="Headers of the request sent to get a new access token.", 2117 examples=[ 2118 { 2119 "Authorization": "<AUTH_TOKEN>", 2120 "Content-Type": "application/x-www-form-urlencoded", 2121 } 2122 ], 2123 title="Refresh Request Headers", 2124 ) 2125 send_refresh_request_as_query_params: Optional[bool] = Field( 2126 False, 2127 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2128 examples=[True], 2129 title="Send Refresh Request As Query Params", 2130 ) 2131 scopes: Optional[List[str]] = Field( 2132 None, 2133 description="List of scopes that should be granted to the access token.", 2134 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2135 title="Scopes", 2136 ) 2137 token_expiry_date: Optional[str] = Field( 2138 None, 2139 description="The access token expiry date.", 2140 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2141 title="Token Expiry Date", 2142 ) 2143 token_expiry_date_format: Optional[str] = Field( 2144 None, 2145 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2146 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2147 title="Token Expiry Date Format", 2148 ) 2149 refresh_token_error_status_codes: Optional[List[int]] = Field( 2150 None, 2151 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2152 examples=[[400, 500]], 2153 title="Refresh Token Error Status Codes", 2154 ) 2155 refresh_token_error_key: Optional[str] = Field( 2156 None, 2157 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2158 examples=["error"], 2159 title="Refresh Token Error Key", 2160 ) 2161 refresh_token_error_values: Optional[List[str]] = Field( 2162 None, 2163 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2164 examples=[["invalid_grant", "invalid_permissions"]], 2165 title="Refresh Token Error Values", 2166 ) 2167 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2168 None, 2169 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2170 title="Refresh Token Updater", 2171 ) 2172 profile_assertion: Optional[JwtAuthenticator] = Field( 2173 None, 2174 description="The authenticator being used to authenticate the client authenticator.", 2175 title="Profile Assertion", 2176 ) 2177 use_profile_assertion: Optional[bool] = Field( 2178 False, 2179 description="Enable using profile assertion as a flow for OAuth authorization.", 2180 title="Use Profile Assertion", 2181 ) 2182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2183 2184 2185class FixedWindowCallRatePolicy(BaseModel): 2186 class Config: 2187 extra = Extra.allow 2188 2189 type: Literal["FixedWindowCallRatePolicy"] 2190 period: str = Field( 2191 ..., description="The time interval for the rate limit window.", title="Period" 2192 ) 2193 call_limit: int = Field( 2194 ..., 2195 description="The maximum number of calls allowed within the period.", 2196 title="Call Limit", 2197 ) 2198 matchers: List[HttpRequestRegexMatcher] = Field( 2199 ..., 2200 description="List of matchers that define which requests this policy applies to.", 2201 title="Matchers", 2202 ) 2203 2204 2205class MovingWindowCallRatePolicy(BaseModel): 2206 class Config: 2207 extra = Extra.allow 2208 2209 type: Literal["MovingWindowCallRatePolicy"] 2210 rates: List[Rate] = Field( 2211 ..., 2212 description="List of rates that define the call limits for different time intervals.", 2213 title="Rates", 2214 ) 2215 matchers: List[HttpRequestRegexMatcher] = Field( 2216 ..., 2217 description="List of matchers that define which requests this policy applies to.", 2218 title="Matchers", 2219 ) 2220 2221 2222class UnlimitedCallRatePolicy(BaseModel): 2223 class Config: 2224 extra = Extra.allow 2225 2226 type: Literal["UnlimitedCallRatePolicy"] 2227 matchers: List[HttpRequestRegexMatcher] = Field( 2228 ..., 2229 description="List of matchers that define which requests this policy applies to.", 2230 title="Matchers", 2231 ) 2232 2233 2234class DefaultErrorHandler(BaseModel): 2235 type: Literal["DefaultErrorHandler"] 2236 backoff_strategies: Optional[ 2237 List[ 2238 Union[ 2239 ConstantBackoffStrategy, 2240 ExponentialBackoffStrategy, 2241 WaitTimeFromHeader, 2242 WaitUntilTimeFromHeader, 2243 CustomBackoffStrategy, 2244 ] 2245 ] 2246 ] = Field( 2247 None, 2248 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2249 title="Backoff Strategies", 2250 ) 2251 max_retries: Optional[Union[int, str]] = Field( 2252 5, 2253 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2254 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2255 title="Max Retry Count", 2256 ) 2257 response_filters: Optional[List[HttpResponseFilter]] = Field( 2258 None, 2259 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2260 title="Response Filters", 2261 ) 2262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2263 2264 2265class DefaultPaginator(BaseModel): 2266 type: Literal["DefaultPaginator"] 2267 pagination_strategy: Union[ 2268 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2269 ] = Field( 2270 ..., 2271 description="Strategy defining how records are paginated.", 2272 title="Pagination Strategy", 2273 ) 2274 page_size_option: Optional[RequestOption] = Field( 2275 None, title="Inject Page Size Into Outgoing HTTP Request" 2276 ) 2277 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2278 None, 2279 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2280 title="Inject Page Token Into Outgoing HTTP Request", 2281 ) 2282 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2283 2284 2285class RecordExpander(BaseModel): 2286 type: Literal["RecordExpander"] 2287 expand_records_from_field: List[str] = Field( 2288 ..., 2289 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2290 examples=[ 2291 ["lines", "data"], 2292 ["items"], 2293 ["nested", "array"], 2294 ["sections", "*", "items"], 2295 ], 2296 title="Expand Records From Field", 2297 ) 2298 remain_original_record: Optional[bool] = Field( 2299 False, 2300 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2301 title="Remain Original Record", 2302 ) 2303 on_no_records: Optional[OnNoRecords] = Field( 2304 OnNoRecords.skip, 2305 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2306 title="On No Records", 2307 ) 2308 truncation_indicator_path: Optional[List[str]] = Field( 2309 None, 2310 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2311 examples=[["data", "object", "lines", "has_more"]], 2312 title="Truncation Indicator Path", 2313 ) 2314 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2315 None, 2316 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2317 title="Truncated List Retriever", 2318 ) 2319 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2320 2321 2322class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2323 type: Literal["ApiKey"] 2324 inject_into: RequestOption = Field( 2325 ..., 2326 description="Configure how the API Key will be sent in requests to the source API.", 2327 examples=[ 2328 {"inject_into": "header", "field_name": "Authorization"}, 2329 {"inject_into": "request_parameter", "field_name": "authKey"}, 2330 ], 2331 title="Inject API Key Into Outgoing HTTP Request", 2332 ) 2333 api_token: Optional[str] = Field( 2334 "{{ session_token }}", 2335 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2336 examples=[ 2337 "{{ session_token }}", 2338 "Token {{ session_token }}", 2339 "Bearer {{ session_token }}", 2340 ], 2341 title="API Token Template", 2342 ) 2343 2344 2345class JsonSchemaPropertySelector(BaseModel): 2346 type: Literal["JsonSchemaPropertySelector"] 2347 transformations: Optional[ 2348 List[ 2349 Union[ 2350 AddFields, 2351 RemoveFields, 2352 KeysToLower, 2353 KeysToSnakeCase, 2354 FlattenFields, 2355 DpathFlattenFields, 2356 KeysReplace, 2357 CustomTransformation, 2358 ] 2359 ] 2360 ] = Field( 2361 None, 2362 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2363 title="Transformations", 2364 ) 2365 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2366 2367 2368class ListPartitionRouter(BaseModel): 2369 type: Literal["ListPartitionRouter"] 2370 cursor_field: str = Field( 2371 ..., 2372 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2373 examples=["section", "{{ config['section_key'] }}"], 2374 title="Current Partition Value Identifier", 2375 ) 2376 values: Union[str, List[str]] = Field( 2377 ..., 2378 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2379 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2380 title="Partition Values", 2381 ) 2382 request_option: Optional[RequestOption] = Field( 2383 None, 2384 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2385 title="Inject Partition Value Into Outgoing HTTP Request", 2386 ) 2387 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2388 2389 2390class PaginationReset(BaseModel): 2391 type: Literal["PaginationReset"] 2392 action: Action1 2393 limits: Optional[PaginationResetLimits] = None 2394 2395 2396class GzipDecoder(BaseModel): 2397 type: Literal["GzipDecoder"] 2398 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2399 2400 2401class RequestBodyGraphQL(BaseModel): 2402 type: Literal["RequestBodyGraphQL"] 2403 value: RequestBodyGraphQlQuery 2404 2405 2406class DpathValidator(BaseModel): 2407 type: Literal["DpathValidator"] 2408 field_path: List[str] = Field( 2409 ..., 2410 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2411 examples=[ 2412 ["data"], 2413 ["data", "records"], 2414 ["data", "{{ parameters.name }}"], 2415 ["data", "*", "record"], 2416 ], 2417 title="Field Path", 2418 ) 2419 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2420 ..., 2421 description="The condition that the specified config value will be evaluated against", 2422 title="Validation Strategy", 2423 ) 2424 2425 2426class PredicateValidator(BaseModel): 2427 type: Literal["PredicateValidator"] 2428 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2429 ..., 2430 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2431 examples=[ 2432 "test-value", 2433 "{{ config['api_version'] }}", 2434 "{{ config['tenant_id'] }}", 2435 123, 2436 ], 2437 title="Value", 2438 ) 2439 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2440 ..., 2441 description="The validation strategy to apply to the value.", 2442 title="Validation Strategy", 2443 ) 2444 2445 2446class ConfigAddFields(BaseModel): 2447 type: Literal["ConfigAddFields"] 2448 fields: List[AddedFieldDefinition] = Field( 2449 ..., 2450 description="A list of transformations (path and corresponding value) that will be added to the config.", 2451 title="Fields", 2452 ) 2453 condition: Optional[str] = Field( 2454 "", 2455 description="Fields will be added if expression is evaluated to True.", 2456 examples=[ 2457 "{{ config['environemnt'] == 'sandbox' }}", 2458 "{{ property is integer }}", 2459 "{{ property|length > 5 }}", 2460 "{{ property == 'some_string_to_match' }}", 2461 ], 2462 ) 2463 2464 2465class CompositeErrorHandler(BaseModel): 2466 type: Literal["CompositeErrorHandler"] 2467 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2468 Field( 2469 ..., 2470 description="List of error handlers to iterate on to determine how to handle a failed response.", 2471 title="Error Handlers", 2472 ) 2473 ) 2474 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2475 2476 2477class HTTPAPIBudget(BaseModel): 2478 class Config: 2479 extra = Extra.allow 2480 2481 type: Literal["HTTPAPIBudget"] 2482 policies: List[ 2483 Union[ 2484 FixedWindowCallRatePolicy, 2485 MovingWindowCallRatePolicy, 2486 UnlimitedCallRatePolicy, 2487 ] 2488 ] = Field( 2489 ..., 2490 description="List of call rate policies that define how many calls are allowed.", 2491 title="Policies", 2492 ) 2493 ratelimit_reset_header: Optional[str] = Field( 2494 "ratelimit-reset", 2495 description="The HTTP response header name that indicates when the rate limit resets.", 2496 title="Rate Limit Reset Header", 2497 ) 2498 ratelimit_remaining_header: Optional[str] = Field( 2499 "ratelimit-remaining", 2500 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2501 title="Rate Limit Remaining Header", 2502 ) 2503 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2504 [429], 2505 description="List of HTTP status codes that indicate a rate limit has been hit.", 2506 title="Status Codes for Rate Limit Hit", 2507 ) 2508 2509 2510class DpathExtractor(BaseModel): 2511 type: Literal["DpathExtractor"] 2512 field_path: List[str] = Field( 2513 ..., 2514 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2515 examples=[ 2516 ["data"], 2517 ["data", "records"], 2518 ["data", "{{ parameters.name }}"], 2519 ["data", "*", "record"], 2520 ], 2521 title="Field Path", 2522 ) 2523 record_expander: Optional[RecordExpander] = Field( 2524 None, 2525 description="Optional component to expand records by extracting items from nested array fields.", 2526 title="Record Expander", 2527 ) 2528 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2529 2530 2531class ZipfileDecoder(BaseModel): 2532 class Config: 2533 extra = Extra.allow 2534 2535 type: Literal["ZipfileDecoder"] 2536 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2537 ..., 2538 description="Parser to parse the decompressed data from the zipfile(s).", 2539 title="Parser", 2540 ) 2541 2542 2543class RecordSelector(BaseModel): 2544 type: Literal["RecordSelector"] 2545 extractor: Union[DpathExtractor, CustomRecordExtractor] 2546 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2547 None, 2548 description="Responsible for filtering records to be emitted by the Source.", 2549 title="Record Filter", 2550 ) 2551 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2552 None, 2553 description="Responsible for normalization according to the schema.", 2554 title="Schema Normalization", 2555 ) 2556 transform_before_filtering: Optional[bool] = Field( 2557 None, 2558 description="If true, transformation will be applied before record filtering.", 2559 title="Transform Before Filtering", 2560 ) 2561 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2562 2563 2564class ConfigMigration(BaseModel): 2565 type: Literal["ConfigMigration"] 2566 description: Optional[str] = Field( 2567 None, description="The description/purpose of the config migration." 2568 ) 2569 transformations: List[ 2570 Union[ 2571 ConfigRemapField, 2572 ConfigAddFields, 2573 ConfigRemoveFields, 2574 CustomConfigTransformation, 2575 ] 2576 ] = Field( 2577 ..., 2578 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2579 title="Transformations", 2580 ) 2581 2582 2583class ConfigNormalizationRules(BaseModel): 2584 class Config: 2585 extra = Extra.forbid 2586 2587 type: Literal["ConfigNormalizationRules"] 2588 config_migrations: Optional[List[ConfigMigration]] = Field( 2589 [], 2590 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2591 title="Config Migrations", 2592 ) 2593 transformations: Optional[ 2594 List[ 2595 Union[ 2596 ConfigRemapField, 2597 ConfigAddFields, 2598 ConfigRemoveFields, 2599 CustomConfigTransformation, 2600 ] 2601 ] 2602 ] = Field( 2603 [], 2604 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2605 title="Transformations", 2606 ) 2607 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2608 [], 2609 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2610 title="Validations", 2611 ) 2612 2613 2614class Spec(BaseModel): 2615 type: Literal["Spec"] 2616 connection_specification: Dict[str, Any] = Field( 2617 ..., 2618 description="A connection specification describing how a the connector can be configured.", 2619 title="Connection Specification", 2620 ) 2621 documentation_url: Optional[str] = Field( 2622 None, 2623 description="URL of the connector's documentation page.", 2624 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2625 title="Documentation URL", 2626 ) 2627 advanced_auth: Optional[AuthFlow] = Field( 2628 None, 2629 description="Advanced specification for configuring the authentication flow.", 2630 title="Advanced Auth", 2631 ) 2632 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2633 None, title="Config Normalization Rules" 2634 ) 2635 2636 2637class DeclarativeSource1(BaseModel): 2638 class Config: 2639 extra = Extra.forbid 2640 2641 type: Literal["DeclarativeSource"] 2642 check: Union[CheckStream, CheckDynamicStream] 2643 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2644 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2645 version: str = Field( 2646 ..., 2647 description="The version of the Airbyte CDK used to build and test the source.", 2648 ) 2649 schemas: Optional[Schemas] = None 2650 definitions: Optional[Dict[str, Any]] = None 2651 spec: Optional[Spec] = None 2652 concurrency_level: Optional[ConcurrencyLevel] = None 2653 api_budget: Optional[HTTPAPIBudget] = None 2654 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2655 None, 2656 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2657 title="Stream Groups", 2658 ) 2659 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2660 None, 2661 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2662 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2663 title="Maximum Concurrent Asynchronous Jobs", 2664 ) 2665 metadata: Optional[Dict[str, Any]] = Field( 2666 None, 2667 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2668 ) 2669 description: Optional[str] = Field( 2670 None, 2671 description="A description of the connector. It will be presented on the Source documentation page.", 2672 ) 2673 2674 2675class DeclarativeSource2(BaseModel): 2676 class Config: 2677 extra = Extra.forbid 2678 2679 type: Literal["DeclarativeSource"] 2680 check: Union[CheckStream, CheckDynamicStream] 2681 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2682 None 2683 ) 2684 dynamic_streams: List[DynamicDeclarativeStream] 2685 version: str = Field( 2686 ..., 2687 description="The version of the Airbyte CDK used to build and test the source.", 2688 ) 2689 schemas: Optional[Schemas] = None 2690 definitions: Optional[Dict[str, Any]] = None 2691 spec: Optional[Spec] = None 2692 concurrency_level: Optional[ConcurrencyLevel] = None 2693 api_budget: Optional[HTTPAPIBudget] = None 2694 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2695 None, 2696 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2697 title="Stream Groups", 2698 ) 2699 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2700 None, 2701 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2702 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2703 title="Maximum Concurrent Asynchronous Jobs", 2704 ) 2705 metadata: Optional[Dict[str, Any]] = Field( 2706 None, 2707 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2708 ) 2709 description: Optional[str] = Field( 2710 None, 2711 description="A description of the connector. It will be presented on the Source documentation page.", 2712 ) 2713 2714 2715class DeclarativeSource(BaseModel): 2716 class Config: 2717 extra = Extra.forbid 2718 2719 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2720 ..., 2721 description="An API source that extracts data according to its declarative components.", 2722 title="DeclarativeSource", 2723 ) 2724 2725 2726class SelectiveAuthenticator(BaseModel): 2727 class Config: 2728 extra = Extra.allow 2729 2730 type: Literal["SelectiveAuthenticator"] 2731 authenticator_selection_path: List[str] = Field( 2732 ..., 2733 description="Path of the field in config with selected authenticator name", 2734 examples=[["auth"], ["auth", "type"]], 2735 title="Authenticator Selection Path", 2736 ) 2737 authenticators: Dict[ 2738 str, 2739 Union[ 2740 ApiKeyAuthenticator, 2741 BasicHttpAuthenticator, 2742 BearerAuthenticator, 2743 OAuthAuthenticator, 2744 JwtAuthenticator, 2745 SessionTokenAuthenticator, 2746 LegacySessionTokenAuthenticator, 2747 CustomAuthenticator, 2748 NoAuth, 2749 RateLimitedMultipleTokenAuthenticator, 2750 ], 2751 ] = Field( 2752 ..., 2753 description="Authenticators to select from.", 2754 examples=[ 2755 { 2756 "authenticators": { 2757 "token": "#/definitions/ApiKeyAuthenticator", 2758 "oauth": "#/definitions/OAuthAuthenticator", 2759 "jwt": "#/definitions/JwtAuthenticator", 2760 } 2761 } 2762 ], 2763 title="Authenticators", 2764 ) 2765 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2766 2767 2768class ConditionalStreams(BaseModel): 2769 type: Literal["ConditionalStreams"] 2770 condition: str = Field( 2771 ..., 2772 description="Condition that will be evaluated to determine if a set of streams should be available.", 2773 examples=["{{ config['is_sandbox'] }}"], 2774 title="Condition", 2775 ) 2776 streams: List[DeclarativeStream] = Field( 2777 ..., 2778 description="Streams that will be used during an operation based on the condition.", 2779 title="Streams", 2780 ) 2781 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2782 2783 2784class FileUploader(BaseModel): 2785 type: Literal["FileUploader"] 2786 requester: Union[HttpRequester, CustomRequester] = Field( 2787 ..., 2788 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2789 ) 2790 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2791 ..., 2792 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2793 ) 2794 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2795 None, 2796 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2797 ) 2798 filename_extractor: Optional[str] = Field( 2799 None, 2800 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2801 examples=[ 2802 "{{ record.id }}/{{ record.file_name }}/", 2803 "{{ record.id }}_{{ record.file_name }}/", 2804 ], 2805 ) 2806 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2807 2808 2809class DeclarativeStream(BaseModel): 2810 class Config: 2811 extra = Extra.allow 2812 2813 type: Literal["DeclarativeStream"] 2814 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2815 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2816 ..., 2817 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2818 title="Retriever", 2819 ) 2820 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2821 None, 2822 description="Component used to fetch data incrementally based on a time field in the data.", 2823 title="Incremental Sync", 2824 ) 2825 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2826 schema_loader: Optional[ 2827 Union[ 2828 InlineSchemaLoader, 2829 DynamicSchemaLoader, 2830 JsonFileSchemaLoader, 2831 List[ 2832 Union[ 2833 InlineSchemaLoader, 2834 DynamicSchemaLoader, 2835 JsonFileSchemaLoader, 2836 CustomSchemaLoader, 2837 ] 2838 ], 2839 CustomSchemaLoader, 2840 ] 2841 ] = Field( 2842 None, 2843 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2844 title="Schema Loader", 2845 ) 2846 transformations: Optional[ 2847 List[ 2848 Union[ 2849 AddFields, 2850 RemoveFields, 2851 KeysToLower, 2852 KeysToSnakeCase, 2853 FlattenFields, 2854 DpathFlattenFields, 2855 KeysReplace, 2856 CustomTransformation, 2857 ] 2858 ] 2859 ] = Field( 2860 None, 2861 description="A list of transformations to be applied to each output record.", 2862 title="Transformations", 2863 ) 2864 state_migrations: Optional[ 2865 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2866 ] = Field( 2867 [], 2868 description="Array of state migrations to be applied on the input state", 2869 title="State Migrations", 2870 ) 2871 file_uploader: Optional[FileUploader] = Field( 2872 None, 2873 description="(experimental) Describes how to fetch a file", 2874 title="File Uploader", 2875 ) 2876 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2877 2878 2879class SessionTokenAuthenticator(BaseModel): 2880 type: Literal["SessionTokenAuthenticator"] 2881 login_requester: HttpRequester = Field( 2882 ..., 2883 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2884 examples=[ 2885 { 2886 "type": "HttpRequester", 2887 "url_base": "https://my_api.com", 2888 "path": "/login", 2889 "authenticator": { 2890 "type": "BasicHttpAuthenticator", 2891 "username": "{{ config.username }}", 2892 "password": "{{ config.password }}", 2893 }, 2894 } 2895 ], 2896 title="Login Requester", 2897 ) 2898 session_token_path: List[str] = Field( 2899 ..., 2900 description="The path in the response body returned from the login requester to the session token.", 2901 examples=[["access_token"], ["result", "token"]], 2902 title="Session Token Path", 2903 ) 2904 expiration_duration: Optional[str] = Field( 2905 None, 2906 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2907 examples=["PT1H", "P1D"], 2908 title="Expiration Duration", 2909 ) 2910 request_authentication: Union[ 2911 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2912 ] = Field( 2913 ..., 2914 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2915 title="Data Request Authentication", 2916 ) 2917 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2918 None, description="Component used to decode the response.", title="Decoder" 2919 ) 2920 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2921 2922 2923class HttpRequester(BaseModelWithDeprecations): 2924 type: Literal["HttpRequester"] 2925 url_base: Optional[str] = Field( 2926 None, 2927 deprecated=True, 2928 deprecation_message="Use `url` field instead.", 2929 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2930 examples=[ 2931 "https://connect.squareup.com/v2", 2932 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2933 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2934 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2935 ], 2936 title="API Base URL", 2937 ) 2938 url: Optional[str] = Field( 2939 None, 2940 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2941 examples=[ 2942 "https://connect.squareup.com/v2", 2943 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2944 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2945 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2946 ], 2947 title="API Endpoint URL", 2948 ) 2949 path: Optional[str] = Field( 2950 None, 2951 deprecated=True, 2952 deprecation_message="Use `url` field instead.", 2953 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2954 examples=[ 2955 "/products", 2956 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2957 "/trades/{{ config['symbol_id'] }}/history", 2958 ], 2959 title="URL Path", 2960 ) 2961 http_method: Optional[HttpMethod] = Field( 2962 HttpMethod.GET, 2963 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2964 examples=["GET", "POST"], 2965 title="HTTP Method", 2966 ) 2967 authenticator: Optional[ 2968 Union[ 2969 ApiKeyAuthenticator, 2970 BasicHttpAuthenticator, 2971 BearerAuthenticator, 2972 OAuthAuthenticator, 2973 JwtAuthenticator, 2974 SessionTokenAuthenticator, 2975 SelectiveAuthenticator, 2976 CustomAuthenticator, 2977 NoAuth, 2978 LegacySessionTokenAuthenticator, 2979 RateLimitedMultipleTokenAuthenticator, 2980 ] 2981 ] = Field( 2982 None, 2983 description="Authentication method to use for requests sent to the API.", 2984 title="Authenticator", 2985 ) 2986 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2987 None, 2988 deprecated=True, 2989 deprecation_message="Use `query_properties` field instead.", 2990 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2991 title="Fetch Properties from Endpoint", 2992 ) 2993 query_properties: Optional[QueryProperties] = Field( 2994 None, 2995 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2996 title="Query Properties", 2997 ) 2998 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2999 None, 3000 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3001 examples=[ 3002 {"unit": "day"}, 3003 { 3004 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3005 }, 3006 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3007 {"sort_by[asc]": "updated_at"}, 3008 ], 3009 title="Query Parameters", 3010 ) 3011 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3012 None, 3013 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3014 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3015 title="Request Headers", 3016 ) 3017 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3018 None, 3019 deprecated=True, 3020 deprecation_message="Use `request_body` field instead.", 3021 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3022 examples=[ 3023 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3024 ], 3025 title="Request Body Payload (Non-JSON)", 3026 ) 3027 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3028 None, 3029 deprecated=True, 3030 deprecation_message="Use `request_body` field instead.", 3031 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3032 examples=[ 3033 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3034 {"key": "{{ config['value'] }}"}, 3035 {"sort": {"field": "updated_at", "order": "ascending"}}, 3036 ], 3037 title="Request Body JSON Payload", 3038 ) 3039 request_body: Optional[ 3040 Union[ 3041 RequestBodyPlainText, 3042 RequestBodyUrlEncodedForm, 3043 RequestBodyJsonObject, 3044 RequestBodyGraphQL, 3045 ] 3046 ] = Field( 3047 None, 3048 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3049 title="Request Body", 3050 ) 3051 error_handler: Optional[ 3052 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3053 ] = Field( 3054 None, 3055 description="Error handler component that defines how to handle errors.", 3056 title="Error Handler", 3057 ) 3058 use_cache: Optional[bool] = Field( 3059 False, 3060 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3061 title="Use Cache", 3062 ) 3063 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3064 3065 3066class DynamicSchemaLoader(BaseModel): 3067 type: Literal["DynamicSchemaLoader"] 3068 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3069 ..., 3070 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3071 title="Retriever", 3072 ) 3073 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3074 None, 3075 description="Responsible for filtering fields to be added to json schema.", 3076 title="Schema Filter", 3077 ) 3078 schema_transformations: Optional[ 3079 List[ 3080 Union[ 3081 AddFields, 3082 RemoveFields, 3083 KeysToLower, 3084 KeysToSnakeCase, 3085 FlattenFields, 3086 DpathFlattenFields, 3087 KeysReplace, 3088 CustomTransformation, 3089 ] 3090 ] 3091 ] = Field( 3092 None, 3093 description="A list of transformations to be applied to the schema.", 3094 title="Schema Transformations", 3095 ) 3096 schema_type_identifier: SchemaTypeIdentifier 3097 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3098 3099 3100class ParentStreamConfig(BaseModel): 3101 type: Literal["ParentStreamConfig"] 3102 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3103 ..., description="Reference to the parent stream.", title="Parent Stream" 3104 ) 3105 parent_key: str = Field( 3106 ..., 3107 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3108 examples=["id", "{{ config['parent_record_id'] }}"], 3109 title="Parent Key", 3110 ) 3111 partition_field: str = Field( 3112 ..., 3113 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3114 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3115 title="Current Parent Key Value Identifier", 3116 ) 3117 request_option: Optional[RequestOption] = Field( 3118 None, 3119 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3120 title="Request Option", 3121 ) 3122 incremental_dependency: Optional[bool] = Field( 3123 False, 3124 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3125 title="Incremental Dependency", 3126 ) 3127 lazy_read_pointer: Optional[List[str]] = Field( 3128 [], 3129 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3130 title="Lazy Read Pointer", 3131 ) 3132 extra_fields: Optional[List[List[str]]] = Field( 3133 None, 3134 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3135 title="Extra Fields", 3136 ) 3137 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3138 3139 3140class PropertiesFromEndpoint(BaseModel): 3141 type: Literal["PropertiesFromEndpoint"] 3142 property_field_path: List[str] = Field( 3143 ..., 3144 description="Describes the path to the field that should be extracted", 3145 examples=[["name"]], 3146 ) 3147 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3148 ..., 3149 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3150 ) 3151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3152 3153 3154class QueryProperties(BaseModel): 3155 type: Literal["QueryProperties"] 3156 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3157 ..., 3158 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3159 title="Property List", 3160 ) 3161 always_include_properties: Optional[List[str]] = Field( 3162 None, 3163 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3164 title="Always Include Properties", 3165 ) 3166 property_chunking: Optional[PropertyChunking] = Field( 3167 None, 3168 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3169 title="Property Chunking", 3170 ) 3171 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3172 None, 3173 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3174 title="Property Selector", 3175 ) 3176 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3177 3178 3179class StateDelegatingStream(BaseModel): 3180 type: Literal["StateDelegatingStream"] 3181 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3182 full_refresh_stream: DeclarativeStream = Field( 3183 ..., 3184 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3185 title="Full Refresh Stream", 3186 ) 3187 incremental_stream: DeclarativeStream = Field( 3188 ..., 3189 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3190 title="Incremental Stream", 3191 ) 3192 api_retention_period: Optional[str] = Field( 3193 None, 3194 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3195 examples=["P30D", "P90D", "P1Y"], 3196 title="API Retention Period", 3197 ) 3198 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3199 3200 3201class SimpleRetriever(BaseModel): 3202 type: Literal["SimpleRetriever"] 3203 requester: Union[HttpRequester, CustomRequester] = Field( 3204 ..., 3205 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3206 ) 3207 decoder: Optional[ 3208 Union[ 3209 JsonDecoder, 3210 JsonItemsDecoder, 3211 XmlDecoder, 3212 CsvDecoder, 3213 JsonlDecoder, 3214 GzipDecoder, 3215 IterableDecoder, 3216 ZipfileDecoder, 3217 CustomDecoder, 3218 ] 3219 ] = Field( 3220 None, 3221 description="Component decoding the response so records can be extracted.", 3222 title="HTTP Response Format", 3223 ) 3224 record_selector: RecordSelector = Field( 3225 ..., 3226 description="Component that describes how to extract records from a HTTP response.", 3227 ) 3228 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3229 None, 3230 description="Paginator component that describes how to navigate through the API's pages.", 3231 ) 3232 pagination_reset: Optional[PaginationReset] = Field( 3233 None, 3234 description="Describes what triggers pagination reset and how to handle it.", 3235 ) 3236 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3237 False, 3238 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3239 ) 3240 partition_router: Optional[ 3241 Union[ 3242 SubstreamPartitionRouter, 3243 ListPartitionRouter, 3244 GroupingPartitionRouter, 3245 UnionPartitionRouter, 3246 CustomPartitionRouter, 3247 List[ 3248 Union[ 3249 SubstreamPartitionRouter, 3250 ListPartitionRouter, 3251 GroupingPartitionRouter, 3252 UnionPartitionRouter, 3253 CustomPartitionRouter, 3254 ] 3255 ], 3256 ] 3257 ] = Field( 3258 None, 3259 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3260 title="Partition Router", 3261 ) 3262 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3263 3264 3265class AsyncRetriever(BaseModel): 3266 type: Literal["AsyncRetriever"] 3267 record_selector: RecordSelector = Field( 3268 ..., 3269 description="Component that describes how to extract records from a HTTP response.", 3270 ) 3271 status_mapping: AsyncJobStatusMap = Field( 3272 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3273 ) 3274 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3275 ..., description="Responsible for fetching the actual status of the async job." 3276 ) 3277 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3278 None, 3279 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3280 ) 3281 download_extractor: Optional[ 3282 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3283 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3284 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3285 ..., 3286 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3287 ) 3288 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3289 ..., 3290 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3291 ) 3292 polling_job_timeout: Optional[Union[int, str]] = Field( 3293 None, 3294 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3295 ) 3296 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3297 None, 3298 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3299 ge=1, 3300 ) 3301 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3302 None, 3303 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3304 ) 3305 download_requester: Union[HttpRequester, CustomRequester] = Field( 3306 ..., 3307 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3308 ) 3309 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3310 None, 3311 description="Paginator component that describes how to navigate through the API's pages during download.", 3312 ) 3313 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3314 None, 3315 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3316 ) 3317 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3318 None, 3319 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3320 ) 3321 partition_router: Optional[ 3322 Union[ 3323 ListPartitionRouter, 3324 SubstreamPartitionRouter, 3325 GroupingPartitionRouter, 3326 UnionPartitionRouter, 3327 CustomPartitionRouter, 3328 List[ 3329 Union[ 3330 ListPartitionRouter, 3331 SubstreamPartitionRouter, 3332 GroupingPartitionRouter, 3333 UnionPartitionRouter, 3334 CustomPartitionRouter, 3335 ] 3336 ], 3337 ] 3338 ] = Field( 3339 [], 3340 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3341 title="Partition Router", 3342 ) 3343 decoder: Optional[ 3344 Union[ 3345 CsvDecoder, 3346 GzipDecoder, 3347 JsonDecoder, 3348 JsonItemsDecoder, 3349 JsonlDecoder, 3350 IterableDecoder, 3351 XmlDecoder, 3352 ZipfileDecoder, 3353 CustomDecoder, 3354 ] 3355 ] = Field( 3356 None, 3357 description="Component decoding the response so records can be extracted.", 3358 title="HTTP Response Format", 3359 ) 3360 download_decoder: Optional[ 3361 Union[ 3362 CsvDecoder, 3363 GzipDecoder, 3364 JsonDecoder, 3365 JsonItemsDecoder, 3366 JsonlDecoder, 3367 IterableDecoder, 3368 XmlDecoder, 3369 ZipfileDecoder, 3370 CustomDecoder, 3371 ] 3372 ] = Field( 3373 None, 3374 description="Component decoding the download response so records can be extracted.", 3375 title="Download HTTP Response Format", 3376 ) 3377 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3378 3379 3380class BlockSimultaneousSyncsAction(BaseModel): 3381 type: Literal["BlockSimultaneousSyncsAction"] 3382 3383 3384class StreamGroup(BaseModel): 3385 streams: List[str] = Field( 3386 ..., 3387 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3388 title="Streams", 3389 ) 3390 action: BlockSimultaneousSyncsAction = Field( 3391 ..., 3392 description="The action to apply to streams in this group.", 3393 title="Action", 3394 ) 3395 3396 3397class SubstreamPartitionRouter(BaseModel): 3398 type: Literal["SubstreamPartitionRouter"] 3399 parent_stream_configs: List[ParentStreamConfig] = Field( 3400 ..., 3401 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3402 title="Parent Stream Configs", 3403 ) 3404 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3405 3406 3407class GroupingPartitionRouter(BaseModel): 3408 type: Literal["GroupingPartitionRouter"] 3409 group_size: int = Field( 3410 ..., 3411 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3412 examples=[10, 50], 3413 title="Group Size", 3414 ) 3415 underlying_partition_router: Union[ 3416 ListPartitionRouter, 3417 SubstreamPartitionRouter, 3418 "UnionPartitionRouter", 3419 CustomPartitionRouter, 3420 ] = Field( 3421 ..., 3422 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3423 title="Underlying Partition Router", 3424 ) 3425 deduplicate: Optional[bool] = Field( 3426 True, 3427 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3428 title="Deduplicate Partitions", 3429 ) 3430 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3431 3432 3433class UnionPartitionRouter(BaseModel): 3434 type: Literal["UnionPartitionRouter"] 3435 partition_field: str = Field( 3436 ..., 3437 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3438 examples=["repository", "{{ config['partition_field'] }}"], 3439 title="Partition Field", 3440 ) 3441 partition_routers: List[ 3442 Union[ 3443 ListPartitionRouter, 3444 SubstreamPartitionRouter, 3445 UnionPartitionRouter, 3446 CustomPartitionRouter, 3447 ] 3448 ] = Field( 3449 ..., 3450 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3451 title="Partition Routers", 3452 ) 3453 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3454 3455 3456class HttpComponentsResolver(BaseModel): 3457 type: Literal["HttpComponentsResolver"] 3458 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3459 ..., 3460 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3461 title="Retriever", 3462 ) 3463 components_mapping: List[ComponentMappingDefinition] 3464 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3465 3466 3467class DynamicDeclarativeStream(BaseModel): 3468 type: Literal["DynamicDeclarativeStream"] 3469 name: Optional[str] = Field( 3470 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3471 ) 3472 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3473 ..., description="Reference to the stream template.", title="Stream Template" 3474 ) 3475 components_resolver: Union[ 3476 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3477 ] = Field( 3478 ..., 3479 description="Component resolve and populates stream templates with components values.", 3480 title="Components Resolver", 3481 ) 3482 use_parent_parameters: Optional[bool] = Field( 3483 True, 3484 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3485 title="Use Parent Parameters", 3486 ) 3487 3488 3489ComplexFieldType.update_forward_refs() 3490GzipDecoder.update_forward_refs() 3491CompositeErrorHandler.update_forward_refs() 3492DeclarativeSource1.update_forward_refs() 3493DeclarativeSource2.update_forward_refs() 3494SelectiveAuthenticator.update_forward_refs() 3495ConditionalStreams.update_forward_refs() 3496FileUploader.update_forward_refs() 3497DeclarativeStream.update_forward_refs() 3498SessionTokenAuthenticator.update_forward_refs() 3499RecordExpander.update_forward_refs() 3500HttpRequester.update_forward_refs() 3501DynamicSchemaLoader.update_forward_refs() 3502ParentStreamConfig.update_forward_refs() 3503PropertiesFromEndpoint.update_forward_refs() 3504SimpleRetriever.update_forward_refs() 3505AsyncRetriever.update_forward_refs() 3506GroupingPartitionRouter.update_forward_refs() 3507UnionPartitionRouter.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 ) 81 config_overrides: Optional[Dict[str, Any]] = Field( 82 None, 83 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 84 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 85 title="Config Overrides", 86 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
89class ConcurrencyLevel(BaseModel): 90 type: Optional[Literal["ConcurrencyLevel"]] = None 91 default_concurrency: Union[int, str] = Field( 92 ..., 93 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 94 examples=[10, "{{ config['num_workers'] or 10 }}"], 95 title="Default Concurrency", 96 ) 97 max_concurrency: Optional[int] = Field( 98 None, 99 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 100 examples=[20, 100], 101 title="Max Concurrency", 102 ) 103 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
106class ConstantBackoffStrategy(BaseModel): 107 type: Literal["ConstantBackoffStrategy"] 108 backoff_time_in_seconds: Union[float, str] = Field( 109 ..., 110 description="Backoff time in seconds.", 111 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 112 title="Backoff Time", 113 ) 114 jitter_range_in_seconds: Optional[float] = Field( 115 None, 116 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 117 examples=[15], 118 ge=0, 119 title="Jitter Range", 120 ) 121 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
124class CursorPagination(BaseModel): 125 type: Literal["CursorPagination"] 126 cursor_value: str = Field( 127 ..., 128 description="Value of the cursor defining the next page to fetch.", 129 examples=[ 130 "{{ headers.link.next.cursor }}", 131 "{{ last_record['key'] }}", 132 "{{ response['nextPage'] }}", 133 ], 134 title="Cursor Value", 135 ) 136 page_size: Optional[Union[int, str]] = Field( 137 None, 138 description="The number of records to include in each pages.", 139 examples=[100, "{{ config['page_size'] }}"], 140 title="Page Size", 141 ) 142 stop_condition: Optional[str] = Field( 143 None, 144 description="Template string evaluating when to stop paginating.", 145 examples=[ 146 "{{ response.data.has_more is false }}", 147 "{{ 'next' not in headers['link'] }}", 148 ], 149 title="Stop Condition", 150 ) 151 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
154class CustomAuthenticator(BaseModel): 155 class Config: 156 extra = Extra.allow 157 158 type: Literal["CustomAuthenticator"] 159 class_name: str = Field( 160 ..., 161 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 162 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 163 title="Class Name", 164 ) 165 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
168class CustomBackoffStrategy(BaseModel): 169 class Config: 170 extra = Extra.allow 171 172 type: Literal["CustomBackoffStrategy"] 173 class_name: str = Field( 174 ..., 175 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 176 examples=["source_railz.components.MyCustomBackoffStrategy"], 177 title="Class Name", 178 ) 179 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
182class CustomErrorHandler(BaseModel): 183 class Config: 184 extra = Extra.allow 185 186 type: Literal["CustomErrorHandler"] 187 class_name: str = Field( 188 ..., 189 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 190 examples=["source_railz.components.MyCustomErrorHandler"], 191 title="Class Name", 192 ) 193 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
196class CustomPaginationStrategy(BaseModel): 197 class Config: 198 extra = Extra.allow 199 200 type: Literal["CustomPaginationStrategy"] 201 class_name: str = Field( 202 ..., 203 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 204 examples=["source_railz.components.MyCustomPaginationStrategy"], 205 title="Class Name", 206 ) 207 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
210class CustomRecordExtractor(BaseModel): 211 class Config: 212 extra = Extra.allow 213 214 type: Literal["CustomRecordExtractor"] 215 class_name: str = Field( 216 ..., 217 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 218 examples=["source_railz.components.MyCustomRecordExtractor"], 219 title="Class Name", 220 ) 221 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
224class CustomRecordFilter(BaseModel): 225 class Config: 226 extra = Extra.allow 227 228 type: Literal["CustomRecordFilter"] 229 class_name: str = Field( 230 ..., 231 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 232 examples=["source_railz.components.MyCustomCustomRecordFilter"], 233 title="Class Name", 234 ) 235 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
238class CustomRequester(BaseModel): 239 class Config: 240 extra = Extra.allow 241 242 type: Literal["CustomRequester"] 243 class_name: str = Field( 244 ..., 245 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 246 examples=["source_railz.components.MyCustomRecordExtractor"], 247 title="Class Name", 248 ) 249 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
252class CustomRetriever(BaseModel): 253 class Config: 254 extra = Extra.allow 255 256 type: Literal["CustomRetriever"] 257 class_name: str = Field( 258 ..., 259 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 260 examples=["source_railz.components.MyCustomRetriever"], 261 title="Class Name", 262 ) 263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
266class CustomPartitionRouter(BaseModel): 267 class Config: 268 extra = Extra.allow 269 270 type: Literal["CustomPartitionRouter"] 271 class_name: str = Field( 272 ..., 273 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 274 examples=["source_railz.components.MyCustomPartitionRouter"], 275 title="Class Name", 276 ) 277 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
280class CustomSchemaLoader(BaseModel): 281 class Config: 282 extra = Extra.allow 283 284 type: Literal["CustomSchemaLoader"] 285 class_name: str = Field( 286 ..., 287 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 288 examples=["source_railz.components.MyCustomSchemaLoader"], 289 title="Class Name", 290 ) 291 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
294class CustomSchemaNormalization(BaseModel): 295 class Config: 296 extra = Extra.allow 297 298 type: Literal["CustomSchemaNormalization"] 299 class_name: str = Field( 300 ..., 301 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 302 examples=[ 303 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 304 ], 305 title="Class Name", 306 ) 307 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
310class CustomStateMigration(BaseModel): 311 class Config: 312 extra = Extra.allow 313 314 type: Literal["CustomStateMigration"] 315 class_name: str = Field( 316 ..., 317 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 318 examples=["source_railz.components.MyCustomStateMigration"], 319 title="Class Name", 320 ) 321 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
324class CustomTransformation(BaseModel): 325 class Config: 326 extra = Extra.allow 327 328 type: Literal["CustomTransformation"] 329 class_name: str = Field( 330 ..., 331 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 332 examples=["source_railz.components.MyCustomTransformation"], 333 title="Class Name", 334 ) 335 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
355class Algorithm(Enum): 356 HS256 = "HS256" 357 HS384 = "HS384" 358 HS512 = "HS512" 359 ES256 = "ES256" 360 ES256K = "ES256K" 361 ES384 = "ES384" 362 ES512 = "ES512" 363 RS256 = "RS256" 364 RS384 = "RS384" 365 RS512 = "RS512" 366 PS256 = "PS256" 367 PS384 = "PS384" 368 PS512 = "PS512" 369 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
372class JwtHeaders(BaseModel): 373 class Config: 374 extra = Extra.forbid 375 376 kid: Optional[str] = Field( 377 None, 378 description="Private key ID for user account.", 379 examples=["{{ config['kid'] }}"], 380 title="Key Identifier", 381 ) 382 typ: Optional[str] = Field( 383 "JWT", 384 description="The media type of the complete JWT.", 385 examples=["JWT"], 386 title="Type", 387 ) 388 cty: Optional[str] = Field( 389 None, 390 description="Content type of JWT header.", 391 examples=["JWT"], 392 title="Content Type", 393 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
396class JwtPayload(BaseModel): 397 class Config: 398 extra = Extra.forbid 399 400 iss: Optional[str] = Field( 401 None, 402 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 403 examples=["{{ config['iss'] }}"], 404 title="Issuer", 405 ) 406 sub: Optional[str] = Field( 407 None, 408 description="The subject of the JWT. Commonly defined by the API.", 409 title="Subject", 410 ) 411 aud: Optional[str] = Field( 412 None, 413 description="The recipient that the JWT is intended for. Commonly defined by the API.", 414 examples=["appstoreconnect-v1"], 415 title="Audience", 416 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
419class RefreshTokenUpdater(BaseModel): 420 refresh_token_name: Optional[str] = Field( 421 "refresh_token", 422 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 423 examples=["refresh_token"], 424 title="Refresh Token Property Name", 425 ) 426 access_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "access_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "access_token"], ["access_token"]], 430 title="Config Path To Access Token", 431 ) 432 refresh_token_config_path: Optional[List[str]] = Field( 433 ["credentials", "refresh_token"], 434 description="Config path to the access token. Make sure the field actually exists in the config.", 435 examples=[["credentials", "refresh_token"], ["refresh_token"]], 436 title="Config Path To Refresh Token", 437 ) 438 token_expiry_date_config_path: Optional[List[str]] = Field( 439 ["credentials", "token_expiry_date"], 440 description="Config path to the expiry date. Make sure actually exists in the config.", 441 examples=[["credentials", "token_expiry_date"]], 442 title="Config Path To Expiry Date", 443 ) 444 refresh_token_error_status_codes: Optional[List[int]] = Field( 445 [], 446 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 447 examples=[[400, 500]], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 449 ) 450 refresh_token_error_key: Optional[str] = Field( 451 "", 452 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 453 examples=["error"], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 455 ) 456 refresh_token_error_values: Optional[List[str]] = Field( 457 [], 458 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 459 examples=[["invalid_grant", "invalid_permissions"]], 460 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 461 )
class
Rate(pydantic.v1.main.BaseModel):
464class Rate(BaseModel): 465 class Config: 466 extra = Extra.allow 467 468 limit: Union[int, str] = Field( 469 ..., 470 description="The maximum number of calls allowed within the interval.", 471 title="Limit", 472 ) 473 interval: str = Field( 474 ..., 475 description="The time interval for the rate limit.", 476 examples=["PT1H", "P1D"], 477 title="Interval", 478 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
481class HttpRequestRegexMatcher(BaseModel): 482 class Config: 483 extra = Extra.allow 484 485 method: Optional[str] = Field( 486 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 487 ) 488 url_base: Optional[str] = Field( 489 None, 490 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 491 title="URL Base", 492 ) 493 url_path_pattern: Optional[str] = Field( 494 None, 495 description="A regular expression pattern to match the URL path.", 496 title="URL Path Pattern", 497 ) 498 params: Optional[Dict[str, Any]] = Field( 499 None, description="The query parameters to match.", title="Parameters" 500 ) 501 headers: Optional[Dict[str, Any]] = Field( 502 None, description="The headers to match.", title="Headers" 503 ) 504 weight: Optional[Union[int, str]] = Field( 505 None, 506 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 507 title="Weight", 508 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
511class ResponseToFileExtractor(BaseModel): 512 type: Literal["ResponseToFileExtractor"] 513 preserve_na_values: Optional[bool] = Field( 514 False, 515 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 516 title="Preserve NA Values", 517 ) 518 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
526class ExponentialBackoffStrategy(BaseModel): 527 type: Literal["ExponentialBackoffStrategy"] 528 factor: Optional[Union[float, str]] = Field( 529 5, 530 description="Multiplicative constant applied on each retry.", 531 examples=[5, 5.5, "10"], 532 title="Factor", 533 ) 534 jitter_range_in_seconds: Optional[float] = Field( 535 None, 536 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 537 examples=[2], 538 ge=0, 539 title="Jitter Range", 540 ) 541 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
544class GroupByKeyMergeStrategy(BaseModel): 545 type: Literal["GroupByKeyMergeStrategy"] 546 key: Union[str, List[str]] = Field( 547 ..., 548 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 549 examples=["id", ["parent_id", "end_date"]], 550 title="Key", 551 ) 552 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
QuotaStatusSource(pydantic.v1.main.BaseModel):
564class QuotaStatusSource(BaseModel): 565 type: Literal["QuotaStatusSource"] 566 url: str = Field( 567 ..., 568 description="The full URL of the quota status endpoint.", 569 examples=[ 570 "https://api.github.com/rate_limit", 571 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 572 ], 573 title="URL", 574 ) 575 http_method: Optional[HttpMethod] = Field( 576 HttpMethod.GET, 577 description="The HTTP method used to fetch the quota status.", 578 title="HTTP Method", 579 ) 580 request_headers: Optional[Dict[str, str]] = Field( 581 None, 582 description="Additional headers to send with the quota status request.", 583 title="Request Headers", 584 ) 585 unavailable_status_codes: Optional[List[int]] = Field( 586 None, 587 description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", 588 examples=[[404]], 589 title="Unavailable Status Codes", 590 unique_items=True, 591 ) 592 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
http_method: Optional[HttpMethod]
class
TokenQuota(pydantic.v1.main.BaseModel):
595class TokenQuota(BaseModel): 596 type: Literal["TokenQuota"] 597 name: str = Field( 598 ..., 599 description="Name of the quota pool.", 600 examples=["rest", "graphql"], 601 title="Name", 602 ) 603 remaining_path: List[str] = Field( 604 ..., 605 description="Path to the remaining call count for this pool in the quota status response.", 606 examples=[["resources", "core", "remaining"]], 607 title="Remaining Path", 608 ) 609 reset_path: List[str] = Field( 610 ..., 611 description="Path to the quota reset timestamp for this pool in the quota status response.", 612 examples=[["resources", "core", "reset"]], 613 title="Reset Path", 614 ) 615 limit_path: Optional[List[str]] = Field( 616 None, 617 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 618 examples=[["resources", "core", "limit"]], 619 title="Limit Path", 620 ) 621 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 622 None, 623 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 624 title="Matchers", 625 ) 626 remaining_header: Optional[str] = Field( 627 None, 628 description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", 629 examples=["X-RateLimit-Remaining"], 630 title="Remaining Header", 631 ) 632 reset_header: Optional[str] = Field( 633 None, 634 description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", 635 examples=["X-RateLimit-Reset"], 636 title="Reset Header", 637 ) 638 limit_header: Optional[str] = Field( 639 None, 640 description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", 641 examples=["X-RateLimit-Limit"], 642 title="Limit Header", 643 ) 644 exhaustion_status_codes: Optional[List[int]] = Field( 645 None, 646 description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", 647 examples=[[429]], 648 title="Exhaustion Status Codes", 649 ) 650 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
matchers: Optional[List[HttpRequestRegexMatcher]]
class
RateLimitedMultipleTokenAuthenticator(pydantic.v1.main.BaseModel):
653class RateLimitedMultipleTokenAuthenticator(BaseModel): 654 type: Literal["RateLimitedMultipleTokenAuthenticator"] 655 tokens: Union[str, List[str]] = Field( 656 ..., 657 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 658 examples=[ 659 "{{ config['credentials']['personal_access_token'] }}", 660 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 661 ], 662 title="Tokens", 663 ) 664 token_delimiter: Optional[str] = Field( 665 ",", 666 description="Delimiter used to split a single token string into multiple tokens.", 667 title="Token Delimiter", 668 ) 669 auth_method: Optional[str] = Field( 670 "Bearer", 671 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 672 examples=["Bearer", "token"], 673 title="Auth Method", 674 ) 675 header: Optional[str] = Field( 676 "Authorization", 677 description="The name of the HTTP header in which to inject the token.", 678 title="Header Name", 679 ) 680 quota_status_source: QuotaStatusSource = Field( 681 ..., 682 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 683 title="Quota Status Source", 684 ) 685 quotas: List[TokenQuota] = Field( 686 ..., 687 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 688 min_items=1, 689 title="Quota Pools", 690 ) 691 max_wait_time: Optional[str] = Field( 692 "PT2H", 693 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 694 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 695 title="Maximum Wait Time", 696 ) 697 budget_reserve_fraction: Optional[float] = Field( 698 0.1, 699 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 700 title="Budget Reserve Fraction", 701 ) 702 budget_min_reserve: Optional[int] = Field( 703 50, 704 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 705 title="Budget Minimum Reserve", 706 ) 707 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
quota_status_source: QuotaStatusSource
quotas: List[TokenQuota]
class
Action(enum.Enum):
710class Action(Enum): 711 SUCCESS = "SUCCESS" 712 FAIL = "FAIL" 713 RETRY = "RETRY" 714 IGNORE = "IGNORE" 715 RESET_PAGINATION = "RESET_PAGINATION" 716 RATE_LIMITED = "RATE_LIMITED" 717 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
720class FailureType(Enum): 721 system_error = "system_error" 722 config_error = "config_error" 723 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
726class HttpResponseFilter(BaseModel): 727 type: Literal["HttpResponseFilter"] 728 action: Optional[Action] = Field( 729 None, 730 description="Action to execute if a response matches the filter.", 731 examples=[ 732 "SUCCESS", 733 "FAIL", 734 "RETRY", 735 "IGNORE", 736 "RESET_PAGINATION", 737 "RATE_LIMITED", 738 "REFRESH_TOKEN_THEN_RETRY", 739 ], 740 title="Action", 741 ) 742 failure_type: Optional[FailureType] = Field( 743 None, 744 description="Failure type of traced exception if a response matches the filter.", 745 examples=["system_error", "config_error", "transient_error"], 746 title="Failure Type", 747 ) 748 error_message: Optional[str] = Field( 749 None, 750 description="Error Message to display if the response matches the filter.", 751 title="Error Message", 752 ) 753 error_message_contains: Optional[str] = Field( 754 None, 755 description="Match the response if its error message contains the substring.", 756 example=["This API operation is not enabled for this site"], 757 title="Error Message Substring", 758 ) 759 http_codes: Optional[List[int]] = Field( 760 None, 761 description="Match the response if its HTTP code is included in this list.", 762 examples=[[420, 429], [500]], 763 title="HTTP Codes", 764 unique_items=True, 765 ) 766 predicate: Optional[str] = Field( 767 None, 768 description="Match the response if the predicate evaluates to true.", 769 examples=[ 770 "{{ 'Too much requests' in response }}", 771 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 772 ], 773 title="Predicate", 774 ) 775 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
778class ComplexFieldType(BaseModel): 779 field_type: str 780 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
783class TypesMap(BaseModel): 784 target_type: Union[str, List[str], ComplexFieldType] 785 current_type: Union[str, List[str]] 786 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
789class SchemaTypeIdentifier(BaseModel): 790 type: Optional[Literal["SchemaTypeIdentifier"]] = None 791 schema_pointer: Optional[List[str]] = Field( 792 [], 793 description="List of nested fields defining the schema field path to extract. Defaults to [].", 794 title="Schema Path", 795 ) 796 key_pointer: List[str] = Field( 797 ..., 798 description="List of potentially nested fields describing the full path of the field key to extract.", 799 title="Key Path", 800 ) 801 type_pointer: Optional[List[str]] = Field( 802 None, 803 description="List of potentially nested fields describing the full path of the field type to extract.", 804 title="Type Path", 805 ) 806 types_mapping: Optional[List[TypesMap]] = None 807 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
810class InlineSchemaLoader(BaseModel): 811 type: Literal["InlineSchemaLoader"] 812 schema_: Optional[Dict[str, Any]] = Field( 813 None, 814 alias="schema", 815 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 816 title="Schema", 817 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
820class JsonFileSchemaLoader(BaseModel): 821 type: Literal["JsonFileSchemaLoader"] 822 file_path: Optional[str] = Field( 823 None, 824 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 825 example=["./schemas/users.json"], 826 title="File Path", 827 ) 828 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
835class JsonItemsDecoder(BaseModel): 836 type: Literal["JsonItemsDecoder"] 837 items_path: str = Field( 838 ..., 839 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 840 title="Items Path", 841 ) 842 encoding: Optional[str] = Field( 843 "utf-8", 844 description="The character encoding of the JSON data. Defaults to UTF-8.", 845 title="Encoding", 846 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
863class FlattenFields(BaseModel): 864 type: Literal["FlattenFields"] 865 flatten_lists: Optional[bool] = Field( 866 True, 867 description="Whether to flatten lists or leave it as is. Default is True.", 868 title="Flatten Lists", 869 ) 870 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
873class KeyTransformation(BaseModel): 874 type: Literal["KeyTransformation"] 875 prefix: Optional[str] = Field( 876 None, 877 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 878 examples=["flattened_"], 879 title="Key Prefix", 880 ) 881 suffix: Optional[str] = Field( 882 None, 883 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 884 examples=["_flattened"], 885 title="Key Suffix", 886 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
889class DpathFlattenFields(BaseModel): 890 type: Literal["DpathFlattenFields"] 891 field_path: List[str] = Field( 892 ..., 893 description="A path to field that needs to be flattened.", 894 examples=[["data"], ["data", "*", "field"]], 895 title="Field Path", 896 ) 897 delete_origin_value: Optional[bool] = Field( 898 None, 899 description="Whether to delete the origin value or keep it. Default is False.", 900 title="Delete Origin Value", 901 ) 902 replace_record: Optional[bool] = Field( 903 None, 904 description="Whether to replace the origin record or not. Default is False.", 905 title="Replace Origin Record", 906 ) 907 key_transformation: Optional[KeyTransformation] = Field( 908 None, 909 description="Transformation for object keys. If not provided, original key will be used.", 910 title="Key transformation", 911 ) 912 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
915class KeysReplace(BaseModel): 916 type: Literal["KeysReplace"] 917 old: str = Field( 918 ..., 919 description="Old value to replace.", 920 examples=[ 921 " ", 922 "{{ record.id }}", 923 "{{ config['id'] }}", 924 "{{ stream_slice['id'] }}", 925 ], 926 title="Old value", 927 ) 928 new: str = Field( 929 ..., 930 description="New value to set.", 931 examples=[ 932 "_", 933 "{{ record.id }}", 934 "{{ config['id'] }}", 935 "{{ stream_slice['id'] }}", 936 ], 937 title="New value", 938 ) 939 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
950class CustomDecoder(BaseModel): 951 class Config: 952 extra = Extra.allow 953 954 type: Literal["CustomDecoder"] 955 class_name: str = Field( 956 ..., 957 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 958 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 959 title="Class Name", 960 ) 961 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
964class MinMaxDatetime(BaseModel): 965 type: Literal["MinMaxDatetime"] 966 datetime: str = Field( 967 ..., 968 description="Datetime value.", 969 examples=[ 970 "2021-01-01", 971 "2021-01-01T00:00:00Z", 972 "{{ config['start_time'] }}", 973 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 974 ], 975 title="Datetime", 976 ) 977 datetime_format: Optional[str] = Field( 978 "", 979 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 980 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 981 title="Datetime Format", 982 ) 983 max_datetime: Optional[str] = Field( 984 None, 985 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 986 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 987 title="Max Datetime", 988 ) 989 min_datetime: Optional[str] = Field( 990 None, 991 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 992 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 993 title="Min Datetime", 994 ) 995 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
1025class OauthConnectorInputSpecification(BaseModel): 1026 class Config: 1027 extra = Extra.allow 1028 1029 consent_url: str = Field( 1030 ..., 1031 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 1032 examples=[ 1033 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 1034 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 1035 ], 1036 title="Consent URL", 1037 ) 1038 scope: Optional[str] = Field( 1039 None, 1040 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1041 examples=["user:read user:read_orders workspaces:read"], 1042 title="Scopes", 1043 ) 1044 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1045 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1046 # The CDK schema defines the manifest contract; the platform reads these fields 1047 # during the OAuth consent flow to build the authorization URL. 1048 scopes: Optional[List[OAuthScope]] = Field( 1049 None, 1050 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1051 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1052 title="Scopes", 1053 ) 1054 optional_scopes: Optional[List[OAuthScope]] = Field( 1055 None, 1056 description="Optional OAuth scope objects that may or may not be granted.", 1057 examples=[[{"scope": "admin:read"}]], 1058 title="Optional Scopes", 1059 ) 1060 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1061 ScopesJoinStrategy.space, 1062 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1063 title="Scopes Join Strategy", 1064 ) 1065 access_token_url: str = Field( 1066 ..., 1067 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1068 examples=[ 1069 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1070 ], 1071 title="Access Token URL", 1072 ) 1073 access_token_headers: Optional[Dict[str, Any]] = Field( 1074 None, 1075 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1076 examples=[ 1077 { 1078 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1079 } 1080 ], 1081 title="Access Token Headers", 1082 ) 1083 access_token_params: Optional[Dict[str, Any]] = Field( 1084 None, 1085 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1086 examples=[ 1087 { 1088 "{{ auth_code_key }}": "{{ auth_code_value }}", 1089 "{{ client_id_key }}": "{{ client_id_value }}", 1090 "{{ client_secret_key }}": "{{ client_secret_value }}", 1091 } 1092 ], 1093 title="Access Token Query Params (Json Encoded)", 1094 ) 1095 extract_output: Optional[List[str]] = Field( 1096 None, 1097 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1098 examples=[["access_token", "refresh_token", "other_field"]], 1099 title="Extract Output", 1100 ) 1101 state: Optional[State] = Field( 1102 None, 1103 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1104 examples=[{"min": 7, "max": 128}], 1105 title="Configurable State Query Param", 1106 ) 1107 client_id_key: Optional[str] = Field( 1108 None, 1109 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1110 examples=["my_custom_client_id_key_name"], 1111 title="Client ID Key Override", 1112 ) 1113 client_secret_key: Optional[str] = Field( 1114 None, 1115 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1116 examples=["my_custom_client_secret_key_name"], 1117 title="Client Secret Key Override", 1118 ) 1119 scope_key: Optional[str] = Field( 1120 None, 1121 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1122 examples=["my_custom_scope_key_key_name"], 1123 title="Scopes Key Override", 1124 ) 1125 state_key: Optional[str] = Field( 1126 None, 1127 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1128 examples=["my_custom_state_key_key_name"], 1129 title="State Key Override", 1130 ) 1131 auth_code_key: Optional[str] = Field( 1132 None, 1133 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1134 examples=["my_custom_auth_code_key_name"], 1135 title="Auth Code Key Override", 1136 ) 1137 redirect_uri_key: Optional[str] = Field( 1138 None, 1139 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1140 examples=["my_custom_redirect_uri_key_name"], 1141 title="Redirect URI Key Override", 1142 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1145class OAuthConfigSpecification(BaseModel): 1146 class Config: 1147 extra = Extra.allow 1148 1149 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1150 None, 1151 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1152 examples=[ 1153 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1154 { 1155 "app_id": { 1156 "type": "string", 1157 "path_in_connector_config": ["info", "app_id"], 1158 } 1159 }, 1160 ], 1161 title="OAuth user input", 1162 ) 1163 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1164 None, 1165 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1166 title="DeclarativeOAuth Connector Specification", 1167 ) 1168 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1169 None, 1170 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1171 examples=[ 1172 { 1173 "refresh_token": { 1174 "type": "string,", 1175 "path_in_connector_config": ["credentials", "refresh_token"], 1176 } 1177 } 1178 ], 1179 title="OAuth output specification", 1180 ) 1181 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1182 None, 1183 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1184 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1185 title="OAuth input specification", 1186 ) 1187 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1188 None, 1189 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1190 examples=[ 1191 { 1192 "client_id": { 1193 "type": "string,", 1194 "path_in_connector_config": ["credentials", "client_id"], 1195 }, 1196 "client_secret": { 1197 "type": "string,", 1198 "path_in_connector_config": ["credentials", "client_secret"], 1199 }, 1200 } 1201 ], 1202 title="OAuth server output specification", 1203 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1206class OffsetIncrement(BaseModel): 1207 type: Literal["OffsetIncrement"] 1208 page_size: Optional[Union[int, str]] = Field( 1209 None, 1210 description="The number of records to include in each pages.", 1211 examples=[100, "{{ config['page_size'] }}"], 1212 title="Limit", 1213 ) 1214 inject_on_first_request: Optional[bool] = Field( 1215 False, 1216 description="Using the `offset` with value `0` during the first request", 1217 title="Inject Offset on First Request", 1218 ) 1219 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1222class PageIncrement(BaseModel): 1223 type: Literal["PageIncrement"] 1224 page_size: Optional[Union[int, str]] = Field( 1225 None, 1226 description="The number of records to include in each pages.", 1227 examples=[100, "100", "{{ config['page_size'] }}"], 1228 title="Page Size", 1229 ) 1230 start_from_page: Optional[int] = Field( 1231 0, 1232 description="Index of the first page to request.", 1233 examples=[0, 1], 1234 title="Start From Page", 1235 ) 1236 inject_on_first_request: Optional[bool] = Field( 1237 False, 1238 description="Using the `page number` with value defined by `start_from_page` during the first request", 1239 title="Inject Page Number on First Request", 1240 ) 1241 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1244class PrimaryKey(BaseModel): 1245 __root__: Union[str, List[str], List[List[str]]] = Field( 1246 ..., 1247 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1248 examples=["id", ["code", "type"]], 1249 title="Primary Key", 1250 )
class
PropertyLimitType(enum.Enum):
1253class PropertyLimitType(Enum): 1254 characters = "characters" 1255 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1258class PropertyChunking(BaseModel): 1259 type: Literal["PropertyChunking"] 1260 property_limit_type: PropertyLimitType = Field( 1261 ..., 1262 description="The type used to determine the maximum number of properties per chunk", 1263 title="Property Limit Type", 1264 ) 1265 property_limit: Optional[int] = Field( 1266 None, 1267 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1268 title="Property Limit", 1269 ) 1270 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1271 None, 1272 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1273 title="Record Merge Strategy", 1274 ) 1275 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1278class RecordFilter(BaseModel): 1279 type: Literal["RecordFilter"] 1280 condition: Optional[str] = Field( 1281 "", 1282 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1283 examples=[ 1284 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1285 "{{ record.status in ['active', 'expired'] }}", 1286 ], 1287 title="Condition", 1288 ) 1289 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1297class RemoveFields(BaseModel): 1298 type: Literal["RemoveFields"] 1299 condition: Optional[str] = Field( 1300 "", 1301 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1302 examples=[ 1303 "{{ property|string == '' }}", 1304 "{{ property is integer }}", 1305 "{{ property|length > 5 }}", 1306 "{{ property == 'some_string_to_match' }}", 1307 ], 1308 ) 1309 field_pointers: List[List[str]] = Field( 1310 ..., 1311 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1312 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1313 title="Field Paths", 1314 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1321class InjectInto(Enum): 1322 request_parameter = "request_parameter" 1323 header = "header" 1324 body_data = "body_data" 1325 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1328class RequestOption(BaseModel): 1329 type: Literal["RequestOption"] 1330 inject_into: InjectInto = Field( 1331 ..., 1332 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1333 examples=["request_parameter", "header", "body_data", "body_json"], 1334 title="Inject Into", 1335 ) 1336 field_name: Optional[str] = Field( 1337 None, 1338 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1339 examples=["segment_id"], 1340 title="Field Name", 1341 ) 1342 field_path: Optional[List[str]] = Field( 1343 None, 1344 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1345 examples=[["data", "viewer", "id"]], 1346 title="Field Path", 1347 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1357class LegacySessionTokenAuthenticator(BaseModel): 1358 type: Literal["LegacySessionTokenAuthenticator"] 1359 header: str = Field( 1360 ..., 1361 description="The name of the session token header that will be injected in the request", 1362 examples=["X-Session"], 1363 title="Session Request Header", 1364 ) 1365 login_url: str = Field( 1366 ..., 1367 description="Path of the login URL (do not include the base URL)", 1368 examples=["session"], 1369 title="Login Path", 1370 ) 1371 session_token: Optional[str] = Field( 1372 None, 1373 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1374 example=["{{ config['session_token'] }}"], 1375 title="Session Token", 1376 ) 1377 session_token_response_key: str = Field( 1378 ..., 1379 description="Name of the key of the session token to be extracted from the response", 1380 examples=["id"], 1381 title="Response Token Response Key", 1382 ) 1383 username: Optional[str] = Field( 1384 None, 1385 description="Username used to authenticate and obtain a session token", 1386 examples=[" {{ config['username'] }}"], 1387 title="Username", 1388 ) 1389 password: Optional[str] = Field( 1390 "", 1391 description="Password used to authenticate and obtain a session token", 1392 examples=["{{ config['password'] }}", ""], 1393 title="Password", 1394 ) 1395 validate_session_url: str = Field( 1396 ..., 1397 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1398 examples=["user/current"], 1399 title="Validate Session Path", 1400 ) 1401 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1430class ValueType(Enum): 1431 string = "string" 1432 number = "number" 1433 integer = "integer" 1434 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1437class WaitTimeFromHeader(BaseModel): 1438 type: Literal["WaitTimeFromHeader"] 1439 header: str = Field( 1440 ..., 1441 description="The name of the response header defining how long to wait before retrying.", 1442 examples=["Retry-After"], 1443 title="Response Header Name", 1444 ) 1445 regex: Optional[str] = Field( 1446 None, 1447 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1448 examples=["([-+]?\\d+)"], 1449 title="Extraction Regex", 1450 ) 1451 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1452 None, 1453 description="Stop the stream instead of waiting, when the value extracted from the header is greater than or equal to this value. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1454 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1455 title="Max Waiting Time in Seconds", 1456 ) 1457 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1460class WaitUntilTimeFromHeader(BaseModel): 1461 type: Literal["WaitUntilTimeFromHeader"] 1462 header: str = Field( 1463 ..., 1464 description="The name of the response header defining how long to wait before retrying.", 1465 examples=["wait_time"], 1466 title="Response Header", 1467 ) 1468 min_wait: Optional[Union[float, str]] = Field( 1469 None, 1470 description="Minimum time to wait before retrying.", 1471 examples=[10, "60"], 1472 title="Minimum Wait Time", 1473 ) 1474 regex: Optional[str] = Field( 1475 None, 1476 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1477 examples=["([-+]?\\d+)"], 1478 title="Extraction Regex", 1479 ) 1480 max_waiting_time_in_seconds: Optional[Union[float, str]] = Field( 1481 None, 1482 description="Stop the stream instead of waiting, when the wait this strategy computes is greater than or equal to this value. The comparison is against the computed wait rather than the raw header, since the header holds an absolute timestamp, and it is applied after `min_wait`, so a cap below the floor still wins -- including for the fallback where the header is absent and `min_wait` supplies the wait on its own. Can be a hardcoded number, or a string interpolated from the connector config so that the bound can be changed without a connector release. A value of 0 means never wait. Not evaluated when a rate-limited retry can rotate to another credential with quota; any other retryable error still consults this strategy.", 1483 examples=[3600, "{{ config['max_waiting_time'] * 60 }}"], 1484 title="Max Waiting Time in Seconds", 1485 ) 1486 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1489class ComponentMappingDefinition(BaseModel): 1490 type: Literal["ComponentMappingDefinition"] 1491 field_path: List[str] = Field( 1492 ..., 1493 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1494 examples=[ 1495 ["name"], 1496 ["retriever", "requester", "url"], 1497 ["retriever", "requester", "{{ components_values.field }}"], 1498 ["*", "**", "name"], 1499 ], 1500 title="Field Path", 1501 ) 1502 value: str = Field( 1503 ..., 1504 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1505 examples=[ 1506 "{{ components_values['updates'] }}", 1507 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1508 "{{ config['segment_id'] }}", 1509 "{{ stream_slice['parent_id'] }}", 1510 "{{ stream_slice['extra_fields']['name'] }}", 1511 ], 1512 title="Value", 1513 ) 1514 value_type: Optional[ValueType] = Field( 1515 None, 1516 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1517 title="Value Type", 1518 ) 1519 create_or_update: Optional[bool] = Field( 1520 False, 1521 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1522 title="Create or Update", 1523 ) 1524 condition: Optional[str] = Field( 1525 None, 1526 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1527 examples=[ 1528 "{{ components_values.get('cursor_field', None) }}", 1529 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1530 ], 1531 title="Condition", 1532 ) 1533 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1536class StreamConfig(BaseModel): 1537 type: Literal["StreamConfig"] 1538 configs_pointer: List[str] = Field( 1539 ..., 1540 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1541 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1542 title="Configs Pointer", 1543 ) 1544 default_values: Optional[List[Dict[str, Any]]] = Field( 1545 None, 1546 description="A list of default values, each matching the structure expected from the parsed component value.", 1547 title="Default Values", 1548 ) 1549 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1552class ConfigComponentsResolver(BaseModel): 1553 type: Literal["ConfigComponentsResolver"] 1554 stream_config: Union[List[StreamConfig], StreamConfig] 1555 components_mapping: List[ComponentMappingDefinition] 1556 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1559class StreamParametersDefinition(BaseModel): 1560 type: Literal["StreamParametersDefinition"] 1561 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1562 ..., 1563 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1564 examples=[ 1565 [ 1566 { 1567 "name": "test stream", 1568 "$parameters": {"entity": "test entity"}, 1569 "primary_key": "test key", 1570 } 1571 ] 1572 ], 1573 title="Stream Parameters", 1574 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1577class ParametrizedComponentsResolver(BaseModel): 1578 type: Literal["ParametrizedComponentsResolver"] 1579 stream_parameters: StreamParametersDefinition 1580 components_mapping: List[ComponentMappingDefinition] 1581 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1606class ValidateAdheresToSchema(BaseModel): 1607 type: Literal["ValidateAdheresToSchema"] 1608 base_schema: Union[str, Dict[str, Any]] = Field( 1609 ..., 1610 description="The base JSON schema against which the user-provided schema will be validated.", 1611 examples=[ 1612 "{{ config['report_validation_schema'] }}", 1613 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1614 { 1615 "$schema": "http://json-schema.org/draft-07/schema#", 1616 "title": "Person", 1617 "type": "object", 1618 "properties": { 1619 "name": {"type": "string", "description": "The person's name"}, 1620 "age": { 1621 "type": "integer", 1622 "minimum": 0, 1623 "description": "The person's age", 1624 }, 1625 }, 1626 "required": ["name", "age"], 1627 }, 1628 ], 1629 title="Base JSON Schema", 1630 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1633class CustomValidationStrategy(BaseModel): 1634 class Config: 1635 extra = Extra.allow 1636 1637 type: Literal["CustomValidationStrategy"] 1638 class_name: str = Field( 1639 ..., 1640 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1641 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1642 title="Class Name", 1643 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1646class ConfigRemapField(BaseModel): 1647 type: Literal["ConfigRemapField"] 1648 map: Union[Dict[str, Any], str] = Field( 1649 ..., 1650 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1651 examples=[ 1652 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1653 "{{ config['status_mapping'] }}", 1654 ], 1655 title="Value Mapping", 1656 ) 1657 field_path: List[str] = Field( 1658 ..., 1659 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1660 examples=[ 1661 ["status"], 1662 ["data", "status"], 1663 ["data", "{{ config.name }}", "status"], 1664 ["data", "*", "status"], 1665 ], 1666 title="Field Path", 1667 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1670class ConfigRemoveFields(BaseModel): 1671 type: Literal["ConfigRemoveFields"] 1672 field_pointers: List[List[str]] = Field( 1673 ..., 1674 description="A list of field pointers to be removed from the config.", 1675 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1676 title="Field Pointers", 1677 ) 1678 condition: Optional[str] = Field( 1679 "", 1680 description="Fields will be removed if expression is evaluated to True.", 1681 examples=[ 1682 "{{ config['environemnt'] == 'sandbox' }}", 1683 "{{ property is integer }}", 1684 "{{ property|length > 5 }}", 1685 "{{ property == 'some_string_to_match' }}", 1686 ], 1687 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1690class CustomConfigTransformation(BaseModel): 1691 type: Literal["CustomConfigTransformation"] 1692 class_name: str = Field( 1693 ..., 1694 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1695 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1696 ) 1697 parameters: Optional[Dict[str, Any]] = Field( 1698 None, 1699 alias="$parameters", 1700 description="Additional parameters to be passed to the custom config transformation.", 1701 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1704class AddedFieldDefinition(BaseModel): 1705 type: Literal["AddedFieldDefinition"] 1706 path: List[str] = Field( 1707 ..., 1708 description="List of strings defining the path where to add the value on the record.", 1709 examples=[["segment_id"], ["metadata", "segment_id"]], 1710 title="Path", 1711 ) 1712 value: str = Field( 1713 ..., 1714 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1715 examples=[ 1716 "{{ record['updates'] }}", 1717 "{{ record['MetaData']['LastUpdatedTime'] }}", 1718 "{{ stream_partition['segment_id'] }}", 1719 ], 1720 title="Value", 1721 ) 1722 value_type: Optional[ValueType] = Field( 1723 None, 1724 description="Type of the value. If not specified, the type will be inferred from the value.", 1725 title="Value Type", 1726 ) 1727 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1730class AddFields(BaseModel): 1731 type: Literal["AddFields"] 1732 fields: List[AddedFieldDefinition] = Field( 1733 ..., 1734 description="List of transformations (path and corresponding value) that will be added to the record.", 1735 title="Fields", 1736 ) 1737 condition: Optional[str] = Field( 1738 "", 1739 description="Fields will be added if expression is evaluated to True.", 1740 examples=[ 1741 "{{ property|string == '' }}", 1742 "{{ property is integer }}", 1743 "{{ property|length > 5 }}", 1744 "{{ property == 'some_string_to_match' }}", 1745 ], 1746 ) 1747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1750class ApiKeyAuthenticator(BaseModel): 1751 type: Literal["ApiKeyAuthenticator"] 1752 api_token: Optional[str] = Field( 1753 None, 1754 description="The API key to inject in the request. Fill it in the user inputs.", 1755 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1756 title="API Key", 1757 ) 1758 header: Optional[str] = Field( 1759 None, 1760 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1761 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1762 title="Header Name", 1763 ) 1764 inject_into: Optional[RequestOption] = Field( 1765 None, 1766 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1767 examples=[ 1768 {"inject_into": "header", "field_name": "Authorization"}, 1769 {"inject_into": "request_parameter", "field_name": "authKey"}, 1770 ], 1771 title="Inject API Key Into Outgoing HTTP Request", 1772 ) 1773 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1776class AuthFlow(BaseModel): 1777 auth_flow_type: Optional[AuthFlowType] = Field( 1778 None, description="The type of auth to use", title="Auth flow type" 1779 ) 1780 predicate_key: Optional[List[str]] = Field( 1781 None, 1782 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1783 examples=[["credentials", "auth_type"]], 1784 title="Predicate key", 1785 ) 1786 predicate_value: Optional[str] = Field( 1787 None, 1788 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1789 examples=["Oauth"], 1790 title="Predicate value", 1791 ) 1792 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1795class CheckStream(BaseModel): 1796 type: Literal["CheckStream"] 1797 stream_names: Optional[List[str]] = Field( 1798 None, 1799 description="Names of the streams to try reading from when running a check operation.", 1800 examples=[["users"], ["users", "contacts"]], 1801 title="Stream Names", 1802 ) 1803 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1804 config_overrides: Optional[Dict[str, Any]] = Field( 1805 None, 1806 description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", 1807 examples=[{"max_waiting_time": 0}, {"page_size": 1}], 1808 title="Config Overrides", 1809 )
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1812class IncrementingCountCursor(BaseModel): 1813 type: Literal["IncrementingCountCursor"] 1814 cursor_field: str = Field( 1815 ..., 1816 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1817 examples=["created_at", "{{ config['record_cursor'] }}"], 1818 title="Cursor Field", 1819 ) 1820 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1821 None, 1822 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1823 title="Allow Catalog Defined Cursor Field", 1824 ) 1825 start_value: Optional[Union[str, int]] = Field( 1826 None, 1827 description="The value that determines the earliest record that should be synced.", 1828 examples=[0, "{{ config['start_value'] }}"], 1829 title="Start Value", 1830 ) 1831 start_value_option: Optional[RequestOption] = Field( 1832 None, 1833 description="Optionally configures how the start value will be sent in requests to the source API.", 1834 title="Inject Start Value Into Outgoing HTTP Request", 1835 ) 1836 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1839class DatetimeBasedCursor(BaseModel): 1840 type: Literal["DatetimeBasedCursor"] 1841 clamping: Optional[Clamping] = Field( 1842 None, 1843 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1844 title="Date Range Clamping", 1845 ) 1846 cursor_field: str = Field( 1847 ..., 1848 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1849 examples=["created_at", "{{ config['record_cursor'] }}"], 1850 title="Cursor Field", 1851 ) 1852 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1853 None, 1854 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1855 title="Allow Catalog Defined Cursor Field", 1856 ) 1857 cursor_datetime_formats: Optional[List[str]] = Field( 1858 None, 1859 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1860 examples=[ 1861 "%Y-%m-%d", 1862 "%Y-%m-%d %H:%M:%S", 1863 "%Y-%m-%dT%H:%M:%S", 1864 "%Y-%m-%dT%H:%M:%SZ", 1865 "%Y-%m-%dT%H:%M:%S%z", 1866 "%Y-%m-%dT%H:%M:%S.%fZ", 1867 "%Y-%m-%dT%H:%M:%S.%f%z", 1868 "%Y-%m-%d %H:%M:%S.%f+00:00", 1869 "%s", 1870 "%ms", 1871 ], 1872 title="Cursor Datetime Formats", 1873 ) 1874 start_datetime: Union[MinMaxDatetime, str] = Field( 1875 ..., 1876 description="The datetime that determines the earliest record that should be synced.", 1877 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1878 title="Start Datetime", 1879 ) 1880 start_time_option: Optional[RequestOption] = Field( 1881 None, 1882 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1883 title="Inject Start Time Into Outgoing HTTP Request", 1884 ) 1885 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1886 None, 1887 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1888 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1889 title="End Datetime", 1890 ) 1891 end_time_option: Optional[RequestOption] = Field( 1892 None, 1893 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1894 title="Inject End Time Into Outgoing HTTP Request", 1895 ) 1896 datetime_format: str = Field( 1897 ..., 1898 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1899 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1900 title="Outgoing Datetime Format", 1901 ) 1902 cursor_granularity: Optional[str] = Field( 1903 None, 1904 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1905 examples=["PT1S"], 1906 title="Cursor Granularity", 1907 ) 1908 is_data_feed: Optional[bool] = Field( 1909 None, 1910 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. The last page fetched still holds records that fall outside the cursor window, and those are filtered out as well, so Client-side Incremental Filtering does not need to be enabled alongside this field. Records are kept when their cursor value is within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time, so records dated in the future are filtered out too.", 1911 title="Data Feed API", 1912 ) 1913 is_client_side_incremental: Optional[bool] = Field( 1914 None, 1915 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, keeping only the ones whose cursor value falls within the window that starts at the previous sync's cursor value (or the start date) and ends at the end date, defaulting to the current time. This means that all records would be read from the API, but only the records within that window will be emitted to the destination. This is not needed when Data Feed API is enabled, as a data feed already filters on the same window.", 1916 title="Client-side Incremental Filtering", 1917 ) 1918 is_compare_strictly: Optional[bool] = Field( 1919 False, 1920 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1921 title="Strict Start-End Time Comparison", 1922 ) 1923 global_substream_cursor: Optional[bool] = Field( 1924 False, 1925 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1926 title="Global Substream Cursor", 1927 ) 1928 lookback_window: Optional[str] = Field( 1929 None, 1930 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1931 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1932 title="Lookback Window", 1933 ) 1934 partition_field_end: Optional[str] = Field( 1935 None, 1936 description="Name of the partition start time field.", 1937 examples=["ending_time"], 1938 title="Partition Field End", 1939 ) 1940 partition_field_start: Optional[str] = Field( 1941 None, 1942 description="Name of the partition end time field.", 1943 examples=["starting_time"], 1944 title="Partition Field Start", 1945 ) 1946 step: Optional[str] = Field( 1947 None, 1948 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1949 examples=["P1W", "{{ config['step_increment'] }}"], 1950 title="Step", 1951 ) 1952 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1955class JwtAuthenticator(BaseModel): 1956 type: Literal["JwtAuthenticator"] 1957 secret_key: str = Field( 1958 ..., 1959 description="Secret used to sign the JSON web token.", 1960 examples=["{{ config['secret_key'] }}"], 1961 title="Secret Key", 1962 ) 1963 base64_encode_secret_key: Optional[bool] = Field( 1964 False, 1965 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1966 title="Base64-encode Secret Key", 1967 ) 1968 algorithm: Algorithm = Field( 1969 ..., 1970 description="Algorithm used to sign the JSON web token.", 1971 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1972 title="Algorithm", 1973 ) 1974 token_duration: Optional[int] = Field( 1975 1200, 1976 description="The amount of time in seconds a JWT token can be valid after being issued.", 1977 examples=[1200, 3600], 1978 title="Token Duration", 1979 ) 1980 header_prefix: Optional[str] = Field( 1981 None, 1982 description="The prefix to be used within the Authentication header.", 1983 examples=["Bearer", "Basic"], 1984 title="Header Prefix", 1985 ) 1986 jwt_headers: Optional[JwtHeaders] = Field( 1987 None, 1988 description="JWT headers used when signing JSON web token.", 1989 title="JWT Headers", 1990 ) 1991 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1992 None, 1993 description="Additional headers to be included with the JWT headers object.", 1994 title="Additional JWT Headers", 1995 ) 1996 jwt_payload: Optional[JwtPayload] = Field( 1997 None, 1998 description="JWT Payload used when signing JSON web token.", 1999 title="JWT Payload", 2000 ) 2001 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 2002 None, 2003 description="Additional properties to be added to the JWT payload.", 2004 title="Additional JWT Payload Properties", 2005 ) 2006 passphrase: Optional[str] = Field( 2007 None, 2008 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 2009 examples=["{{ config['passphrase'] }}"], 2010 title="Passphrase", 2011 ) 2012 request_option: Optional[RequestOption] = Field( 2013 None, 2014 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 2015 title="Request Option", 2016 ) 2017 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
2020class OAuthAuthenticator(BaseModel): 2021 type: Literal["OAuthAuthenticator"] 2022 client_id_name: Optional[str] = Field( 2023 "client_id", 2024 description="The name of the property to use to refresh the `access_token`.", 2025 examples=["custom_app_id"], 2026 title="Client ID Property Name", 2027 ) 2028 client_id: Optional[str] = Field( 2029 None, 2030 description="The OAuth client ID. Fill it in the user inputs.", 2031 examples=[ 2032 "{{ config['client_id'] }}", 2033 "{{ config['credentials']['client_id }}", 2034 ], 2035 title="Client ID", 2036 ) 2037 client_secret_name: Optional[str] = Field( 2038 "client_secret", 2039 description="The name of the property to use to refresh the `access_token`.", 2040 examples=["custom_app_secret"], 2041 title="Client Secret Property Name", 2042 ) 2043 client_secret: Optional[str] = Field( 2044 None, 2045 description="The OAuth client secret. Fill it in the user inputs.", 2046 examples=[ 2047 "{{ config['client_secret'] }}", 2048 "{{ config['credentials']['client_secret }}", 2049 ], 2050 title="Client Secret", 2051 ) 2052 refresh_token_name: Optional[str] = Field( 2053 "refresh_token", 2054 description="The name of the property to use to refresh the `access_token`.", 2055 examples=["custom_app_refresh_value"], 2056 title="Refresh Token Property Name", 2057 ) 2058 refresh_token: Optional[str] = Field( 2059 None, 2060 description="Credential artifact used to get a new access token.", 2061 examples=[ 2062 "{{ config['refresh_token'] }}", 2063 "{{ config['credentials]['refresh_token'] }}", 2064 ], 2065 title="Refresh Token", 2066 ) 2067 token_refresh_endpoint: Optional[str] = Field( 2068 None, 2069 description="The full URL to call to obtain a new access token.", 2070 examples=["https://connect.squareup.com/oauth2/token"], 2071 title="Token Refresh Endpoint", 2072 ) 2073 access_token_name: Optional[str] = Field( 2074 "access_token", 2075 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2076 examples=["access_token"], 2077 title="Access Token Property Name", 2078 ) 2079 access_token_value: Optional[str] = Field( 2080 None, 2081 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2082 examples=["secret_access_token_value"], 2083 title="Access Token Value", 2084 ) 2085 expires_in_name: Optional[str] = Field( 2086 "expires_in", 2087 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2088 examples=["expires_in"], 2089 title="Token Expiry Property Name", 2090 ) 2091 grant_type_name: Optional[str] = Field( 2092 "grant_type", 2093 description="The name of the property to use to refresh the `access_token`.", 2094 examples=["custom_grant_type"], 2095 title="Grant Type Property Name", 2096 ) 2097 grant_type: Optional[str] = Field( 2098 "refresh_token", 2099 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2100 examples=["refresh_token", "client_credentials"], 2101 title="Grant Type", 2102 ) 2103 refresh_request_body: Optional[Dict[str, Any]] = Field( 2104 None, 2105 description="Body of the request sent to get a new access token.", 2106 examples=[ 2107 { 2108 "applicationId": "{{ config['application_id'] }}", 2109 "applicationSecret": "{{ config['application_secret'] }}", 2110 "token": "{{ config['token'] }}", 2111 } 2112 ], 2113 title="Refresh Request Body", 2114 ) 2115 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2116 None, 2117 description="Headers of the request sent to get a new access token.", 2118 examples=[ 2119 { 2120 "Authorization": "<AUTH_TOKEN>", 2121 "Content-Type": "application/x-www-form-urlencoded", 2122 } 2123 ], 2124 title="Refresh Request Headers", 2125 ) 2126 send_refresh_request_as_query_params: Optional[bool] = Field( 2127 False, 2128 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2129 examples=[True], 2130 title="Send Refresh Request As Query Params", 2131 ) 2132 scopes: Optional[List[str]] = Field( 2133 None, 2134 description="List of scopes that should be granted to the access token.", 2135 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2136 title="Scopes", 2137 ) 2138 token_expiry_date: Optional[str] = Field( 2139 None, 2140 description="The access token expiry date.", 2141 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2142 title="Token Expiry Date", 2143 ) 2144 token_expiry_date_format: Optional[str] = Field( 2145 None, 2146 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2147 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2148 title="Token Expiry Date Format", 2149 ) 2150 refresh_token_error_status_codes: Optional[List[int]] = Field( 2151 None, 2152 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2153 examples=[[400, 500]], 2154 title="Refresh Token Error Status Codes", 2155 ) 2156 refresh_token_error_key: Optional[str] = Field( 2157 None, 2158 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2159 examples=["error"], 2160 title="Refresh Token Error Key", 2161 ) 2162 refresh_token_error_values: Optional[List[str]] = Field( 2163 None, 2164 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2165 examples=[["invalid_grant", "invalid_permissions"]], 2166 title="Refresh Token Error Values", 2167 ) 2168 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2169 None, 2170 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2171 title="Refresh Token Updater", 2172 ) 2173 profile_assertion: Optional[JwtAuthenticator] = Field( 2174 None, 2175 description="The authenticator being used to authenticate the client authenticator.", 2176 title="Profile Assertion", 2177 ) 2178 use_profile_assertion: Optional[bool] = Field( 2179 False, 2180 description="Enable using profile assertion as a flow for OAuth authorization.", 2181 title="Use Profile Assertion", 2182 ) 2183 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2186class FixedWindowCallRatePolicy(BaseModel): 2187 class Config: 2188 extra = Extra.allow 2189 2190 type: Literal["FixedWindowCallRatePolicy"] 2191 period: str = Field( 2192 ..., description="The time interval for the rate limit window.", title="Period" 2193 ) 2194 call_limit: int = Field( 2195 ..., 2196 description="The maximum number of calls allowed within the period.", 2197 title="Call Limit", 2198 ) 2199 matchers: List[HttpRequestRegexMatcher] = Field( 2200 ..., 2201 description="List of matchers that define which requests this policy applies to.", 2202 title="Matchers", 2203 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2206class MovingWindowCallRatePolicy(BaseModel): 2207 class Config: 2208 extra = Extra.allow 2209 2210 type: Literal["MovingWindowCallRatePolicy"] 2211 rates: List[Rate] = Field( 2212 ..., 2213 description="List of rates that define the call limits for different time intervals.", 2214 title="Rates", 2215 ) 2216 matchers: List[HttpRequestRegexMatcher] = Field( 2217 ..., 2218 description="List of matchers that define which requests this policy applies to.", 2219 title="Matchers", 2220 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2223class UnlimitedCallRatePolicy(BaseModel): 2224 class Config: 2225 extra = Extra.allow 2226 2227 type: Literal["UnlimitedCallRatePolicy"] 2228 matchers: List[HttpRequestRegexMatcher] = Field( 2229 ..., 2230 description="List of matchers that define which requests this policy applies to.", 2231 title="Matchers", 2232 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2235class DefaultErrorHandler(BaseModel): 2236 type: Literal["DefaultErrorHandler"] 2237 backoff_strategies: Optional[ 2238 List[ 2239 Union[ 2240 ConstantBackoffStrategy, 2241 ExponentialBackoffStrategy, 2242 WaitTimeFromHeader, 2243 WaitUntilTimeFromHeader, 2244 CustomBackoffStrategy, 2245 ] 2246 ] 2247 ] = Field( 2248 None, 2249 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2250 title="Backoff Strategies", 2251 ) 2252 max_retries: Optional[Union[int, str]] = Field( 2253 5, 2254 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2255 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2256 title="Max Retry Count", 2257 ) 2258 response_filters: Optional[List[HttpResponseFilter]] = Field( 2259 None, 2260 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2261 title="Response Filters", 2262 ) 2263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2266class DefaultPaginator(BaseModel): 2267 type: Literal["DefaultPaginator"] 2268 pagination_strategy: Union[ 2269 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2270 ] = Field( 2271 ..., 2272 description="Strategy defining how records are paginated.", 2273 title="Pagination Strategy", 2274 ) 2275 page_size_option: Optional[RequestOption] = Field( 2276 None, title="Inject Page Size Into Outgoing HTTP Request" 2277 ) 2278 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2279 None, 2280 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2281 title="Inject Page Token Into Outgoing HTTP Request", 2282 ) 2283 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2286class RecordExpander(BaseModel): 2287 type: Literal["RecordExpander"] 2288 expand_records_from_field: List[str] = Field( 2289 ..., 2290 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2291 examples=[ 2292 ["lines", "data"], 2293 ["items"], 2294 ["nested", "array"], 2295 ["sections", "*", "items"], 2296 ], 2297 title="Expand Records From Field", 2298 ) 2299 remain_original_record: Optional[bool] = Field( 2300 False, 2301 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2302 title="Remain Original Record", 2303 ) 2304 on_no_records: Optional[OnNoRecords] = Field( 2305 OnNoRecords.skip, 2306 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2307 title="On No Records", 2308 ) 2309 truncation_indicator_path: Optional[List[str]] = Field( 2310 None, 2311 description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2312 examples=[["data", "object", "lines", "has_more"]], 2313 title="Truncation Indicator Path", 2314 ) 2315 truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( 2316 None, 2317 description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", 2318 title="Truncated List Retriever", 2319 ) 2320 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
truncated_list_retriever: Union[SimpleRetriever, CustomRetriever, NoneType]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2323class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2324 type: Literal["ApiKey"] 2325 inject_into: RequestOption = Field( 2326 ..., 2327 description="Configure how the API Key will be sent in requests to the source API.", 2328 examples=[ 2329 {"inject_into": "header", "field_name": "Authorization"}, 2330 {"inject_into": "request_parameter", "field_name": "authKey"}, 2331 ], 2332 title="Inject API Key Into Outgoing HTTP Request", 2333 ) 2334 api_token: Optional[str] = Field( 2335 "{{ session_token }}", 2336 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2337 examples=[ 2338 "{{ session_token }}", 2339 "Token {{ session_token }}", 2340 "Bearer {{ session_token }}", 2341 ], 2342 title="API Token Template", 2343 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2346class JsonSchemaPropertySelector(BaseModel): 2347 type: Literal["JsonSchemaPropertySelector"] 2348 transformations: Optional[ 2349 List[ 2350 Union[ 2351 AddFields, 2352 RemoveFields, 2353 KeysToLower, 2354 KeysToSnakeCase, 2355 FlattenFields, 2356 DpathFlattenFields, 2357 KeysReplace, 2358 CustomTransformation, 2359 ] 2360 ] 2361 ] = Field( 2362 None, 2363 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2364 title="Transformations", 2365 ) 2366 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2369class ListPartitionRouter(BaseModel): 2370 type: Literal["ListPartitionRouter"] 2371 cursor_field: str = Field( 2372 ..., 2373 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2374 examples=["section", "{{ config['section_key'] }}"], 2375 title="Current Partition Value Identifier", 2376 ) 2377 values: Union[str, List[str]] = Field( 2378 ..., 2379 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2380 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2381 title="Partition Values", 2382 ) 2383 request_option: Optional[RequestOption] = Field( 2384 None, 2385 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2386 title="Inject Partition Value Into Outgoing HTTP Request", 2387 ) 2388 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2391class PaginationReset(BaseModel): 2392 type: Literal["PaginationReset"] 2393 action: Action1 2394 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2397class GzipDecoder(BaseModel): 2398 type: Literal["GzipDecoder"] 2399 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2402class RequestBodyGraphQL(BaseModel): 2403 type: Literal["RequestBodyGraphQL"] 2404 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2407class DpathValidator(BaseModel): 2408 type: Literal["DpathValidator"] 2409 field_path: List[str] = Field( 2410 ..., 2411 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2412 examples=[ 2413 ["data"], 2414 ["data", "records"], 2415 ["data", "{{ parameters.name }}"], 2416 ["data", "*", "record"], 2417 ], 2418 title="Field Path", 2419 ) 2420 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2421 ..., 2422 description="The condition that the specified config value will be evaluated against", 2423 title="Validation Strategy", 2424 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2427class PredicateValidator(BaseModel): 2428 type: Literal["PredicateValidator"] 2429 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2430 ..., 2431 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2432 examples=[ 2433 "test-value", 2434 "{{ config['api_version'] }}", 2435 "{{ config['tenant_id'] }}", 2436 123, 2437 ], 2438 title="Value", 2439 ) 2440 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2441 ..., 2442 description="The validation strategy to apply to the value.", 2443 title="Validation Strategy", 2444 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2447class ConfigAddFields(BaseModel): 2448 type: Literal["ConfigAddFields"] 2449 fields: List[AddedFieldDefinition] = Field( 2450 ..., 2451 description="A list of transformations (path and corresponding value) that will be added to the config.", 2452 title="Fields", 2453 ) 2454 condition: Optional[str] = Field( 2455 "", 2456 description="Fields will be added if expression is evaluated to True.", 2457 examples=[ 2458 "{{ config['environemnt'] == 'sandbox' }}", 2459 "{{ property is integer }}", 2460 "{{ property|length > 5 }}", 2461 "{{ property == 'some_string_to_match' }}", 2462 ], 2463 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2466class CompositeErrorHandler(BaseModel): 2467 type: Literal["CompositeErrorHandler"] 2468 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2469 Field( 2470 ..., 2471 description="List of error handlers to iterate on to determine how to handle a failed response.", 2472 title="Error Handlers", 2473 ) 2474 ) 2475 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2478class HTTPAPIBudget(BaseModel): 2479 class Config: 2480 extra = Extra.allow 2481 2482 type: Literal["HTTPAPIBudget"] 2483 policies: List[ 2484 Union[ 2485 FixedWindowCallRatePolicy, 2486 MovingWindowCallRatePolicy, 2487 UnlimitedCallRatePolicy, 2488 ] 2489 ] = Field( 2490 ..., 2491 description="List of call rate policies that define how many calls are allowed.", 2492 title="Policies", 2493 ) 2494 ratelimit_reset_header: Optional[str] = Field( 2495 "ratelimit-reset", 2496 description="The HTTP response header name that indicates when the rate limit resets.", 2497 title="Rate Limit Reset Header", 2498 ) 2499 ratelimit_remaining_header: Optional[str] = Field( 2500 "ratelimit-remaining", 2501 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2502 title="Rate Limit Remaining Header", 2503 ) 2504 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2505 [429], 2506 description="List of HTTP status codes that indicate a rate limit has been hit.", 2507 title="Status Codes for Rate Limit Hit", 2508 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2511class DpathExtractor(BaseModel): 2512 type: Literal["DpathExtractor"] 2513 field_path: List[str] = Field( 2514 ..., 2515 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2516 examples=[ 2517 ["data"], 2518 ["data", "records"], 2519 ["data", "{{ parameters.name }}"], 2520 ["data", "*", "record"], 2521 ], 2522 title="Field Path", 2523 ) 2524 record_expander: Optional[RecordExpander] = Field( 2525 None, 2526 description="Optional component to expand records by extracting items from nested array fields.", 2527 title="Record Expander", 2528 ) 2529 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2532class ZipfileDecoder(BaseModel): 2533 class Config: 2534 extra = Extra.allow 2535 2536 type: Literal["ZipfileDecoder"] 2537 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2538 ..., 2539 description="Parser to parse the decompressed data from the zipfile(s).", 2540 title="Parser", 2541 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2544class RecordSelector(BaseModel): 2545 type: Literal["RecordSelector"] 2546 extractor: Union[DpathExtractor, CustomRecordExtractor] 2547 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2548 None, 2549 description="Responsible for filtering records to be emitted by the Source.", 2550 title="Record Filter", 2551 ) 2552 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2553 None, 2554 description="Responsible for normalization according to the schema.", 2555 title="Schema Normalization", 2556 ) 2557 transform_before_filtering: Optional[bool] = Field( 2558 None, 2559 description="If true, transformation will be applied before record filtering.", 2560 title="Transform Before Filtering", 2561 ) 2562 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2565class ConfigMigration(BaseModel): 2566 type: Literal["ConfigMigration"] 2567 description: Optional[str] = Field( 2568 None, description="The description/purpose of the config migration." 2569 ) 2570 transformations: List[ 2571 Union[ 2572 ConfigRemapField, 2573 ConfigAddFields, 2574 ConfigRemoveFields, 2575 CustomConfigTransformation, 2576 ] 2577 ] = Field( 2578 ..., 2579 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2580 title="Transformations", 2581 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2584class ConfigNormalizationRules(BaseModel): 2585 class Config: 2586 extra = Extra.forbid 2587 2588 type: Literal["ConfigNormalizationRules"] 2589 config_migrations: Optional[List[ConfigMigration]] = Field( 2590 [], 2591 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2592 title="Config Migrations", 2593 ) 2594 transformations: Optional[ 2595 List[ 2596 Union[ 2597 ConfigRemapField, 2598 ConfigAddFields, 2599 ConfigRemoveFields, 2600 CustomConfigTransformation, 2601 ] 2602 ] 2603 ] = Field( 2604 [], 2605 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2606 title="Transformations", 2607 ) 2608 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2609 [], 2610 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2611 title="Validations", 2612 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2615class Spec(BaseModel): 2616 type: Literal["Spec"] 2617 connection_specification: Dict[str, Any] = Field( 2618 ..., 2619 description="A connection specification describing how a the connector can be configured.", 2620 title="Connection Specification", 2621 ) 2622 documentation_url: Optional[str] = Field( 2623 None, 2624 description="URL of the connector's documentation page.", 2625 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2626 title="Documentation URL", 2627 ) 2628 advanced_auth: Optional[AuthFlow] = Field( 2629 None, 2630 description="Advanced specification for configuring the authentication flow.", 2631 title="Advanced Auth", 2632 ) 2633 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2634 None, title="Config Normalization Rules" 2635 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2638class DeclarativeSource1(BaseModel): 2639 class Config: 2640 extra = Extra.forbid 2641 2642 type: Literal["DeclarativeSource"] 2643 check: Union[CheckStream, CheckDynamicStream] 2644 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2645 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2646 version: str = Field( 2647 ..., 2648 description="The version of the Airbyte CDK used to build and test the source.", 2649 ) 2650 schemas: Optional[Schemas] = None 2651 definitions: Optional[Dict[str, Any]] = None 2652 spec: Optional[Spec] = None 2653 concurrency_level: Optional[ConcurrencyLevel] = None 2654 api_budget: Optional[HTTPAPIBudget] = None 2655 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2656 None, 2657 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2658 title="Stream Groups", 2659 ) 2660 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2661 None, 2662 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2663 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2664 title="Maximum Concurrent Asynchronous Jobs", 2665 ) 2666 metadata: Optional[Dict[str, Any]] = Field( 2667 None, 2668 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2669 ) 2670 description: Optional[str] = Field( 2671 None, 2672 description="A description of the connector. It will be presented on the Source documentation page.", 2673 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2676class DeclarativeSource2(BaseModel): 2677 class Config: 2678 extra = Extra.forbid 2679 2680 type: Literal["DeclarativeSource"] 2681 check: Union[CheckStream, CheckDynamicStream] 2682 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2683 None 2684 ) 2685 dynamic_streams: List[DynamicDeclarativeStream] 2686 version: str = Field( 2687 ..., 2688 description="The version of the Airbyte CDK used to build and test the source.", 2689 ) 2690 schemas: Optional[Schemas] = None 2691 definitions: Optional[Dict[str, Any]] = None 2692 spec: Optional[Spec] = None 2693 concurrency_level: Optional[ConcurrencyLevel] = None 2694 api_budget: Optional[HTTPAPIBudget] = None 2695 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2696 None, 2697 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2698 title="Stream Groups", 2699 ) 2700 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2701 None, 2702 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2703 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2704 title="Maximum Concurrent Asynchronous Jobs", 2705 ) 2706 metadata: Optional[Dict[str, Any]] = Field( 2707 None, 2708 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2709 ) 2710 description: Optional[str] = Field( 2711 None, 2712 description="A description of the connector. It will be presented on the Source documentation page.", 2713 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2716class DeclarativeSource(BaseModel): 2717 class Config: 2718 extra = Extra.forbid 2719 2720 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2721 ..., 2722 description="An API source that extracts data according to its declarative components.", 2723 title="DeclarativeSource", 2724 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2727class SelectiveAuthenticator(BaseModel): 2728 class Config: 2729 extra = Extra.allow 2730 2731 type: Literal["SelectiveAuthenticator"] 2732 authenticator_selection_path: List[str] = Field( 2733 ..., 2734 description="Path of the field in config with selected authenticator name", 2735 examples=[["auth"], ["auth", "type"]], 2736 title="Authenticator Selection Path", 2737 ) 2738 authenticators: Dict[ 2739 str, 2740 Union[ 2741 ApiKeyAuthenticator, 2742 BasicHttpAuthenticator, 2743 BearerAuthenticator, 2744 OAuthAuthenticator, 2745 JwtAuthenticator, 2746 SessionTokenAuthenticator, 2747 LegacySessionTokenAuthenticator, 2748 CustomAuthenticator, 2749 NoAuth, 2750 RateLimitedMultipleTokenAuthenticator, 2751 ], 2752 ] = Field( 2753 ..., 2754 description="Authenticators to select from.", 2755 examples=[ 2756 { 2757 "authenticators": { 2758 "token": "#/definitions/ApiKeyAuthenticator", 2759 "oauth": "#/definitions/OAuthAuthenticator", 2760 "jwt": "#/definitions/JwtAuthenticator", 2761 } 2762 } 2763 ], 2764 title="Authenticators", 2765 ) 2766 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2769class ConditionalStreams(BaseModel): 2770 type: Literal["ConditionalStreams"] 2771 condition: str = Field( 2772 ..., 2773 description="Condition that will be evaluated to determine if a set of streams should be available.", 2774 examples=["{{ config['is_sandbox'] }}"], 2775 title="Condition", 2776 ) 2777 streams: List[DeclarativeStream] = Field( 2778 ..., 2779 description="Streams that will be used during an operation based on the condition.", 2780 title="Streams", 2781 ) 2782 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2785class FileUploader(BaseModel): 2786 type: Literal["FileUploader"] 2787 requester: Union[HttpRequester, CustomRequester] = Field( 2788 ..., 2789 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2790 ) 2791 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2792 ..., 2793 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2794 ) 2795 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2796 None, 2797 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2798 ) 2799 filename_extractor: Optional[str] = Field( 2800 None, 2801 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2802 examples=[ 2803 "{{ record.id }}/{{ record.file_name }}/", 2804 "{{ record.id }}_{{ record.file_name }}/", 2805 ], 2806 ) 2807 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2810class DeclarativeStream(BaseModel): 2811 class Config: 2812 extra = Extra.allow 2813 2814 type: Literal["DeclarativeStream"] 2815 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2816 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2817 ..., 2818 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2819 title="Retriever", 2820 ) 2821 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2822 None, 2823 description="Component used to fetch data incrementally based on a time field in the data.", 2824 title="Incremental Sync", 2825 ) 2826 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2827 schema_loader: Optional[ 2828 Union[ 2829 InlineSchemaLoader, 2830 DynamicSchemaLoader, 2831 JsonFileSchemaLoader, 2832 List[ 2833 Union[ 2834 InlineSchemaLoader, 2835 DynamicSchemaLoader, 2836 JsonFileSchemaLoader, 2837 CustomSchemaLoader, 2838 ] 2839 ], 2840 CustomSchemaLoader, 2841 ] 2842 ] = Field( 2843 None, 2844 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2845 title="Schema Loader", 2846 ) 2847 transformations: Optional[ 2848 List[ 2849 Union[ 2850 AddFields, 2851 RemoveFields, 2852 KeysToLower, 2853 KeysToSnakeCase, 2854 FlattenFields, 2855 DpathFlattenFields, 2856 KeysReplace, 2857 CustomTransformation, 2858 ] 2859 ] 2860 ] = Field( 2861 None, 2862 description="A list of transformations to be applied to each output record.", 2863 title="Transformations", 2864 ) 2865 state_migrations: Optional[ 2866 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2867 ] = Field( 2868 [], 2869 description="Array of state migrations to be applied on the input state", 2870 title="State Migrations", 2871 ) 2872 file_uploader: Optional[FileUploader] = Field( 2873 None, 2874 description="(experimental) Describes how to fetch a file", 2875 title="File Uploader", 2876 ) 2877 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2880class SessionTokenAuthenticator(BaseModel): 2881 type: Literal["SessionTokenAuthenticator"] 2882 login_requester: HttpRequester = Field( 2883 ..., 2884 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2885 examples=[ 2886 { 2887 "type": "HttpRequester", 2888 "url_base": "https://my_api.com", 2889 "path": "/login", 2890 "authenticator": { 2891 "type": "BasicHttpAuthenticator", 2892 "username": "{{ config.username }}", 2893 "password": "{{ config.password }}", 2894 }, 2895 } 2896 ], 2897 title="Login Requester", 2898 ) 2899 session_token_path: List[str] = Field( 2900 ..., 2901 description="The path in the response body returned from the login requester to the session token.", 2902 examples=[["access_token"], ["result", "token"]], 2903 title="Session Token Path", 2904 ) 2905 expiration_duration: Optional[str] = Field( 2906 None, 2907 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2908 examples=["PT1H", "P1D"], 2909 title="Expiration Duration", 2910 ) 2911 request_authentication: Union[ 2912 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2913 ] = Field( 2914 ..., 2915 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2916 title="Data Request Authentication", 2917 ) 2918 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2919 None, description="Component used to decode the response.", title="Decoder" 2920 ) 2921 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2924class HttpRequester(BaseModelWithDeprecations): 2925 type: Literal["HttpRequester"] 2926 url_base: Optional[str] = Field( 2927 None, 2928 deprecated=True, 2929 deprecation_message="Use `url` field instead.", 2930 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2931 examples=[ 2932 "https://connect.squareup.com/v2", 2933 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2934 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2935 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2936 ], 2937 title="API Base URL", 2938 ) 2939 url: Optional[str] = Field( 2940 None, 2941 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2942 examples=[ 2943 "https://connect.squareup.com/v2", 2944 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2945 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2946 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2947 ], 2948 title="API Endpoint URL", 2949 ) 2950 path: Optional[str] = Field( 2951 None, 2952 deprecated=True, 2953 deprecation_message="Use `url` field instead.", 2954 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2955 examples=[ 2956 "/products", 2957 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2958 "/trades/{{ config['symbol_id'] }}/history", 2959 ], 2960 title="URL Path", 2961 ) 2962 http_method: Optional[HttpMethod] = Field( 2963 HttpMethod.GET, 2964 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2965 examples=["GET", "POST"], 2966 title="HTTP Method", 2967 ) 2968 authenticator: Optional[ 2969 Union[ 2970 ApiKeyAuthenticator, 2971 BasicHttpAuthenticator, 2972 BearerAuthenticator, 2973 OAuthAuthenticator, 2974 JwtAuthenticator, 2975 SessionTokenAuthenticator, 2976 SelectiveAuthenticator, 2977 CustomAuthenticator, 2978 NoAuth, 2979 LegacySessionTokenAuthenticator, 2980 RateLimitedMultipleTokenAuthenticator, 2981 ] 2982 ] = Field( 2983 None, 2984 description="Authentication method to use for requests sent to the API.", 2985 title="Authenticator", 2986 ) 2987 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2988 None, 2989 deprecated=True, 2990 deprecation_message="Use `query_properties` field instead.", 2991 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2992 title="Fetch Properties from Endpoint", 2993 ) 2994 query_properties: Optional[QueryProperties] = Field( 2995 None, 2996 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2997 title="Query Properties", 2998 ) 2999 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 3000 None, 3001 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 3002 examples=[ 3003 {"unit": "day"}, 3004 { 3005 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 3006 }, 3007 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 3008 {"sort_by[asc]": "updated_at"}, 3009 ], 3010 title="Query Parameters", 3011 ) 3012 request_headers: Optional[Union[Dict[str, str], str]] = Field( 3013 None, 3014 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 3015 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 3016 title="Request Headers", 3017 ) 3018 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 3019 None, 3020 deprecated=True, 3021 deprecation_message="Use `request_body` field instead.", 3022 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 3023 examples=[ 3024 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 3025 ], 3026 title="Request Body Payload (Non-JSON)", 3027 ) 3028 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 3029 None, 3030 deprecated=True, 3031 deprecation_message="Use `request_body` field instead.", 3032 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 3033 examples=[ 3034 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 3035 {"key": "{{ config['value'] }}"}, 3036 {"sort": {"field": "updated_at", "order": "ascending"}}, 3037 ], 3038 title="Request Body JSON Payload", 3039 ) 3040 request_body: Optional[ 3041 Union[ 3042 RequestBodyPlainText, 3043 RequestBodyUrlEncodedForm, 3044 RequestBodyJsonObject, 3045 RequestBodyGraphQL, 3046 ] 3047 ] = Field( 3048 None, 3049 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 3050 title="Request Body", 3051 ) 3052 error_handler: Optional[ 3053 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 3054 ] = Field( 3055 None, 3056 description="Error handler component that defines how to handle errors.", 3057 title="Error Handler", 3058 ) 3059 use_cache: Optional[bool] = Field( 3060 False, 3061 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3062 title="Use Cache", 3063 ) 3064 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, RateLimitedMultipleTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
3067class DynamicSchemaLoader(BaseModel): 3068 type: Literal["DynamicSchemaLoader"] 3069 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3070 ..., 3071 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3072 title="Retriever", 3073 ) 3074 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3075 None, 3076 description="Responsible for filtering fields to be added to json schema.", 3077 title="Schema Filter", 3078 ) 3079 schema_transformations: Optional[ 3080 List[ 3081 Union[ 3082 AddFields, 3083 RemoveFields, 3084 KeysToLower, 3085 KeysToSnakeCase, 3086 FlattenFields, 3087 DpathFlattenFields, 3088 KeysReplace, 3089 CustomTransformation, 3090 ] 3091 ] 3092 ] = Field( 3093 None, 3094 description="A list of transformations to be applied to the schema.", 3095 title="Schema Transformations", 3096 ) 3097 schema_type_identifier: SchemaTypeIdentifier 3098 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
3101class ParentStreamConfig(BaseModel): 3102 type: Literal["ParentStreamConfig"] 3103 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3104 ..., description="Reference to the parent stream.", title="Parent Stream" 3105 ) 3106 parent_key: str = Field( 3107 ..., 3108 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3109 examples=["id", "{{ config['parent_record_id'] }}"], 3110 title="Parent Key", 3111 ) 3112 partition_field: str = Field( 3113 ..., 3114 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3115 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3116 title="Current Parent Key Value Identifier", 3117 ) 3118 request_option: Optional[RequestOption] = Field( 3119 None, 3120 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3121 title="Request Option", 3122 ) 3123 incremental_dependency: Optional[bool] = Field( 3124 False, 3125 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3126 title="Incremental Dependency", 3127 ) 3128 lazy_read_pointer: Optional[List[str]] = Field( 3129 [], 3130 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3131 title="Lazy Read Pointer", 3132 ) 3133 extra_fields: Optional[List[List[str]]] = Field( 3134 None, 3135 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3136 title="Extra Fields", 3137 ) 3138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
3141class PropertiesFromEndpoint(BaseModel): 3142 type: Literal["PropertiesFromEndpoint"] 3143 property_field_path: List[str] = Field( 3144 ..., 3145 description="Describes the path to the field that should be extracted", 3146 examples=[["name"]], 3147 ) 3148 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3149 ..., 3150 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3151 ) 3152 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
3155class QueryProperties(BaseModel): 3156 type: Literal["QueryProperties"] 3157 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3158 ..., 3159 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3160 title="Property List", 3161 ) 3162 always_include_properties: Optional[List[str]] = Field( 3163 None, 3164 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3165 title="Always Include Properties", 3166 ) 3167 property_chunking: Optional[PropertyChunking] = Field( 3168 None, 3169 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3170 title="Property Chunking", 3171 ) 3172 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3173 None, 3174 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3175 title="Property Selector", 3176 ) 3177 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3180class StateDelegatingStream(BaseModel): 3181 type: Literal["StateDelegatingStream"] 3182 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3183 full_refresh_stream: DeclarativeStream = Field( 3184 ..., 3185 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3186 title="Full Refresh Stream", 3187 ) 3188 incremental_stream: DeclarativeStream = Field( 3189 ..., 3190 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3191 title="Incremental Stream", 3192 ) 3193 api_retention_period: Optional[str] = Field( 3194 None, 3195 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3196 examples=["P30D", "P90D", "P1Y"], 3197 title="API Retention Period", 3198 ) 3199 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3202class SimpleRetriever(BaseModel): 3203 type: Literal["SimpleRetriever"] 3204 requester: Union[HttpRequester, CustomRequester] = Field( 3205 ..., 3206 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3207 ) 3208 decoder: Optional[ 3209 Union[ 3210 JsonDecoder, 3211 JsonItemsDecoder, 3212 XmlDecoder, 3213 CsvDecoder, 3214 JsonlDecoder, 3215 GzipDecoder, 3216 IterableDecoder, 3217 ZipfileDecoder, 3218 CustomDecoder, 3219 ] 3220 ] = Field( 3221 None, 3222 description="Component decoding the response so records can be extracted.", 3223 title="HTTP Response Format", 3224 ) 3225 record_selector: RecordSelector = Field( 3226 ..., 3227 description="Component that describes how to extract records from a HTTP response.", 3228 ) 3229 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3230 None, 3231 description="Paginator component that describes how to navigate through the API's pages.", 3232 ) 3233 pagination_reset: Optional[PaginationReset] = Field( 3234 None, 3235 description="Describes what triggers pagination reset and how to handle it.", 3236 ) 3237 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3238 False, 3239 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3240 ) 3241 partition_router: Optional[ 3242 Union[ 3243 SubstreamPartitionRouter, 3244 ListPartitionRouter, 3245 GroupingPartitionRouter, 3246 UnionPartitionRouter, 3247 CustomPartitionRouter, 3248 List[ 3249 Union[ 3250 SubstreamPartitionRouter, 3251 ListPartitionRouter, 3252 GroupingPartitionRouter, 3253 UnionPartitionRouter, 3254 CustomPartitionRouter, 3255 ] 3256 ], 3257 ] 3258 ] = Field( 3259 None, 3260 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3261 title="Partition Router", 3262 ) 3263 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3266class AsyncRetriever(BaseModel): 3267 type: Literal["AsyncRetriever"] 3268 record_selector: RecordSelector = Field( 3269 ..., 3270 description="Component that describes how to extract records from a HTTP response.", 3271 ) 3272 status_mapping: AsyncJobStatusMap = Field( 3273 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3274 ) 3275 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3276 ..., description="Responsible for fetching the actual status of the async job." 3277 ) 3278 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3279 None, 3280 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3281 ) 3282 download_extractor: Optional[ 3283 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3284 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3285 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3286 ..., 3287 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3288 ) 3289 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3290 ..., 3291 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3292 ) 3293 polling_job_timeout: Optional[Union[int, str]] = Field( 3294 None, 3295 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3296 ) 3297 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3298 None, 3299 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3300 ge=1, 3301 ) 3302 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3303 None, 3304 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3305 ) 3306 download_requester: Union[HttpRequester, CustomRequester] = Field( 3307 ..., 3308 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3309 ) 3310 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3311 None, 3312 description="Paginator component that describes how to navigate through the API's pages during download.", 3313 ) 3314 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3315 None, 3316 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3317 ) 3318 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3319 None, 3320 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3321 ) 3322 partition_router: Optional[ 3323 Union[ 3324 ListPartitionRouter, 3325 SubstreamPartitionRouter, 3326 GroupingPartitionRouter, 3327 UnionPartitionRouter, 3328 CustomPartitionRouter, 3329 List[ 3330 Union[ 3331 ListPartitionRouter, 3332 SubstreamPartitionRouter, 3333 GroupingPartitionRouter, 3334 UnionPartitionRouter, 3335 CustomPartitionRouter, 3336 ] 3337 ], 3338 ] 3339 ] = Field( 3340 [], 3341 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3342 title="Partition Router", 3343 ) 3344 decoder: Optional[ 3345 Union[ 3346 CsvDecoder, 3347 GzipDecoder, 3348 JsonDecoder, 3349 JsonItemsDecoder, 3350 JsonlDecoder, 3351 IterableDecoder, 3352 XmlDecoder, 3353 ZipfileDecoder, 3354 CustomDecoder, 3355 ] 3356 ] = Field( 3357 None, 3358 description="Component decoding the response so records can be extracted.", 3359 title="HTTP Response Format", 3360 ) 3361 download_decoder: Optional[ 3362 Union[ 3363 CsvDecoder, 3364 GzipDecoder, 3365 JsonDecoder, 3366 JsonItemsDecoder, 3367 JsonlDecoder, 3368 IterableDecoder, 3369 XmlDecoder, 3370 ZipfileDecoder, 3371 CustomDecoder, 3372 ] 3373 ] = Field( 3374 None, 3375 description="Component decoding the download response so records can be extracted.", 3376 title="Download HTTP Response Format", 3377 ) 3378 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3385class StreamGroup(BaseModel): 3386 streams: List[str] = Field( 3387 ..., 3388 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3389 title="Streams", 3390 ) 3391 action: BlockSimultaneousSyncsAction = Field( 3392 ..., 3393 description="The action to apply to streams in this group.", 3394 title="Action", 3395 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3398class SubstreamPartitionRouter(BaseModel): 3399 type: Literal["SubstreamPartitionRouter"] 3400 parent_stream_configs: List[ParentStreamConfig] = Field( 3401 ..., 3402 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3403 title="Parent Stream Configs", 3404 ) 3405 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3408class GroupingPartitionRouter(BaseModel): 3409 type: Literal["GroupingPartitionRouter"] 3410 group_size: int = Field( 3411 ..., 3412 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3413 examples=[10, 50], 3414 title="Group Size", 3415 ) 3416 underlying_partition_router: Union[ 3417 ListPartitionRouter, 3418 SubstreamPartitionRouter, 3419 "UnionPartitionRouter", 3420 CustomPartitionRouter, 3421 ] = Field( 3422 ..., 3423 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3424 title="Underlying Partition Router", 3425 ) 3426 deduplicate: Optional[bool] = Field( 3427 True, 3428 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3429 title="Deduplicate Partitions", 3430 ) 3431 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]
class
UnionPartitionRouter(pydantic.v1.main.BaseModel):
3434class UnionPartitionRouter(BaseModel): 3435 type: Literal["UnionPartitionRouter"] 3436 partition_field: str = Field( 3437 ..., 3438 description="The single partition key that all child partition routers' slices are normalized to. Each child router must emit this key in its partitions. Interpolation is evaluated once when the connector is built, using the connector config and $parameters.", 3439 examples=["repository", "{{ config['partition_field'] }}"], 3440 title="Partition Field", 3441 ) 3442 partition_routers: List[ 3443 Union[ 3444 ListPartitionRouter, 3445 SubstreamPartitionRouter, 3446 UnionPartitionRouter, 3447 CustomPartitionRouter, 3448 ] 3449 ] = Field( 3450 ..., 3451 description="The child partition routers whose partitions are unioned. Request options are not supported on child partition routers; partition values should be consumed via interpolation (e.g. `stream_partition`).", 3452 title="Partition Routers", 3453 ) 3454 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
partition_routers: List[Union[ListPartitionRouter, SubstreamPartitionRouter, UnionPartitionRouter, CustomPartitionRouter]]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3457class HttpComponentsResolver(BaseModel): 3458 type: Literal["HttpComponentsResolver"] 3459 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3460 ..., 3461 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3462 title="Retriever", 3463 ) 3464 components_mapping: List[ComponentMappingDefinition] 3465 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3468class DynamicDeclarativeStream(BaseModel): 3469 type: Literal["DynamicDeclarativeStream"] 3470 name: Optional[str] = Field( 3471 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3472 ) 3473 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3474 ..., description="Reference to the stream template.", title="Stream Template" 3475 ) 3476 components_resolver: Union[ 3477 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3478 ] = Field( 3479 ..., 3480 description="Component resolve and populates stream templates with components values.", 3481 title="Components Resolver", 3482 ) 3483 use_parent_parameters: Optional[bool] = Field( 3484 True, 3485 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3486 title="Use Parent Parameters", 3487 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]