airbyte_cdk.sources.declarative.models.declarative_component_schema
1# generated by datamodel-codegen: 2# filename: declarative_component_schema.yaml 3 4from __future__ import annotations 5 6from enum import Enum 7from typing import Any, Dict, List, Literal, Optional, Union 8 9from pydantic.v1 import BaseModel, Extra, Field 10 11from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( 12 BaseModelWithDeprecations, 13) 14 15 16class AuthFlowType(Enum): 17 oauth2_0 = "oauth2.0" 18 oauth1_0 = "oauth1.0" 19 20 21class ScopesJoinStrategy(Enum): 22 space = "space" 23 comma = "comma" 24 plus = "plus" 25 26 27class BasicHttpAuthenticator(BaseModel): 28 type: Literal["BasicHttpAuthenticator"] 29 username: str = Field( 30 ..., 31 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 32 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 33 title="Username", 34 ) 35 password: Optional[str] = Field( 36 "", 37 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 38 examples=["{{ config['password'] }}", ""], 39 title="Password", 40 ) 41 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 42 43 44class BearerAuthenticator(BaseModel): 45 type: Literal["BearerAuthenticator"] 46 api_token: str = Field( 47 ..., 48 description="Token to inject as request header for authenticating with the API.", 49 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 50 title="Bearer Token", 51 ) 52 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 53 54 55class DynamicStreamCheckConfig(BaseModel): 56 type: Literal["DynamicStreamCheckConfig"] 57 dynamic_stream_name: str = Field( 58 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 59 ) 60 stream_count: Optional[int] = Field( 61 None, 62 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 63 ge=1, 64 title="Stream Count", 65 ) 66 67 68class CheckDynamicStream(BaseModel): 69 type: Literal["CheckDynamicStream"] 70 stream_count: int = Field( 71 ..., 72 description="Numbers of the streams to try reading from when running a check operation.", 73 title="Stream Count", 74 ) 75 use_check_availability: Optional[bool] = Field( 76 True, 77 description="Enables stream check availability. This field is automatically set by the CDK.", 78 title="Use Check Availability", 79 ) 80 81 82class ConcurrencyLevel(BaseModel): 83 type: Optional[Literal["ConcurrencyLevel"]] = None 84 default_concurrency: Union[int, str] = Field( 85 ..., 86 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 87 examples=[10, "{{ config['num_workers'] or 10 }}"], 88 title="Default Concurrency", 89 ) 90 max_concurrency: Optional[int] = Field( 91 None, 92 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 93 examples=[20, 100], 94 title="Max Concurrency", 95 ) 96 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 97 98 99class ConstantBackoffStrategy(BaseModel): 100 type: Literal["ConstantBackoffStrategy"] 101 backoff_time_in_seconds: Union[float, str] = Field( 102 ..., 103 description="Backoff time in seconds.", 104 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 105 title="Backoff Time", 106 ) 107 jitter_range_in_seconds: Optional[float] = Field( 108 None, 109 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 110 examples=[15], 111 ge=0, 112 title="Jitter Range", 113 ) 114 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 115 116 117class CursorPagination(BaseModel): 118 type: Literal["CursorPagination"] 119 cursor_value: str = Field( 120 ..., 121 description="Value of the cursor defining the next page to fetch.", 122 examples=[ 123 "{{ headers.link.next.cursor }}", 124 "{{ last_record['key'] }}", 125 "{{ response['nextPage'] }}", 126 ], 127 title="Cursor Value", 128 ) 129 page_size: Optional[Union[int, str]] = Field( 130 None, 131 description="The number of records to include in each pages.", 132 examples=[100, "{{ config['page_size'] }}"], 133 title="Page Size", 134 ) 135 stop_condition: Optional[str] = Field( 136 None, 137 description="Template string evaluating when to stop paginating.", 138 examples=[ 139 "{{ response.data.has_more is false }}", 140 "{{ 'next' not in headers['link'] }}", 141 ], 142 title="Stop Condition", 143 ) 144 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 145 146 147class CustomAuthenticator(BaseModel): 148 class Config: 149 extra = Extra.allow 150 151 type: Literal["CustomAuthenticator"] 152 class_name: str = Field( 153 ..., 154 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 155 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 156 title="Class Name", 157 ) 158 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 159 160 161class CustomBackoffStrategy(BaseModel): 162 class Config: 163 extra = Extra.allow 164 165 type: Literal["CustomBackoffStrategy"] 166 class_name: str = Field( 167 ..., 168 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 169 examples=["source_railz.components.MyCustomBackoffStrategy"], 170 title="Class Name", 171 ) 172 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 173 174 175class CustomErrorHandler(BaseModel): 176 class Config: 177 extra = Extra.allow 178 179 type: Literal["CustomErrorHandler"] 180 class_name: str = Field( 181 ..., 182 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 183 examples=["source_railz.components.MyCustomErrorHandler"], 184 title="Class Name", 185 ) 186 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 187 188 189class CustomPaginationStrategy(BaseModel): 190 class Config: 191 extra = Extra.allow 192 193 type: Literal["CustomPaginationStrategy"] 194 class_name: str = Field( 195 ..., 196 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 197 examples=["source_railz.components.MyCustomPaginationStrategy"], 198 title="Class Name", 199 ) 200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 201 202 203class CustomRecordExtractor(BaseModel): 204 class Config: 205 extra = Extra.allow 206 207 type: Literal["CustomRecordExtractor"] 208 class_name: str = Field( 209 ..., 210 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 211 examples=["source_railz.components.MyCustomRecordExtractor"], 212 title="Class Name", 213 ) 214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 215 216 217class CustomRecordFilter(BaseModel): 218 class Config: 219 extra = Extra.allow 220 221 type: Literal["CustomRecordFilter"] 222 class_name: str = Field( 223 ..., 224 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 225 examples=["source_railz.components.MyCustomCustomRecordFilter"], 226 title="Class Name", 227 ) 228 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 229 230 231class CustomRequester(BaseModel): 232 class Config: 233 extra = Extra.allow 234 235 type: Literal["CustomRequester"] 236 class_name: str = Field( 237 ..., 238 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 239 examples=["source_railz.components.MyCustomRecordExtractor"], 240 title="Class Name", 241 ) 242 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 243 244 245class CustomRetriever(BaseModel): 246 class Config: 247 extra = Extra.allow 248 249 type: Literal["CustomRetriever"] 250 class_name: str = Field( 251 ..., 252 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 253 examples=["source_railz.components.MyCustomRetriever"], 254 title="Class Name", 255 ) 256 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 257 258 259class CustomPartitionRouter(BaseModel): 260 class Config: 261 extra = Extra.allow 262 263 type: Literal["CustomPartitionRouter"] 264 class_name: str = Field( 265 ..., 266 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 267 examples=["source_railz.components.MyCustomPartitionRouter"], 268 title="Class Name", 269 ) 270 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 271 272 273class CustomSchemaLoader(BaseModel): 274 class Config: 275 extra = Extra.allow 276 277 type: Literal["CustomSchemaLoader"] 278 class_name: str = Field( 279 ..., 280 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 281 examples=["source_railz.components.MyCustomSchemaLoader"], 282 title="Class Name", 283 ) 284 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 285 286 287class CustomSchemaNormalization(BaseModel): 288 class Config: 289 extra = Extra.allow 290 291 type: Literal["CustomSchemaNormalization"] 292 class_name: str = Field( 293 ..., 294 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 295 examples=[ 296 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 297 ], 298 title="Class Name", 299 ) 300 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 301 302 303class CustomStateMigration(BaseModel): 304 class Config: 305 extra = Extra.allow 306 307 type: Literal["CustomStateMigration"] 308 class_name: str = Field( 309 ..., 310 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 311 examples=["source_railz.components.MyCustomStateMigration"], 312 title="Class Name", 313 ) 314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 315 316 317class CustomTransformation(BaseModel): 318 class Config: 319 extra = Extra.allow 320 321 type: Literal["CustomTransformation"] 322 class_name: str = Field( 323 ..., 324 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 325 examples=["source_railz.components.MyCustomTransformation"], 326 title="Class Name", 327 ) 328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 329 330 331class LegacyToPerPartitionStateMigration(BaseModel): 332 class Config: 333 extra = Extra.allow 334 335 type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None 336 337 338class Clamping(BaseModel): 339 target: str = Field( 340 ..., 341 description="The period of time that datetime windows will be clamped by", 342 examples=["DAY", "WEEK", "MONTH", "{{ config['target'] }}"], 343 title="Target", 344 ) 345 target_details: Optional[Dict[str, Any]] = None 346 347 348class Algorithm(Enum): 349 HS256 = "HS256" 350 HS384 = "HS384" 351 HS512 = "HS512" 352 ES256 = "ES256" 353 ES256K = "ES256K" 354 ES384 = "ES384" 355 ES512 = "ES512" 356 RS256 = "RS256" 357 RS384 = "RS384" 358 RS512 = "RS512" 359 PS256 = "PS256" 360 PS384 = "PS384" 361 PS512 = "PS512" 362 EdDSA = "EdDSA" 363 364 365class JwtHeaders(BaseModel): 366 class Config: 367 extra = Extra.forbid 368 369 kid: Optional[str] = Field( 370 None, 371 description="Private key ID for user account.", 372 examples=["{{ config['kid'] }}"], 373 title="Key Identifier", 374 ) 375 typ: Optional[str] = Field( 376 "JWT", 377 description="The media type of the complete JWT.", 378 examples=["JWT"], 379 title="Type", 380 ) 381 cty: Optional[str] = Field( 382 None, 383 description="Content type of JWT header.", 384 examples=["JWT"], 385 title="Content Type", 386 ) 387 388 389class JwtPayload(BaseModel): 390 class Config: 391 extra = Extra.forbid 392 393 iss: Optional[str] = Field( 394 None, 395 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 396 examples=["{{ config['iss'] }}"], 397 title="Issuer", 398 ) 399 sub: Optional[str] = Field( 400 None, 401 description="The subject of the JWT. Commonly defined by the API.", 402 title="Subject", 403 ) 404 aud: Optional[str] = Field( 405 None, 406 description="The recipient that the JWT is intended for. Commonly defined by the API.", 407 examples=["appstoreconnect-v1"], 408 title="Audience", 409 ) 410 411 412class RefreshTokenUpdater(BaseModel): 413 refresh_token_name: Optional[str] = Field( 414 "refresh_token", 415 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 416 examples=["refresh_token"], 417 title="Refresh Token Property Name", 418 ) 419 access_token_config_path: Optional[List[str]] = Field( 420 ["credentials", "access_token"], 421 description="Config path to the access token. Make sure the field actually exists in the config.", 422 examples=[["credentials", "access_token"], ["access_token"]], 423 title="Config Path To Access Token", 424 ) 425 refresh_token_config_path: Optional[List[str]] = Field( 426 ["credentials", "refresh_token"], 427 description="Config path to the access token. Make sure the field actually exists in the config.", 428 examples=[["credentials", "refresh_token"], ["refresh_token"]], 429 title="Config Path To Refresh Token", 430 ) 431 token_expiry_date_config_path: Optional[List[str]] = Field( 432 ["credentials", "token_expiry_date"], 433 description="Config path to the expiry date. Make sure actually exists in the config.", 434 examples=[["credentials", "token_expiry_date"]], 435 title="Config Path To Expiry Date", 436 ) 437 refresh_token_error_status_codes: Optional[List[int]] = Field( 438 [], 439 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 440 examples=[[400, 500]], 441 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 442 ) 443 refresh_token_error_key: Optional[str] = Field( 444 "", 445 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 446 examples=["error"], 447 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 448 ) 449 refresh_token_error_values: Optional[List[str]] = Field( 450 [], 451 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 452 examples=[["invalid_grant", "invalid_permissions"]], 453 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 454 ) 455 456 457class Rate(BaseModel): 458 class Config: 459 extra = Extra.allow 460 461 limit: Union[int, str] = Field( 462 ..., 463 description="The maximum number of calls allowed within the interval.", 464 title="Limit", 465 ) 466 interval: str = Field( 467 ..., 468 description="The time interval for the rate limit.", 469 examples=["PT1H", "P1D"], 470 title="Interval", 471 ) 472 473 474class HttpRequestRegexMatcher(BaseModel): 475 class Config: 476 extra = Extra.allow 477 478 method: Optional[str] = Field( 479 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 480 ) 481 url_base: Optional[str] = Field( 482 None, 483 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 484 title="URL Base", 485 ) 486 url_path_pattern: Optional[str] = Field( 487 None, 488 description="A regular expression pattern to match the URL path.", 489 title="URL Path Pattern", 490 ) 491 params: Optional[Dict[str, Any]] = Field( 492 None, description="The query parameters to match.", title="Parameters" 493 ) 494 headers: Optional[Dict[str, Any]] = Field( 495 None, description="The headers to match.", title="Headers" 496 ) 497 weight: Optional[Union[int, str]] = Field( 498 None, 499 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 500 title="Weight", 501 ) 502 503 504class ResponseToFileExtractor(BaseModel): 505 type: Literal["ResponseToFileExtractor"] 506 preserve_na_values: Optional[bool] = Field( 507 False, 508 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 509 title="Preserve NA Values", 510 ) 511 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 512 513 514class OnNoRecords(Enum): 515 skip = "skip" 516 emit_parent = "emit_parent" 517 518 519class ExponentialBackoffStrategy(BaseModel): 520 type: Literal["ExponentialBackoffStrategy"] 521 factor: Optional[Union[float, str]] = Field( 522 5, 523 description="Multiplicative constant applied on each retry.", 524 examples=[5, 5.5, "10"], 525 title="Factor", 526 ) 527 jitter_range_in_seconds: Optional[float] = Field( 528 None, 529 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 530 examples=[2], 531 ge=0, 532 title="Jitter Range", 533 ) 534 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 535 536 537class GroupByKeyMergeStrategy(BaseModel): 538 type: Literal["GroupByKeyMergeStrategy"] 539 key: Union[str, List[str]] = Field( 540 ..., 541 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 542 examples=["id", ["parent_id", "end_date"]], 543 title="Key", 544 ) 545 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 546 547 548class SessionTokenRequestBearerAuthenticator(BaseModel): 549 type: Literal["Bearer"] 550 551 552class HttpMethod(Enum): 553 GET = "GET" 554 POST = "POST" 555 556 557class QuotaStatusSource(BaseModel): 558 type: Literal["QuotaStatusSource"] 559 url: str = Field( 560 ..., 561 description="The full URL of the quota status endpoint.", 562 examples=[ 563 "https://api.github.com/rate_limit", 564 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 565 ], 566 title="URL", 567 ) 568 http_method: Optional[HttpMethod] = Field( 569 HttpMethod.GET, 570 description="The HTTP method used to fetch the quota status.", 571 title="HTTP Method", 572 ) 573 request_headers: Optional[Dict[str, str]] = Field( 574 None, 575 description="Additional headers to send with the quota status request.", 576 title="Request Headers", 577 ) 578 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 579 580 581class TokenQuota(BaseModel): 582 type: Literal["TokenQuota"] 583 name: str = Field( 584 ..., 585 description="Name of the quota pool.", 586 examples=["rest", "graphql"], 587 title="Name", 588 ) 589 remaining_path: List[str] = Field( 590 ..., 591 description="Path to the remaining call count for this pool in the quota status response.", 592 examples=[["resources", "core", "remaining"]], 593 title="Remaining Path", 594 ) 595 reset_path: List[str] = Field( 596 ..., 597 description="Path to the quota reset timestamp for this pool in the quota status response.", 598 examples=[["resources", "core", "reset"]], 599 title="Reset Path", 600 ) 601 limit_path: Optional[List[str]] = Field( 602 None, 603 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 604 examples=[["resources", "core", "limit"]], 605 title="Limit Path", 606 ) 607 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 608 None, 609 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 610 title="Matchers", 611 ) 612 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 613 614 615class RateLimitedMultipleTokenAuthenticator(BaseModel): 616 type: Literal["RateLimitedMultipleTokenAuthenticator"] 617 tokens: Union[str, List[str]] = Field( 618 ..., 619 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 620 examples=[ 621 "{{ config['credentials']['personal_access_token'] }}", 622 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 623 ], 624 title="Tokens", 625 ) 626 token_delimiter: Optional[str] = Field( 627 ",", 628 description="Delimiter used to split a single token string into multiple tokens.", 629 title="Token Delimiter", 630 ) 631 auth_method: Optional[str] = Field( 632 "Bearer", 633 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 634 examples=["Bearer", "token"], 635 title="Auth Method", 636 ) 637 header: Optional[str] = Field( 638 "Authorization", 639 description="The name of the HTTP header in which to inject the token.", 640 title="Header Name", 641 ) 642 quota_status_source: QuotaStatusSource = Field( 643 ..., 644 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 645 title="Quota Status Source", 646 ) 647 quotas: List[TokenQuota] = Field( 648 ..., 649 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 650 min_items=1, 651 title="Quota Pools", 652 ) 653 max_wait_time: Optional[str] = Field( 654 "PT2H", 655 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 656 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 657 title="Maximum Wait Time", 658 ) 659 budget_reserve_fraction: Optional[float] = Field( 660 0.1, 661 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 662 title="Budget Reserve Fraction", 663 ) 664 budget_min_reserve: Optional[int] = Field( 665 50, 666 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 667 title="Budget Minimum Reserve", 668 ) 669 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 670 671 672class Action(Enum): 673 SUCCESS = "SUCCESS" 674 FAIL = "FAIL" 675 RETRY = "RETRY" 676 IGNORE = "IGNORE" 677 RESET_PAGINATION = "RESET_PAGINATION" 678 RATE_LIMITED = "RATE_LIMITED" 679 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" 680 681 682class FailureType(Enum): 683 system_error = "system_error" 684 config_error = "config_error" 685 transient_error = "transient_error" 686 687 688class HttpResponseFilter(BaseModel): 689 type: Literal["HttpResponseFilter"] 690 action: Optional[Action] = Field( 691 None, 692 description="Action to execute if a response matches the filter.", 693 examples=[ 694 "SUCCESS", 695 "FAIL", 696 "RETRY", 697 "IGNORE", 698 "RESET_PAGINATION", 699 "RATE_LIMITED", 700 "REFRESH_TOKEN_THEN_RETRY", 701 ], 702 title="Action", 703 ) 704 failure_type: Optional[FailureType] = Field( 705 None, 706 description="Failure type of traced exception if a response matches the filter.", 707 examples=["system_error", "config_error", "transient_error"], 708 title="Failure Type", 709 ) 710 error_message: Optional[str] = Field( 711 None, 712 description="Error Message to display if the response matches the filter.", 713 title="Error Message", 714 ) 715 error_message_contains: Optional[str] = Field( 716 None, 717 description="Match the response if its error message contains the substring.", 718 example=["This API operation is not enabled for this site"], 719 title="Error Message Substring", 720 ) 721 http_codes: Optional[List[int]] = Field( 722 None, 723 description="Match the response if its HTTP code is included in this list.", 724 examples=[[420, 429], [500]], 725 title="HTTP Codes", 726 unique_items=True, 727 ) 728 predicate: Optional[str] = Field( 729 None, 730 description="Match the response if the predicate evaluates to true.", 731 examples=[ 732 "{{ 'Too much requests' in response }}", 733 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 734 ], 735 title="Predicate", 736 ) 737 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 738 739 740class ComplexFieldType(BaseModel): 741 field_type: str 742 items: Optional[Union[str, ComplexFieldType]] = None 743 744 745class TypesMap(BaseModel): 746 target_type: Union[str, List[str], ComplexFieldType] 747 current_type: Union[str, List[str]] 748 condition: Optional[str] = None 749 750 751class SchemaTypeIdentifier(BaseModel): 752 type: Optional[Literal["SchemaTypeIdentifier"]] = None 753 schema_pointer: Optional[List[str]] = Field( 754 [], 755 description="List of nested fields defining the schema field path to extract. Defaults to [].", 756 title="Schema Path", 757 ) 758 key_pointer: List[str] = Field( 759 ..., 760 description="List of potentially nested fields describing the full path of the field key to extract.", 761 title="Key Path", 762 ) 763 type_pointer: Optional[List[str]] = Field( 764 None, 765 description="List of potentially nested fields describing the full path of the field type to extract.", 766 title="Type Path", 767 ) 768 types_mapping: Optional[List[TypesMap]] = None 769 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 770 771 772class InlineSchemaLoader(BaseModel): 773 type: Literal["InlineSchemaLoader"] 774 schema_: Optional[Dict[str, Any]] = Field( 775 None, 776 alias="schema", 777 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 778 title="Schema", 779 ) 780 781 782class JsonFileSchemaLoader(BaseModel): 783 type: Literal["JsonFileSchemaLoader"] 784 file_path: Optional[str] = Field( 785 None, 786 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 787 example=["./schemas/users.json"], 788 title="File Path", 789 ) 790 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 791 792 793class JsonDecoder(BaseModel): 794 type: Literal["JsonDecoder"] 795 796 797class JsonItemsDecoder(BaseModel): 798 type: Literal["JsonItemsDecoder"] 799 items_path: str = Field( 800 ..., 801 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 802 title="Items Path", 803 ) 804 encoding: Optional[str] = Field( 805 "utf-8", 806 description="The character encoding of the JSON data. Defaults to UTF-8.", 807 title="Encoding", 808 ) 809 810 811class JsonlDecoder(BaseModel): 812 type: Literal["JsonlDecoder"] 813 814 815class KeysToLower(BaseModel): 816 type: Literal["KeysToLower"] 817 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 818 819 820class KeysToSnakeCase(BaseModel): 821 type: Literal["KeysToSnakeCase"] 822 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 823 824 825class FlattenFields(BaseModel): 826 type: Literal["FlattenFields"] 827 flatten_lists: Optional[bool] = Field( 828 True, 829 description="Whether to flatten lists or leave it as is. Default is True.", 830 title="Flatten Lists", 831 ) 832 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 833 834 835class KeyTransformation(BaseModel): 836 type: Literal["KeyTransformation"] 837 prefix: Optional[str] = Field( 838 None, 839 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 840 examples=["flattened_"], 841 title="Key Prefix", 842 ) 843 suffix: Optional[str] = Field( 844 None, 845 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 846 examples=["_flattened"], 847 title="Key Suffix", 848 ) 849 850 851class DpathFlattenFields(BaseModel): 852 type: Literal["DpathFlattenFields"] 853 field_path: List[str] = Field( 854 ..., 855 description="A path to field that needs to be flattened.", 856 examples=[["data"], ["data", "*", "field"]], 857 title="Field Path", 858 ) 859 delete_origin_value: Optional[bool] = Field( 860 None, 861 description="Whether to delete the origin value or keep it. Default is False.", 862 title="Delete Origin Value", 863 ) 864 replace_record: Optional[bool] = Field( 865 None, 866 description="Whether to replace the origin record or not. Default is False.", 867 title="Replace Origin Record", 868 ) 869 key_transformation: Optional[KeyTransformation] = Field( 870 None, 871 description="Transformation for object keys. If not provided, original key will be used.", 872 title="Key transformation", 873 ) 874 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 875 876 877class KeysReplace(BaseModel): 878 type: Literal["KeysReplace"] 879 old: str = Field( 880 ..., 881 description="Old value to replace.", 882 examples=[ 883 " ", 884 "{{ record.id }}", 885 "{{ config['id'] }}", 886 "{{ stream_slice['id'] }}", 887 ], 888 title="Old value", 889 ) 890 new: str = Field( 891 ..., 892 description="New value to set.", 893 examples=[ 894 "_", 895 "{{ record.id }}", 896 "{{ config['id'] }}", 897 "{{ stream_slice['id'] }}", 898 ], 899 title="New value", 900 ) 901 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 902 903 904class IterableDecoder(BaseModel): 905 type: Literal["IterableDecoder"] 906 907 908class XmlDecoder(BaseModel): 909 type: Literal["XmlDecoder"] 910 911 912class CustomDecoder(BaseModel): 913 class Config: 914 extra = Extra.allow 915 916 type: Literal["CustomDecoder"] 917 class_name: str = Field( 918 ..., 919 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 920 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 921 title="Class Name", 922 ) 923 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 924 925 926class MinMaxDatetime(BaseModel): 927 type: Literal["MinMaxDatetime"] 928 datetime: str = Field( 929 ..., 930 description="Datetime value.", 931 examples=[ 932 "2021-01-01", 933 "2021-01-01T00:00:00Z", 934 "{{ config['start_time'] }}", 935 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 936 ], 937 title="Datetime", 938 ) 939 datetime_format: Optional[str] = Field( 940 "", 941 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 942 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 943 title="Datetime Format", 944 ) 945 max_datetime: Optional[str] = Field( 946 None, 947 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 948 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 949 title="Max Datetime", 950 ) 951 min_datetime: Optional[str] = Field( 952 None, 953 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 954 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 955 title="Min Datetime", 956 ) 957 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 958 959 960class NoAuth(BaseModel): 961 type: Literal["NoAuth"] 962 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 963 964 965class NoPagination(BaseModel): 966 type: Literal["NoPagination"] 967 968 969class State(BaseModel): 970 class Config: 971 extra = Extra.allow 972 973 min: int 974 max: int 975 976 977class OAuthScope(BaseModel): 978 class Config: 979 extra = Extra.allow 980 981 scope: str = Field( 982 ..., 983 description="The OAuth scope string to request from the provider.", 984 ) 985 986 987class OauthConnectorInputSpecification(BaseModel): 988 class Config: 989 extra = Extra.allow 990 991 consent_url: str = Field( 992 ..., 993 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 994 examples=[ 995 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 996 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 997 ], 998 title="Consent URL", 999 ) 1000 scope: Optional[str] = Field( 1001 None, 1002 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1003 examples=["user:read user:read_orders workspaces:read"], 1004 title="Scopes", 1005 ) 1006 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1007 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1008 # The CDK schema defines the manifest contract; the platform reads these fields 1009 # during the OAuth consent flow to build the authorization URL. 1010 scopes: Optional[List[OAuthScope]] = Field( 1011 None, 1012 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1013 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1014 title="Scopes", 1015 ) 1016 optional_scopes: Optional[List[OAuthScope]] = Field( 1017 None, 1018 description="Optional OAuth scope objects that may or may not be granted.", 1019 examples=[[{"scope": "admin:read"}]], 1020 title="Optional Scopes", 1021 ) 1022 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1023 ScopesJoinStrategy.space, 1024 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1025 title="Scopes Join Strategy", 1026 ) 1027 access_token_url: str = Field( 1028 ..., 1029 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1030 examples=[ 1031 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1032 ], 1033 title="Access Token URL", 1034 ) 1035 access_token_headers: Optional[Dict[str, Any]] = Field( 1036 None, 1037 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1038 examples=[ 1039 { 1040 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1041 } 1042 ], 1043 title="Access Token Headers", 1044 ) 1045 access_token_params: Optional[Dict[str, Any]] = Field( 1046 None, 1047 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1048 examples=[ 1049 { 1050 "{{ auth_code_key }}": "{{ auth_code_value }}", 1051 "{{ client_id_key }}": "{{ client_id_value }}", 1052 "{{ client_secret_key }}": "{{ client_secret_value }}", 1053 } 1054 ], 1055 title="Access Token Query Params (Json Encoded)", 1056 ) 1057 extract_output: Optional[List[str]] = Field( 1058 None, 1059 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1060 examples=[["access_token", "refresh_token", "other_field"]], 1061 title="Extract Output", 1062 ) 1063 state: Optional[State] = Field( 1064 None, 1065 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1066 examples=[{"min": 7, "max": 128}], 1067 title="Configurable State Query Param", 1068 ) 1069 client_id_key: Optional[str] = Field( 1070 None, 1071 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1072 examples=["my_custom_client_id_key_name"], 1073 title="Client ID Key Override", 1074 ) 1075 client_secret_key: Optional[str] = Field( 1076 None, 1077 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1078 examples=["my_custom_client_secret_key_name"], 1079 title="Client Secret Key Override", 1080 ) 1081 scope_key: Optional[str] = Field( 1082 None, 1083 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1084 examples=["my_custom_scope_key_key_name"], 1085 title="Scopes Key Override", 1086 ) 1087 state_key: Optional[str] = Field( 1088 None, 1089 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1090 examples=["my_custom_state_key_key_name"], 1091 title="State Key Override", 1092 ) 1093 auth_code_key: Optional[str] = Field( 1094 None, 1095 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1096 examples=["my_custom_auth_code_key_name"], 1097 title="Auth Code Key Override", 1098 ) 1099 redirect_uri_key: Optional[str] = Field( 1100 None, 1101 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1102 examples=["my_custom_redirect_uri_key_name"], 1103 title="Redirect URI Key Override", 1104 ) 1105 1106 1107class OAuthConfigSpecification(BaseModel): 1108 class Config: 1109 extra = Extra.allow 1110 1111 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1112 None, 1113 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1114 examples=[ 1115 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1116 { 1117 "app_id": { 1118 "type": "string", 1119 "path_in_connector_config": ["info", "app_id"], 1120 } 1121 }, 1122 ], 1123 title="OAuth user input", 1124 ) 1125 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1126 None, 1127 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1128 title="DeclarativeOAuth Connector Specification", 1129 ) 1130 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1131 None, 1132 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1133 examples=[ 1134 { 1135 "refresh_token": { 1136 "type": "string,", 1137 "path_in_connector_config": ["credentials", "refresh_token"], 1138 } 1139 } 1140 ], 1141 title="OAuth output specification", 1142 ) 1143 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1144 None, 1145 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1146 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1147 title="OAuth input specification", 1148 ) 1149 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1150 None, 1151 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1152 examples=[ 1153 { 1154 "client_id": { 1155 "type": "string,", 1156 "path_in_connector_config": ["credentials", "client_id"], 1157 }, 1158 "client_secret": { 1159 "type": "string,", 1160 "path_in_connector_config": ["credentials", "client_secret"], 1161 }, 1162 } 1163 ], 1164 title="OAuth server output specification", 1165 ) 1166 1167 1168class OffsetIncrement(BaseModel): 1169 type: Literal["OffsetIncrement"] 1170 page_size: Optional[Union[int, str]] = Field( 1171 None, 1172 description="The number of records to include in each pages.", 1173 examples=[100, "{{ config['page_size'] }}"], 1174 title="Limit", 1175 ) 1176 inject_on_first_request: Optional[bool] = Field( 1177 False, 1178 description="Using the `offset` with value `0` during the first request", 1179 title="Inject Offset on First Request", 1180 ) 1181 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1182 1183 1184class PageIncrement(BaseModel): 1185 type: Literal["PageIncrement"] 1186 page_size: Optional[Union[int, str]] = Field( 1187 None, 1188 description="The number of records to include in each pages.", 1189 examples=[100, "100", "{{ config['page_size'] }}"], 1190 title="Page Size", 1191 ) 1192 start_from_page: Optional[int] = Field( 1193 0, 1194 description="Index of the first page to request.", 1195 examples=[0, 1], 1196 title="Start From Page", 1197 ) 1198 inject_on_first_request: Optional[bool] = Field( 1199 False, 1200 description="Using the `page number` with value defined by `start_from_page` during the first request", 1201 title="Inject Page Number on First Request", 1202 ) 1203 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1204 1205 1206class PrimaryKey(BaseModel): 1207 __root__: Union[str, List[str], List[List[str]]] = Field( 1208 ..., 1209 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1210 examples=["id", ["code", "type"]], 1211 title="Primary Key", 1212 ) 1213 1214 1215class PropertyLimitType(Enum): 1216 characters = "characters" 1217 property_count = "property_count" 1218 1219 1220class PropertyChunking(BaseModel): 1221 type: Literal["PropertyChunking"] 1222 property_limit_type: PropertyLimitType = Field( 1223 ..., 1224 description="The type used to determine the maximum number of properties per chunk", 1225 title="Property Limit Type", 1226 ) 1227 property_limit: Optional[int] = Field( 1228 None, 1229 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1230 title="Property Limit", 1231 ) 1232 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1233 None, 1234 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1235 title="Record Merge Strategy", 1236 ) 1237 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1238 1239 1240class RecordFilter(BaseModel): 1241 type: Literal["RecordFilter"] 1242 condition: Optional[str] = Field( 1243 "", 1244 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1245 examples=[ 1246 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1247 "{{ record.status in ['active', 'expired'] }}", 1248 ], 1249 title="Condition", 1250 ) 1251 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1252 1253 1254class SchemaNormalization(Enum): 1255 Default = "Default" 1256 None_ = "None" 1257 1258 1259class RemoveFields(BaseModel): 1260 type: Literal["RemoveFields"] 1261 condition: Optional[str] = Field( 1262 "", 1263 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1264 examples=[ 1265 "{{ property|string == '' }}", 1266 "{{ property is integer }}", 1267 "{{ property|length > 5 }}", 1268 "{{ property == 'some_string_to_match' }}", 1269 ], 1270 ) 1271 field_pointers: List[List[str]] = Field( 1272 ..., 1273 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1274 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1275 title="Field Paths", 1276 ) 1277 1278 1279class RequestPath(BaseModel): 1280 type: Literal["RequestPath"] 1281 1282 1283class InjectInto(Enum): 1284 request_parameter = "request_parameter" 1285 header = "header" 1286 body_data = "body_data" 1287 body_json = "body_json" 1288 1289 1290class RequestOption(BaseModel): 1291 type: Literal["RequestOption"] 1292 inject_into: InjectInto = Field( 1293 ..., 1294 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1295 examples=["request_parameter", "header", "body_data", "body_json"], 1296 title="Inject Into", 1297 ) 1298 field_name: Optional[str] = Field( 1299 None, 1300 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1301 examples=["segment_id"], 1302 title="Field Name", 1303 ) 1304 field_path: Optional[List[str]] = Field( 1305 None, 1306 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1307 examples=[["data", "viewer", "id"]], 1308 title="Field Path", 1309 ) 1310 1311 1312class Schemas(BaseModel): 1313 pass 1314 1315 class Config: 1316 extra = Extra.allow 1317 1318 1319class LegacySessionTokenAuthenticator(BaseModel): 1320 type: Literal["LegacySessionTokenAuthenticator"] 1321 header: str = Field( 1322 ..., 1323 description="The name of the session token header that will be injected in the request", 1324 examples=["X-Session"], 1325 title="Session Request Header", 1326 ) 1327 login_url: str = Field( 1328 ..., 1329 description="Path of the login URL (do not include the base URL)", 1330 examples=["session"], 1331 title="Login Path", 1332 ) 1333 session_token: Optional[str] = Field( 1334 None, 1335 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1336 example=["{{ config['session_token'] }}"], 1337 title="Session Token", 1338 ) 1339 session_token_response_key: str = Field( 1340 ..., 1341 description="Name of the key of the session token to be extracted from the response", 1342 examples=["id"], 1343 title="Response Token Response Key", 1344 ) 1345 username: Optional[str] = Field( 1346 None, 1347 description="Username used to authenticate and obtain a session token", 1348 examples=[" {{ config['username'] }}"], 1349 title="Username", 1350 ) 1351 password: Optional[str] = Field( 1352 "", 1353 description="Password used to authenticate and obtain a session token", 1354 examples=["{{ config['password'] }}", ""], 1355 title="Password", 1356 ) 1357 validate_session_url: str = Field( 1358 ..., 1359 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1360 examples=["user/current"], 1361 title="Validate Session Path", 1362 ) 1363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1364 1365 1366class Action1(Enum): 1367 SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR" 1368 RESET = "RESET" 1369 1370 1371class PaginationResetLimits(BaseModel): 1372 type: Literal["PaginationResetLimits"] 1373 number_of_records: Optional[int] = None 1374 1375 1376class CsvDecoder(BaseModel): 1377 type: Literal["CsvDecoder"] 1378 encoding: Optional[str] = "utf-8" 1379 delimiter: Optional[str] = "," 1380 set_values_to_none: Optional[List[str]] = None 1381 1382 1383class AsyncJobStatusMap(BaseModel): 1384 type: Optional[Literal["AsyncJobStatusMap"]] = None 1385 running: List[str] 1386 completed: List[str] 1387 failed: List[str] 1388 timeout: List[str] 1389 skipped: Optional[List[str]] = None 1390 1391 1392class ValueType(Enum): 1393 string = "string" 1394 number = "number" 1395 integer = "integer" 1396 boolean = "boolean" 1397 1398 1399class WaitTimeFromHeader(BaseModel): 1400 type: Literal["WaitTimeFromHeader"] 1401 header: str = Field( 1402 ..., 1403 description="The name of the response header defining how long to wait before retrying.", 1404 examples=["Retry-After"], 1405 title="Response Header Name", 1406 ) 1407 regex: Optional[str] = Field( 1408 None, 1409 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1410 examples=["([-+]?\\d+)"], 1411 title="Extraction Regex", 1412 ) 1413 max_waiting_time_in_seconds: Optional[float] = Field( 1414 None, 1415 description="Given the value extracted from the header is greater than this value, stop the stream.", 1416 examples=[3600], 1417 title="Max Waiting Time in Seconds", 1418 ) 1419 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1420 1421 1422class WaitUntilTimeFromHeader(BaseModel): 1423 type: Literal["WaitUntilTimeFromHeader"] 1424 header: str = Field( 1425 ..., 1426 description="The name of the response header defining how long to wait before retrying.", 1427 examples=["wait_time"], 1428 title="Response Header", 1429 ) 1430 min_wait: Optional[Union[float, str]] = Field( 1431 None, 1432 description="Minimum time to wait before retrying.", 1433 examples=[10, "60"], 1434 title="Minimum Wait Time", 1435 ) 1436 regex: Optional[str] = Field( 1437 None, 1438 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1439 examples=["([-+]?\\d+)"], 1440 title="Extraction Regex", 1441 ) 1442 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1443 1444 1445class ComponentMappingDefinition(BaseModel): 1446 type: Literal["ComponentMappingDefinition"] 1447 field_path: List[str] = Field( 1448 ..., 1449 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1450 examples=[ 1451 ["name"], 1452 ["retriever", "requester", "url"], 1453 ["retriever", "requester", "{{ components_values.field }}"], 1454 ["*", "**", "name"], 1455 ], 1456 title="Field Path", 1457 ) 1458 value: str = Field( 1459 ..., 1460 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1461 examples=[ 1462 "{{ components_values['updates'] }}", 1463 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1464 "{{ config['segment_id'] }}", 1465 "{{ stream_slice['parent_id'] }}", 1466 "{{ stream_slice['extra_fields']['name'] }}", 1467 ], 1468 title="Value", 1469 ) 1470 value_type: Optional[ValueType] = Field( 1471 None, 1472 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1473 title="Value Type", 1474 ) 1475 create_or_update: Optional[bool] = Field( 1476 False, 1477 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1478 title="Create or Update", 1479 ) 1480 condition: Optional[str] = Field( 1481 None, 1482 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1483 examples=[ 1484 "{{ components_values.get('cursor_field', None) }}", 1485 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1486 ], 1487 title="Condition", 1488 ) 1489 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1490 1491 1492class StreamConfig(BaseModel): 1493 type: Literal["StreamConfig"] 1494 configs_pointer: List[str] = Field( 1495 ..., 1496 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1497 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1498 title="Configs Pointer", 1499 ) 1500 default_values: Optional[List[Dict[str, Any]]] = Field( 1501 None, 1502 description="A list of default values, each matching the structure expected from the parsed component value.", 1503 title="Default Values", 1504 ) 1505 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1506 1507 1508class ConfigComponentsResolver(BaseModel): 1509 type: Literal["ConfigComponentsResolver"] 1510 stream_config: Union[List[StreamConfig], StreamConfig] 1511 components_mapping: List[ComponentMappingDefinition] 1512 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1513 1514 1515class StreamParametersDefinition(BaseModel): 1516 type: Literal["StreamParametersDefinition"] 1517 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1518 ..., 1519 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1520 examples=[ 1521 [ 1522 { 1523 "name": "test stream", 1524 "$parameters": {"entity": "test entity"}, 1525 "primary_key": "test key", 1526 } 1527 ] 1528 ], 1529 title="Stream Parameters", 1530 ) 1531 1532 1533class ParametrizedComponentsResolver(BaseModel): 1534 type: Literal["ParametrizedComponentsResolver"] 1535 stream_parameters: StreamParametersDefinition 1536 components_mapping: List[ComponentMappingDefinition] 1537 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1538 1539 1540class RequestBodyPlainText(BaseModel): 1541 type: Literal["RequestBodyPlainText"] 1542 value: str 1543 1544 1545class RequestBodyUrlEncodedForm(BaseModel): 1546 type: Literal["RequestBodyUrlEncodedForm"] 1547 value: Dict[str, str] 1548 1549 1550class RequestBodyJsonObject(BaseModel): 1551 type: Literal["RequestBodyJsonObject"] 1552 value: Dict[str, Any] 1553 1554 1555class RequestBodyGraphQlQuery(BaseModel): 1556 class Config: 1557 extra = Extra.allow 1558 1559 query: str = Field(..., description="The GraphQL query to be executed") 1560 1561 1562class ValidateAdheresToSchema(BaseModel): 1563 type: Literal["ValidateAdheresToSchema"] 1564 base_schema: Union[str, Dict[str, Any]] = Field( 1565 ..., 1566 description="The base JSON schema against which the user-provided schema will be validated.", 1567 examples=[ 1568 "{{ config['report_validation_schema'] }}", 1569 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1570 { 1571 "$schema": "http://json-schema.org/draft-07/schema#", 1572 "title": "Person", 1573 "type": "object", 1574 "properties": { 1575 "name": {"type": "string", "description": "The person's name"}, 1576 "age": { 1577 "type": "integer", 1578 "minimum": 0, 1579 "description": "The person's age", 1580 }, 1581 }, 1582 "required": ["name", "age"], 1583 }, 1584 ], 1585 title="Base JSON Schema", 1586 ) 1587 1588 1589class CustomValidationStrategy(BaseModel): 1590 class Config: 1591 extra = Extra.allow 1592 1593 type: Literal["CustomValidationStrategy"] 1594 class_name: str = Field( 1595 ..., 1596 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1597 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1598 title="Class Name", 1599 ) 1600 1601 1602class ConfigRemapField(BaseModel): 1603 type: Literal["ConfigRemapField"] 1604 map: Union[Dict[str, Any], str] = Field( 1605 ..., 1606 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1607 examples=[ 1608 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1609 "{{ config['status_mapping'] }}", 1610 ], 1611 title="Value Mapping", 1612 ) 1613 field_path: List[str] = Field( 1614 ..., 1615 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1616 examples=[ 1617 ["status"], 1618 ["data", "status"], 1619 ["data", "{{ config.name }}", "status"], 1620 ["data", "*", "status"], 1621 ], 1622 title="Field Path", 1623 ) 1624 1625 1626class ConfigRemoveFields(BaseModel): 1627 type: Literal["ConfigRemoveFields"] 1628 field_pointers: List[List[str]] = Field( 1629 ..., 1630 description="A list of field pointers to be removed from the config.", 1631 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1632 title="Field Pointers", 1633 ) 1634 condition: Optional[str] = Field( 1635 "", 1636 description="Fields will be removed if expression is evaluated to True.", 1637 examples=[ 1638 "{{ config['environemnt'] == 'sandbox' }}", 1639 "{{ property is integer }}", 1640 "{{ property|length > 5 }}", 1641 "{{ property == 'some_string_to_match' }}", 1642 ], 1643 ) 1644 1645 1646class CustomConfigTransformation(BaseModel): 1647 type: Literal["CustomConfigTransformation"] 1648 class_name: str = Field( 1649 ..., 1650 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1651 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1652 ) 1653 parameters: Optional[Dict[str, Any]] = Field( 1654 None, 1655 alias="$parameters", 1656 description="Additional parameters to be passed to the custom config transformation.", 1657 ) 1658 1659 1660class AddedFieldDefinition(BaseModel): 1661 type: Literal["AddedFieldDefinition"] 1662 path: List[str] = Field( 1663 ..., 1664 description="List of strings defining the path where to add the value on the record.", 1665 examples=[["segment_id"], ["metadata", "segment_id"]], 1666 title="Path", 1667 ) 1668 value: str = Field( 1669 ..., 1670 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1671 examples=[ 1672 "{{ record['updates'] }}", 1673 "{{ record['MetaData']['LastUpdatedTime'] }}", 1674 "{{ stream_partition['segment_id'] }}", 1675 ], 1676 title="Value", 1677 ) 1678 value_type: Optional[ValueType] = Field( 1679 None, 1680 description="Type of the value. If not specified, the type will be inferred from the value.", 1681 title="Value Type", 1682 ) 1683 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1684 1685 1686class AddFields(BaseModel): 1687 type: Literal["AddFields"] 1688 fields: List[AddedFieldDefinition] = Field( 1689 ..., 1690 description="List of transformations (path and corresponding value) that will be added to the record.", 1691 title="Fields", 1692 ) 1693 condition: Optional[str] = Field( 1694 "", 1695 description="Fields will be added if expression is evaluated to True.", 1696 examples=[ 1697 "{{ property|string == '' }}", 1698 "{{ property is integer }}", 1699 "{{ property|length > 5 }}", 1700 "{{ property == 'some_string_to_match' }}", 1701 ], 1702 ) 1703 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1704 1705 1706class ApiKeyAuthenticator(BaseModel): 1707 type: Literal["ApiKeyAuthenticator"] 1708 api_token: Optional[str] = Field( 1709 None, 1710 description="The API key to inject in the request. Fill it in the user inputs.", 1711 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1712 title="API Key", 1713 ) 1714 header: Optional[str] = Field( 1715 None, 1716 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1717 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1718 title="Header Name", 1719 ) 1720 inject_into: Optional[RequestOption] = Field( 1721 None, 1722 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1723 examples=[ 1724 {"inject_into": "header", "field_name": "Authorization"}, 1725 {"inject_into": "request_parameter", "field_name": "authKey"}, 1726 ], 1727 title="Inject API Key Into Outgoing HTTP Request", 1728 ) 1729 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1730 1731 1732class AuthFlow(BaseModel): 1733 auth_flow_type: Optional[AuthFlowType] = Field( 1734 None, description="The type of auth to use", title="Auth flow type" 1735 ) 1736 predicate_key: Optional[List[str]] = Field( 1737 None, 1738 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1739 examples=[["credentials", "auth_type"]], 1740 title="Predicate key", 1741 ) 1742 predicate_value: Optional[str] = Field( 1743 None, 1744 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1745 examples=["Oauth"], 1746 title="Predicate value", 1747 ) 1748 oauth_config_specification: Optional[OAuthConfigSpecification] = None 1749 1750 1751class CheckStream(BaseModel): 1752 type: Literal["CheckStream"] 1753 stream_names: Optional[List[str]] = Field( 1754 None, 1755 description="Names of the streams to try reading from when running a check operation.", 1756 examples=[["users"], ["users", "contacts"]], 1757 title="Stream Names", 1758 ) 1759 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None 1760 1761 1762class IncrementingCountCursor(BaseModel): 1763 type: Literal["IncrementingCountCursor"] 1764 cursor_field: str = Field( 1765 ..., 1766 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1767 examples=["created_at", "{{ config['record_cursor'] }}"], 1768 title="Cursor Field", 1769 ) 1770 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1771 None, 1772 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1773 title="Allow Catalog Defined Cursor Field", 1774 ) 1775 start_value: Optional[Union[str, int]] = Field( 1776 None, 1777 description="The value that determines the earliest record that should be synced.", 1778 examples=[0, "{{ config['start_value'] }}"], 1779 title="Start Value", 1780 ) 1781 start_value_option: Optional[RequestOption] = Field( 1782 None, 1783 description="Optionally configures how the start value will be sent in requests to the source API.", 1784 title="Inject Start Value Into Outgoing HTTP Request", 1785 ) 1786 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1787 1788 1789class DatetimeBasedCursor(BaseModel): 1790 type: Literal["DatetimeBasedCursor"] 1791 clamping: Optional[Clamping] = Field( 1792 None, 1793 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1794 title="Date Range Clamping", 1795 ) 1796 cursor_field: str = Field( 1797 ..., 1798 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1799 examples=["created_at", "{{ config['record_cursor'] }}"], 1800 title="Cursor Field", 1801 ) 1802 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1803 None, 1804 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1805 title="Allow Catalog Defined Cursor Field", 1806 ) 1807 cursor_datetime_formats: Optional[List[str]] = Field( 1808 None, 1809 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1810 examples=[ 1811 "%Y-%m-%d", 1812 "%Y-%m-%d %H:%M:%S", 1813 "%Y-%m-%dT%H:%M:%S", 1814 "%Y-%m-%dT%H:%M:%SZ", 1815 "%Y-%m-%dT%H:%M:%S%z", 1816 "%Y-%m-%dT%H:%M:%S.%fZ", 1817 "%Y-%m-%dT%H:%M:%S.%f%z", 1818 "%Y-%m-%d %H:%M:%S.%f+00:00", 1819 "%s", 1820 "%ms", 1821 ], 1822 title="Cursor Datetime Formats", 1823 ) 1824 start_datetime: Union[MinMaxDatetime, str] = Field( 1825 ..., 1826 description="The datetime that determines the earliest record that should be synced.", 1827 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1828 title="Start Datetime", 1829 ) 1830 start_time_option: Optional[RequestOption] = Field( 1831 None, 1832 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1833 title="Inject Start Time Into Outgoing HTTP Request", 1834 ) 1835 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1836 None, 1837 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1838 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1839 title="End Datetime", 1840 ) 1841 end_time_option: Optional[RequestOption] = Field( 1842 None, 1843 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1844 title="Inject End Time Into Outgoing HTTP Request", 1845 ) 1846 datetime_format: str = Field( 1847 ..., 1848 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1849 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1850 title="Outgoing Datetime Format", 1851 ) 1852 cursor_granularity: Optional[str] = Field( 1853 None, 1854 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1855 examples=["PT1S"], 1856 title="Cursor Granularity", 1857 ) 1858 is_data_feed: Optional[bool] = Field( 1859 None, 1860 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1861 title="Data Feed API", 1862 ) 1863 is_client_side_incremental: Optional[bool] = Field( 1864 None, 1865 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1866 title="Client-side Incremental Filtering", 1867 ) 1868 is_compare_strictly: Optional[bool] = Field( 1869 False, 1870 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1871 title="Strict Start-End Time Comparison", 1872 ) 1873 global_substream_cursor: Optional[bool] = Field( 1874 False, 1875 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1876 title="Global Substream Cursor", 1877 ) 1878 lookback_window: Optional[str] = Field( 1879 None, 1880 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1881 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1882 title="Lookback Window", 1883 ) 1884 partition_field_end: Optional[str] = Field( 1885 None, 1886 description="Name of the partition start time field.", 1887 examples=["ending_time"], 1888 title="Partition Field End", 1889 ) 1890 partition_field_start: Optional[str] = Field( 1891 None, 1892 description="Name of the partition end time field.", 1893 examples=["starting_time"], 1894 title="Partition Field Start", 1895 ) 1896 step: Optional[str] = Field( 1897 None, 1898 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1899 examples=["P1W", "{{ config['step_increment'] }}"], 1900 title="Step", 1901 ) 1902 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1903 1904 1905class JwtAuthenticator(BaseModel): 1906 type: Literal["JwtAuthenticator"] 1907 secret_key: str = Field( 1908 ..., 1909 description="Secret used to sign the JSON web token.", 1910 examples=["{{ config['secret_key'] }}"], 1911 title="Secret Key", 1912 ) 1913 base64_encode_secret_key: Optional[bool] = Field( 1914 False, 1915 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1916 title="Base64-encode Secret Key", 1917 ) 1918 algorithm: Algorithm = Field( 1919 ..., 1920 description="Algorithm used to sign the JSON web token.", 1921 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1922 title="Algorithm", 1923 ) 1924 token_duration: Optional[int] = Field( 1925 1200, 1926 description="The amount of time in seconds a JWT token can be valid after being issued.", 1927 examples=[1200, 3600], 1928 title="Token Duration", 1929 ) 1930 header_prefix: Optional[str] = Field( 1931 None, 1932 description="The prefix to be used within the Authentication header.", 1933 examples=["Bearer", "Basic"], 1934 title="Header Prefix", 1935 ) 1936 jwt_headers: Optional[JwtHeaders] = Field( 1937 None, 1938 description="JWT headers used when signing JSON web token.", 1939 title="JWT Headers", 1940 ) 1941 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1942 None, 1943 description="Additional headers to be included with the JWT headers object.", 1944 title="Additional JWT Headers", 1945 ) 1946 jwt_payload: Optional[JwtPayload] = Field( 1947 None, 1948 description="JWT Payload used when signing JSON web token.", 1949 title="JWT Payload", 1950 ) 1951 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1952 None, 1953 description="Additional properties to be added to the JWT payload.", 1954 title="Additional JWT Payload Properties", 1955 ) 1956 passphrase: Optional[str] = Field( 1957 None, 1958 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1959 examples=["{{ config['passphrase'] }}"], 1960 title="Passphrase", 1961 ) 1962 request_option: Optional[RequestOption] = Field( 1963 None, 1964 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1965 title="Request Option", 1966 ) 1967 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 1968 1969 1970class OAuthAuthenticator(BaseModel): 1971 type: Literal["OAuthAuthenticator"] 1972 client_id_name: Optional[str] = Field( 1973 "client_id", 1974 description="The name of the property to use to refresh the `access_token`.", 1975 examples=["custom_app_id"], 1976 title="Client ID Property Name", 1977 ) 1978 client_id: Optional[str] = Field( 1979 None, 1980 description="The OAuth client ID. Fill it in the user inputs.", 1981 examples=[ 1982 "{{ config['client_id'] }}", 1983 "{{ config['credentials']['client_id }}", 1984 ], 1985 title="Client ID", 1986 ) 1987 client_secret_name: Optional[str] = Field( 1988 "client_secret", 1989 description="The name of the property to use to refresh the `access_token`.", 1990 examples=["custom_app_secret"], 1991 title="Client Secret Property Name", 1992 ) 1993 client_secret: Optional[str] = Field( 1994 None, 1995 description="The OAuth client secret. Fill it in the user inputs.", 1996 examples=[ 1997 "{{ config['client_secret'] }}", 1998 "{{ config['credentials']['client_secret }}", 1999 ], 2000 title="Client Secret", 2001 ) 2002 refresh_token_name: Optional[str] = Field( 2003 "refresh_token", 2004 description="The name of the property to use to refresh the `access_token`.", 2005 examples=["custom_app_refresh_value"], 2006 title="Refresh Token Property Name", 2007 ) 2008 refresh_token: Optional[str] = Field( 2009 None, 2010 description="Credential artifact used to get a new access token.", 2011 examples=[ 2012 "{{ config['refresh_token'] }}", 2013 "{{ config['credentials]['refresh_token'] }}", 2014 ], 2015 title="Refresh Token", 2016 ) 2017 token_refresh_endpoint: Optional[str] = Field( 2018 None, 2019 description="The full URL to call to obtain a new access token.", 2020 examples=["https://connect.squareup.com/oauth2/token"], 2021 title="Token Refresh Endpoint", 2022 ) 2023 access_token_name: Optional[str] = Field( 2024 "access_token", 2025 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2026 examples=["access_token"], 2027 title="Access Token Property Name", 2028 ) 2029 access_token_value: Optional[str] = Field( 2030 None, 2031 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2032 examples=["secret_access_token_value"], 2033 title="Access Token Value", 2034 ) 2035 expires_in_name: Optional[str] = Field( 2036 "expires_in", 2037 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2038 examples=["expires_in"], 2039 title="Token Expiry Property Name", 2040 ) 2041 grant_type_name: Optional[str] = Field( 2042 "grant_type", 2043 description="The name of the property to use to refresh the `access_token`.", 2044 examples=["custom_grant_type"], 2045 title="Grant Type Property Name", 2046 ) 2047 grant_type: Optional[str] = Field( 2048 "refresh_token", 2049 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2050 examples=["refresh_token", "client_credentials"], 2051 title="Grant Type", 2052 ) 2053 refresh_request_body: Optional[Dict[str, Any]] = Field( 2054 None, 2055 description="Body of the request sent to get a new access token.", 2056 examples=[ 2057 { 2058 "applicationId": "{{ config['application_id'] }}", 2059 "applicationSecret": "{{ config['application_secret'] }}", 2060 "token": "{{ config['token'] }}", 2061 } 2062 ], 2063 title="Refresh Request Body", 2064 ) 2065 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2066 None, 2067 description="Headers of the request sent to get a new access token.", 2068 examples=[ 2069 { 2070 "Authorization": "<AUTH_TOKEN>", 2071 "Content-Type": "application/x-www-form-urlencoded", 2072 } 2073 ], 2074 title="Refresh Request Headers", 2075 ) 2076 send_refresh_request_as_query_params: Optional[bool] = Field( 2077 False, 2078 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2079 examples=[True], 2080 title="Send Refresh Request As Query Params", 2081 ) 2082 scopes: Optional[List[str]] = Field( 2083 None, 2084 description="List of scopes that should be granted to the access token.", 2085 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2086 title="Scopes", 2087 ) 2088 token_expiry_date: Optional[str] = Field( 2089 None, 2090 description="The access token expiry date.", 2091 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2092 title="Token Expiry Date", 2093 ) 2094 token_expiry_date_format: Optional[str] = Field( 2095 None, 2096 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2097 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2098 title="Token Expiry Date Format", 2099 ) 2100 refresh_token_error_status_codes: Optional[List[int]] = Field( 2101 None, 2102 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2103 examples=[[400, 500]], 2104 title="Refresh Token Error Status Codes", 2105 ) 2106 refresh_token_error_key: Optional[str] = Field( 2107 None, 2108 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2109 examples=["error"], 2110 title="Refresh Token Error Key", 2111 ) 2112 refresh_token_error_values: Optional[List[str]] = Field( 2113 None, 2114 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2115 examples=[["invalid_grant", "invalid_permissions"]], 2116 title="Refresh Token Error Values", 2117 ) 2118 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2119 None, 2120 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2121 title="Refresh Token Updater", 2122 ) 2123 profile_assertion: Optional[JwtAuthenticator] = Field( 2124 None, 2125 description="The authenticator being used to authenticate the client authenticator.", 2126 title="Profile Assertion", 2127 ) 2128 use_profile_assertion: Optional[bool] = Field( 2129 False, 2130 description="Enable using profile assertion as a flow for OAuth authorization.", 2131 title="Use Profile Assertion", 2132 ) 2133 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2134 2135 2136class FixedWindowCallRatePolicy(BaseModel): 2137 class Config: 2138 extra = Extra.allow 2139 2140 type: Literal["FixedWindowCallRatePolicy"] 2141 period: str = Field( 2142 ..., description="The time interval for the rate limit window.", title="Period" 2143 ) 2144 call_limit: int = Field( 2145 ..., 2146 description="The maximum number of calls allowed within the period.", 2147 title="Call Limit", 2148 ) 2149 matchers: List[HttpRequestRegexMatcher] = Field( 2150 ..., 2151 description="List of matchers that define which requests this policy applies to.", 2152 title="Matchers", 2153 ) 2154 2155 2156class MovingWindowCallRatePolicy(BaseModel): 2157 class Config: 2158 extra = Extra.allow 2159 2160 type: Literal["MovingWindowCallRatePolicy"] 2161 rates: List[Rate] = Field( 2162 ..., 2163 description="List of rates that define the call limits for different time intervals.", 2164 title="Rates", 2165 ) 2166 matchers: List[HttpRequestRegexMatcher] = Field( 2167 ..., 2168 description="List of matchers that define which requests this policy applies to.", 2169 title="Matchers", 2170 ) 2171 2172 2173class UnlimitedCallRatePolicy(BaseModel): 2174 class Config: 2175 extra = Extra.allow 2176 2177 type: Literal["UnlimitedCallRatePolicy"] 2178 matchers: List[HttpRequestRegexMatcher] = Field( 2179 ..., 2180 description="List of matchers that define which requests this policy applies to.", 2181 title="Matchers", 2182 ) 2183 2184 2185class DefaultErrorHandler(BaseModel): 2186 type: Literal["DefaultErrorHandler"] 2187 backoff_strategies: Optional[ 2188 List[ 2189 Union[ 2190 ConstantBackoffStrategy, 2191 ExponentialBackoffStrategy, 2192 WaitTimeFromHeader, 2193 WaitUntilTimeFromHeader, 2194 CustomBackoffStrategy, 2195 ] 2196 ] 2197 ] = Field( 2198 None, 2199 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2200 title="Backoff Strategies", 2201 ) 2202 max_retries: Optional[Union[int, str]] = Field( 2203 5, 2204 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2205 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2206 title="Max Retry Count", 2207 ) 2208 response_filters: Optional[List[HttpResponseFilter]] = Field( 2209 None, 2210 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2211 title="Response Filters", 2212 ) 2213 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2214 2215 2216class DefaultPaginator(BaseModel): 2217 type: Literal["DefaultPaginator"] 2218 pagination_strategy: Union[ 2219 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2220 ] = Field( 2221 ..., 2222 description="Strategy defining how records are paginated.", 2223 title="Pagination Strategy", 2224 ) 2225 page_size_option: Optional[RequestOption] = Field( 2226 None, title="Inject Page Size Into Outgoing HTTP Request" 2227 ) 2228 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2229 None, 2230 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2231 title="Inject Page Token Into Outgoing HTTP Request", 2232 ) 2233 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2234 2235 2236class RecordExpander(BaseModel): 2237 type: Literal["RecordExpander"] 2238 expand_records_from_field: List[str] = Field( 2239 ..., 2240 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2241 examples=[ 2242 ["lines", "data"], 2243 ["items"], 2244 ["nested", "array"], 2245 ["sections", "*", "items"], 2246 ], 2247 title="Expand Records From Field", 2248 ) 2249 remain_original_record: Optional[bool] = Field( 2250 False, 2251 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2252 title="Remain Original Record", 2253 ) 2254 on_no_records: Optional[OnNoRecords] = Field( 2255 OnNoRecords.skip, 2256 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2257 title="On No Records", 2258 ) 2259 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2260 2261 2262class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2263 type: Literal["ApiKey"] 2264 inject_into: RequestOption = Field( 2265 ..., 2266 description="Configure how the API Key will be sent in requests to the source API.", 2267 examples=[ 2268 {"inject_into": "header", "field_name": "Authorization"}, 2269 {"inject_into": "request_parameter", "field_name": "authKey"}, 2270 ], 2271 title="Inject API Key Into Outgoing HTTP Request", 2272 ) 2273 api_token: Optional[str] = Field( 2274 "{{ session_token }}", 2275 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2276 examples=[ 2277 "{{ session_token }}", 2278 "Token {{ session_token }}", 2279 "Bearer {{ session_token }}", 2280 ], 2281 title="API Token Template", 2282 ) 2283 2284 2285class JsonSchemaPropertySelector(BaseModel): 2286 type: Literal["JsonSchemaPropertySelector"] 2287 transformations: Optional[ 2288 List[ 2289 Union[ 2290 AddFields, 2291 RemoveFields, 2292 KeysToLower, 2293 KeysToSnakeCase, 2294 FlattenFields, 2295 DpathFlattenFields, 2296 KeysReplace, 2297 CustomTransformation, 2298 ] 2299 ] 2300 ] = Field( 2301 None, 2302 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2303 title="Transformations", 2304 ) 2305 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2306 2307 2308class ListPartitionRouter(BaseModel): 2309 type: Literal["ListPartitionRouter"] 2310 cursor_field: str = Field( 2311 ..., 2312 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2313 examples=["section", "{{ config['section_key'] }}"], 2314 title="Current Partition Value Identifier", 2315 ) 2316 values: Union[str, List[str]] = Field( 2317 ..., 2318 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2319 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2320 title="Partition Values", 2321 ) 2322 request_option: Optional[RequestOption] = Field( 2323 None, 2324 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2325 title="Inject Partition Value Into Outgoing HTTP Request", 2326 ) 2327 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2328 2329 2330class PaginationReset(BaseModel): 2331 type: Literal["PaginationReset"] 2332 action: Action1 2333 limits: Optional[PaginationResetLimits] = None 2334 2335 2336class GzipDecoder(BaseModel): 2337 type: Literal["GzipDecoder"] 2338 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] 2339 2340 2341class RequestBodyGraphQL(BaseModel): 2342 type: Literal["RequestBodyGraphQL"] 2343 value: RequestBodyGraphQlQuery 2344 2345 2346class DpathValidator(BaseModel): 2347 type: Literal["DpathValidator"] 2348 field_path: List[str] = Field( 2349 ..., 2350 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2351 examples=[ 2352 ["data"], 2353 ["data", "records"], 2354 ["data", "{{ parameters.name }}"], 2355 ["data", "*", "record"], 2356 ], 2357 title="Field Path", 2358 ) 2359 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2360 ..., 2361 description="The condition that the specified config value will be evaluated against", 2362 title="Validation Strategy", 2363 ) 2364 2365 2366class PredicateValidator(BaseModel): 2367 type: Literal["PredicateValidator"] 2368 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2369 ..., 2370 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2371 examples=[ 2372 "test-value", 2373 "{{ config['api_version'] }}", 2374 "{{ config['tenant_id'] }}", 2375 123, 2376 ], 2377 title="Value", 2378 ) 2379 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2380 ..., 2381 description="The validation strategy to apply to the value.", 2382 title="Validation Strategy", 2383 ) 2384 2385 2386class ConfigAddFields(BaseModel): 2387 type: Literal["ConfigAddFields"] 2388 fields: List[AddedFieldDefinition] = Field( 2389 ..., 2390 description="A list of transformations (path and corresponding value) that will be added to the config.", 2391 title="Fields", 2392 ) 2393 condition: Optional[str] = Field( 2394 "", 2395 description="Fields will be added if expression is evaluated to True.", 2396 examples=[ 2397 "{{ config['environemnt'] == 'sandbox' }}", 2398 "{{ property is integer }}", 2399 "{{ property|length > 5 }}", 2400 "{{ property == 'some_string_to_match' }}", 2401 ], 2402 ) 2403 2404 2405class CompositeErrorHandler(BaseModel): 2406 type: Literal["CompositeErrorHandler"] 2407 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2408 Field( 2409 ..., 2410 description="List of error handlers to iterate on to determine how to handle a failed response.", 2411 title="Error Handlers", 2412 ) 2413 ) 2414 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2415 2416 2417class HTTPAPIBudget(BaseModel): 2418 class Config: 2419 extra = Extra.allow 2420 2421 type: Literal["HTTPAPIBudget"] 2422 policies: List[ 2423 Union[ 2424 FixedWindowCallRatePolicy, 2425 MovingWindowCallRatePolicy, 2426 UnlimitedCallRatePolicy, 2427 ] 2428 ] = Field( 2429 ..., 2430 description="List of call rate policies that define how many calls are allowed.", 2431 title="Policies", 2432 ) 2433 ratelimit_reset_header: Optional[str] = Field( 2434 "ratelimit-reset", 2435 description="The HTTP response header name that indicates when the rate limit resets.", 2436 title="Rate Limit Reset Header", 2437 ) 2438 ratelimit_remaining_header: Optional[str] = Field( 2439 "ratelimit-remaining", 2440 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2441 title="Rate Limit Remaining Header", 2442 ) 2443 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2444 [429], 2445 description="List of HTTP status codes that indicate a rate limit has been hit.", 2446 title="Status Codes for Rate Limit Hit", 2447 ) 2448 2449 2450class DpathExtractor(BaseModel): 2451 type: Literal["DpathExtractor"] 2452 field_path: List[str] = Field( 2453 ..., 2454 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2455 examples=[ 2456 ["data"], 2457 ["data", "records"], 2458 ["data", "{{ parameters.name }}"], 2459 ["data", "*", "record"], 2460 ], 2461 title="Field Path", 2462 ) 2463 record_expander: Optional[RecordExpander] = Field( 2464 None, 2465 description="Optional component to expand records by extracting items from nested array fields.", 2466 title="Record Expander", 2467 ) 2468 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2469 2470 2471class ZipfileDecoder(BaseModel): 2472 class Config: 2473 extra = Extra.allow 2474 2475 type: Literal["ZipfileDecoder"] 2476 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2477 ..., 2478 description="Parser to parse the decompressed data from the zipfile(s).", 2479 title="Parser", 2480 ) 2481 2482 2483class RecordSelector(BaseModel): 2484 type: Literal["RecordSelector"] 2485 extractor: Union[DpathExtractor, CustomRecordExtractor] 2486 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2487 None, 2488 description="Responsible for filtering records to be emitted by the Source.", 2489 title="Record Filter", 2490 ) 2491 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2492 None, 2493 description="Responsible for normalization according to the schema.", 2494 title="Schema Normalization", 2495 ) 2496 transform_before_filtering: Optional[bool] = Field( 2497 None, 2498 description="If true, transformation will be applied before record filtering.", 2499 title="Transform Before Filtering", 2500 ) 2501 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2502 2503 2504class ConfigMigration(BaseModel): 2505 type: Literal["ConfigMigration"] 2506 description: Optional[str] = Field( 2507 None, description="The description/purpose of the config migration." 2508 ) 2509 transformations: List[ 2510 Union[ 2511 ConfigRemapField, 2512 ConfigAddFields, 2513 ConfigRemoveFields, 2514 CustomConfigTransformation, 2515 ] 2516 ] = Field( 2517 ..., 2518 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2519 title="Transformations", 2520 ) 2521 2522 2523class ConfigNormalizationRules(BaseModel): 2524 class Config: 2525 extra = Extra.forbid 2526 2527 type: Literal["ConfigNormalizationRules"] 2528 config_migrations: Optional[List[ConfigMigration]] = Field( 2529 [], 2530 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2531 title="Config Migrations", 2532 ) 2533 transformations: Optional[ 2534 List[ 2535 Union[ 2536 ConfigRemapField, 2537 ConfigAddFields, 2538 ConfigRemoveFields, 2539 CustomConfigTransformation, 2540 ] 2541 ] 2542 ] = Field( 2543 [], 2544 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2545 title="Transformations", 2546 ) 2547 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2548 [], 2549 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2550 title="Validations", 2551 ) 2552 2553 2554class Spec(BaseModel): 2555 type: Literal["Spec"] 2556 connection_specification: Dict[str, Any] = Field( 2557 ..., 2558 description="A connection specification describing how a the connector can be configured.", 2559 title="Connection Specification", 2560 ) 2561 documentation_url: Optional[str] = Field( 2562 None, 2563 description="URL of the connector's documentation page.", 2564 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2565 title="Documentation URL", 2566 ) 2567 advanced_auth: Optional[AuthFlow] = Field( 2568 None, 2569 description="Advanced specification for configuring the authentication flow.", 2570 title="Advanced Auth", 2571 ) 2572 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2573 None, title="Config Normalization Rules" 2574 ) 2575 2576 2577class DeclarativeSource1(BaseModel): 2578 class Config: 2579 extra = Extra.forbid 2580 2581 type: Literal["DeclarativeSource"] 2582 check: Union[CheckStream, CheckDynamicStream] 2583 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2584 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2585 version: str = Field( 2586 ..., 2587 description="The version of the Airbyte CDK used to build and test the source.", 2588 ) 2589 schemas: Optional[Schemas] = None 2590 definitions: Optional[Dict[str, Any]] = None 2591 spec: Optional[Spec] = None 2592 concurrency_level: Optional[ConcurrencyLevel] = None 2593 api_budget: Optional[HTTPAPIBudget] = None 2594 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2595 None, 2596 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2597 title="Stream Groups", 2598 ) 2599 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2600 None, 2601 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2602 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2603 title="Maximum Concurrent Asynchronous Jobs", 2604 ) 2605 metadata: Optional[Dict[str, Any]] = Field( 2606 None, 2607 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2608 ) 2609 description: Optional[str] = Field( 2610 None, 2611 description="A description of the connector. It will be presented on the Source documentation page.", 2612 ) 2613 2614 2615class DeclarativeSource2(BaseModel): 2616 class Config: 2617 extra = Extra.forbid 2618 2619 type: Literal["DeclarativeSource"] 2620 check: Union[CheckStream, CheckDynamicStream] 2621 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2622 None 2623 ) 2624 dynamic_streams: List[DynamicDeclarativeStream] 2625 version: str = Field( 2626 ..., 2627 description="The version of the Airbyte CDK used to build and test the source.", 2628 ) 2629 schemas: Optional[Schemas] = None 2630 definitions: Optional[Dict[str, Any]] = None 2631 spec: Optional[Spec] = None 2632 concurrency_level: Optional[ConcurrencyLevel] = None 2633 api_budget: Optional[HTTPAPIBudget] = None 2634 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2635 None, 2636 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2637 title="Stream Groups", 2638 ) 2639 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2640 None, 2641 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2642 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2643 title="Maximum Concurrent Asynchronous Jobs", 2644 ) 2645 metadata: Optional[Dict[str, Any]] = Field( 2646 None, 2647 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2648 ) 2649 description: Optional[str] = Field( 2650 None, 2651 description="A description of the connector. It will be presented on the Source documentation page.", 2652 ) 2653 2654 2655class DeclarativeSource(BaseModel): 2656 class Config: 2657 extra = Extra.forbid 2658 2659 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2660 ..., 2661 description="An API source that extracts data according to its declarative components.", 2662 title="DeclarativeSource", 2663 ) 2664 2665 2666class SelectiveAuthenticator(BaseModel): 2667 class Config: 2668 extra = Extra.allow 2669 2670 type: Literal["SelectiveAuthenticator"] 2671 authenticator_selection_path: List[str] = Field( 2672 ..., 2673 description="Path of the field in config with selected authenticator name", 2674 examples=[["auth"], ["auth", "type"]], 2675 title="Authenticator Selection Path", 2676 ) 2677 authenticators: Dict[ 2678 str, 2679 Union[ 2680 ApiKeyAuthenticator, 2681 BasicHttpAuthenticator, 2682 BearerAuthenticator, 2683 OAuthAuthenticator, 2684 JwtAuthenticator, 2685 SessionTokenAuthenticator, 2686 LegacySessionTokenAuthenticator, 2687 CustomAuthenticator, 2688 NoAuth, 2689 RateLimitedMultipleTokenAuthenticator, 2690 ], 2691 ] = Field( 2692 ..., 2693 description="Authenticators to select from.", 2694 examples=[ 2695 { 2696 "authenticators": { 2697 "token": "#/definitions/ApiKeyAuthenticator", 2698 "oauth": "#/definitions/OAuthAuthenticator", 2699 "jwt": "#/definitions/JwtAuthenticator", 2700 } 2701 } 2702 ], 2703 title="Authenticators", 2704 ) 2705 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2706 2707 2708class ConditionalStreams(BaseModel): 2709 type: Literal["ConditionalStreams"] 2710 condition: str = Field( 2711 ..., 2712 description="Condition that will be evaluated to determine if a set of streams should be available.", 2713 examples=["{{ config['is_sandbox'] }}"], 2714 title="Condition", 2715 ) 2716 streams: List[DeclarativeStream] = Field( 2717 ..., 2718 description="Streams that will be used during an operation based on the condition.", 2719 title="Streams", 2720 ) 2721 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2722 2723 2724class FileUploader(BaseModel): 2725 type: Literal["FileUploader"] 2726 requester: Union[HttpRequester, CustomRequester] = Field( 2727 ..., 2728 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2729 ) 2730 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2731 ..., 2732 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2733 ) 2734 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2735 None, 2736 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2737 ) 2738 filename_extractor: Optional[str] = Field( 2739 None, 2740 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2741 examples=[ 2742 "{{ record.id }}/{{ record.file_name }}/", 2743 "{{ record.id }}_{{ record.file_name }}/", 2744 ], 2745 ) 2746 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2747 2748 2749class DeclarativeStream(BaseModel): 2750 class Config: 2751 extra = Extra.allow 2752 2753 type: Literal["DeclarativeStream"] 2754 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2755 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2756 ..., 2757 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2758 title="Retriever", 2759 ) 2760 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2761 None, 2762 description="Component used to fetch data incrementally based on a time field in the data.", 2763 title="Incremental Sync", 2764 ) 2765 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2766 schema_loader: Optional[ 2767 Union[ 2768 InlineSchemaLoader, 2769 DynamicSchemaLoader, 2770 JsonFileSchemaLoader, 2771 List[ 2772 Union[ 2773 InlineSchemaLoader, 2774 DynamicSchemaLoader, 2775 JsonFileSchemaLoader, 2776 CustomSchemaLoader, 2777 ] 2778 ], 2779 CustomSchemaLoader, 2780 ] 2781 ] = Field( 2782 None, 2783 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2784 title="Schema Loader", 2785 ) 2786 transformations: Optional[ 2787 List[ 2788 Union[ 2789 AddFields, 2790 RemoveFields, 2791 KeysToLower, 2792 KeysToSnakeCase, 2793 FlattenFields, 2794 DpathFlattenFields, 2795 KeysReplace, 2796 CustomTransformation, 2797 ] 2798 ] 2799 ] = Field( 2800 None, 2801 description="A list of transformations to be applied to each output record.", 2802 title="Transformations", 2803 ) 2804 state_migrations: Optional[ 2805 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2806 ] = Field( 2807 [], 2808 description="Array of state migrations to be applied on the input state", 2809 title="State Migrations", 2810 ) 2811 file_uploader: Optional[FileUploader] = Field( 2812 None, 2813 description="(experimental) Describes how to fetch a file", 2814 title="File Uploader", 2815 ) 2816 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2817 2818 2819class SessionTokenAuthenticator(BaseModel): 2820 type: Literal["SessionTokenAuthenticator"] 2821 login_requester: HttpRequester = Field( 2822 ..., 2823 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2824 examples=[ 2825 { 2826 "type": "HttpRequester", 2827 "url_base": "https://my_api.com", 2828 "path": "/login", 2829 "authenticator": { 2830 "type": "BasicHttpAuthenticator", 2831 "username": "{{ config.username }}", 2832 "password": "{{ config.password }}", 2833 }, 2834 } 2835 ], 2836 title="Login Requester", 2837 ) 2838 session_token_path: List[str] = Field( 2839 ..., 2840 description="The path in the response body returned from the login requester to the session token.", 2841 examples=[["access_token"], ["result", "token"]], 2842 title="Session Token Path", 2843 ) 2844 expiration_duration: Optional[str] = Field( 2845 None, 2846 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2847 examples=["PT1H", "P1D"], 2848 title="Expiration Duration", 2849 ) 2850 request_authentication: Union[ 2851 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2852 ] = Field( 2853 ..., 2854 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2855 title="Data Request Authentication", 2856 ) 2857 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2858 None, description="Component used to decode the response.", title="Decoder" 2859 ) 2860 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 2861 2862 2863class HttpRequester(BaseModelWithDeprecations): 2864 type: Literal["HttpRequester"] 2865 url_base: Optional[str] = Field( 2866 None, 2867 deprecated=True, 2868 deprecation_message="Use `url` field instead.", 2869 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2870 examples=[ 2871 "https://connect.squareup.com/v2", 2872 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2873 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2874 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2875 ], 2876 title="API Base URL", 2877 ) 2878 url: Optional[str] = Field( 2879 None, 2880 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2881 examples=[ 2882 "https://connect.squareup.com/v2", 2883 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2884 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2885 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2886 ], 2887 title="API Endpoint URL", 2888 ) 2889 path: Optional[str] = Field( 2890 None, 2891 deprecated=True, 2892 deprecation_message="Use `url` field instead.", 2893 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2894 examples=[ 2895 "/products", 2896 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2897 "/trades/{{ config['symbol_id'] }}/history", 2898 ], 2899 title="URL Path", 2900 ) 2901 http_method: Optional[HttpMethod] = Field( 2902 HttpMethod.GET, 2903 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2904 examples=["GET", "POST"], 2905 title="HTTP Method", 2906 ) 2907 authenticator: Optional[ 2908 Union[ 2909 ApiKeyAuthenticator, 2910 BasicHttpAuthenticator, 2911 BearerAuthenticator, 2912 OAuthAuthenticator, 2913 JwtAuthenticator, 2914 SessionTokenAuthenticator, 2915 SelectiveAuthenticator, 2916 CustomAuthenticator, 2917 NoAuth, 2918 LegacySessionTokenAuthenticator, 2919 RateLimitedMultipleTokenAuthenticator, 2920 ] 2921 ] = Field( 2922 None, 2923 description="Authentication method to use for requests sent to the API.", 2924 title="Authenticator", 2925 ) 2926 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2927 None, 2928 deprecated=True, 2929 deprecation_message="Use `query_properties` field instead.", 2930 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2931 title="Fetch Properties from Endpoint", 2932 ) 2933 query_properties: Optional[QueryProperties] = Field( 2934 None, 2935 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2936 title="Query Properties", 2937 ) 2938 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2939 None, 2940 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2941 examples=[ 2942 {"unit": "day"}, 2943 { 2944 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2945 }, 2946 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2947 {"sort_by[asc]": "updated_at"}, 2948 ], 2949 title="Query Parameters", 2950 ) 2951 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2952 None, 2953 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2954 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2955 title="Request Headers", 2956 ) 2957 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2958 None, 2959 deprecated=True, 2960 deprecation_message="Use `request_body` field instead.", 2961 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2962 examples=[ 2963 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2964 ], 2965 title="Request Body Payload (Non-JSON)", 2966 ) 2967 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2968 None, 2969 deprecated=True, 2970 deprecation_message="Use `request_body` field instead.", 2971 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2972 examples=[ 2973 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2974 {"key": "{{ config['value'] }}"}, 2975 {"sort": {"field": "updated_at", "order": "ascending"}}, 2976 ], 2977 title="Request Body JSON Payload", 2978 ) 2979 request_body: Optional[ 2980 Union[ 2981 RequestBodyPlainText, 2982 RequestBodyUrlEncodedForm, 2983 RequestBodyJsonObject, 2984 RequestBodyGraphQL, 2985 ] 2986 ] = Field( 2987 None, 2988 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2989 title="Request Body", 2990 ) 2991 error_handler: Optional[ 2992 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2993 ] = Field( 2994 None, 2995 description="Error handler component that defines how to handle errors.", 2996 title="Error Handler", 2997 ) 2998 use_cache: Optional[bool] = Field( 2999 False, 3000 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3001 title="Use Cache", 3002 ) 3003 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3004 3005 3006class DynamicSchemaLoader(BaseModel): 3007 type: Literal["DynamicSchemaLoader"] 3008 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3009 ..., 3010 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3011 title="Retriever", 3012 ) 3013 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3014 None, 3015 description="Responsible for filtering fields to be added to json schema.", 3016 title="Schema Filter", 3017 ) 3018 schema_transformations: Optional[ 3019 List[ 3020 Union[ 3021 AddFields, 3022 RemoveFields, 3023 KeysToLower, 3024 KeysToSnakeCase, 3025 FlattenFields, 3026 DpathFlattenFields, 3027 KeysReplace, 3028 CustomTransformation, 3029 ] 3030 ] 3031 ] = Field( 3032 None, 3033 description="A list of transformations to be applied to the schema.", 3034 title="Schema Transformations", 3035 ) 3036 schema_type_identifier: SchemaTypeIdentifier 3037 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3038 3039 3040class ParentStreamConfig(BaseModel): 3041 type: Literal["ParentStreamConfig"] 3042 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3043 ..., description="Reference to the parent stream.", title="Parent Stream" 3044 ) 3045 parent_key: str = Field( 3046 ..., 3047 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3048 examples=["id", "{{ config['parent_record_id'] }}"], 3049 title="Parent Key", 3050 ) 3051 partition_field: str = Field( 3052 ..., 3053 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3054 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3055 title="Current Parent Key Value Identifier", 3056 ) 3057 request_option: Optional[RequestOption] = Field( 3058 None, 3059 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3060 title="Request Option", 3061 ) 3062 incremental_dependency: Optional[bool] = Field( 3063 False, 3064 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3065 title="Incremental Dependency", 3066 ) 3067 lazy_read_pointer: Optional[List[str]] = Field( 3068 [], 3069 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3070 title="Lazy Read Pointer", 3071 ) 3072 extra_fields: Optional[List[List[str]]] = Field( 3073 None, 3074 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3075 title="Extra Fields", 3076 ) 3077 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3078 3079 3080class PropertiesFromEndpoint(BaseModel): 3081 type: Literal["PropertiesFromEndpoint"] 3082 property_field_path: List[str] = Field( 3083 ..., 3084 description="Describes the path to the field that should be extracted", 3085 examples=[["name"]], 3086 ) 3087 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3088 ..., 3089 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3090 ) 3091 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3092 3093 3094class QueryProperties(BaseModel): 3095 type: Literal["QueryProperties"] 3096 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3097 ..., 3098 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3099 title="Property List", 3100 ) 3101 always_include_properties: Optional[List[str]] = Field( 3102 None, 3103 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3104 title="Always Include Properties", 3105 ) 3106 property_chunking: Optional[PropertyChunking] = Field( 3107 None, 3108 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3109 title="Property Chunking", 3110 ) 3111 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3112 None, 3113 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3114 title="Property Selector", 3115 ) 3116 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3117 3118 3119class StateDelegatingStream(BaseModel): 3120 type: Literal["StateDelegatingStream"] 3121 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3122 full_refresh_stream: DeclarativeStream = Field( 3123 ..., 3124 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3125 title="Full Refresh Stream", 3126 ) 3127 incremental_stream: DeclarativeStream = Field( 3128 ..., 3129 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3130 title="Incremental Stream", 3131 ) 3132 api_retention_period: Optional[str] = Field( 3133 None, 3134 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3135 examples=["P30D", "P90D", "P1Y"], 3136 title="API Retention Period", 3137 ) 3138 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3139 3140 3141class SimpleRetriever(BaseModel): 3142 type: Literal["SimpleRetriever"] 3143 requester: Union[HttpRequester, CustomRequester] = Field( 3144 ..., 3145 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3146 ) 3147 decoder: Optional[ 3148 Union[ 3149 JsonDecoder, 3150 JsonItemsDecoder, 3151 XmlDecoder, 3152 CsvDecoder, 3153 JsonlDecoder, 3154 GzipDecoder, 3155 IterableDecoder, 3156 ZipfileDecoder, 3157 CustomDecoder, 3158 ] 3159 ] = Field( 3160 None, 3161 description="Component decoding the response so records can be extracted.", 3162 title="HTTP Response Format", 3163 ) 3164 record_selector: RecordSelector = Field( 3165 ..., 3166 description="Component that describes how to extract records from a HTTP response.", 3167 ) 3168 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3169 None, 3170 description="Paginator component that describes how to navigate through the API's pages.", 3171 ) 3172 pagination_reset: Optional[PaginationReset] = Field( 3173 None, 3174 description="Describes what triggers pagination reset and how to handle it.", 3175 ) 3176 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3177 False, 3178 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3179 ) 3180 partition_router: Optional[ 3181 Union[ 3182 SubstreamPartitionRouter, 3183 ListPartitionRouter, 3184 GroupingPartitionRouter, 3185 CustomPartitionRouter, 3186 List[ 3187 Union[ 3188 SubstreamPartitionRouter, 3189 ListPartitionRouter, 3190 GroupingPartitionRouter, 3191 CustomPartitionRouter, 3192 ] 3193 ], 3194 ] 3195 ] = Field( 3196 None, 3197 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3198 title="Partition Router", 3199 ) 3200 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3201 3202 3203class AsyncRetriever(BaseModel): 3204 type: Literal["AsyncRetriever"] 3205 record_selector: RecordSelector = Field( 3206 ..., 3207 description="Component that describes how to extract records from a HTTP response.", 3208 ) 3209 status_mapping: AsyncJobStatusMap = Field( 3210 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3211 ) 3212 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3213 ..., description="Responsible for fetching the actual status of the async job." 3214 ) 3215 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3216 None, 3217 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3218 ) 3219 download_extractor: Optional[ 3220 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3221 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3222 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3223 ..., 3224 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3225 ) 3226 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3227 ..., 3228 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3229 ) 3230 polling_job_timeout: Optional[Union[int, str]] = Field( 3231 None, 3232 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3233 ) 3234 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3235 None, 3236 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3237 ge=1, 3238 ) 3239 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3240 None, 3241 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3242 ) 3243 download_requester: Union[HttpRequester, CustomRequester] = Field( 3244 ..., 3245 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3246 ) 3247 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3248 None, 3249 description="Paginator component that describes how to navigate through the API's pages during download.", 3250 ) 3251 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3252 None, 3253 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3254 ) 3255 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3256 None, 3257 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3258 ) 3259 partition_router: Optional[ 3260 Union[ 3261 ListPartitionRouter, 3262 SubstreamPartitionRouter, 3263 GroupingPartitionRouter, 3264 CustomPartitionRouter, 3265 List[ 3266 Union[ 3267 ListPartitionRouter, 3268 SubstreamPartitionRouter, 3269 GroupingPartitionRouter, 3270 CustomPartitionRouter, 3271 ] 3272 ], 3273 ] 3274 ] = Field( 3275 [], 3276 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3277 title="Partition Router", 3278 ) 3279 decoder: Optional[ 3280 Union[ 3281 CsvDecoder, 3282 GzipDecoder, 3283 JsonDecoder, 3284 JsonItemsDecoder, 3285 JsonlDecoder, 3286 IterableDecoder, 3287 XmlDecoder, 3288 ZipfileDecoder, 3289 CustomDecoder, 3290 ] 3291 ] = Field( 3292 None, 3293 description="Component decoding the response so records can be extracted.", 3294 title="HTTP Response Format", 3295 ) 3296 download_decoder: Optional[ 3297 Union[ 3298 CsvDecoder, 3299 GzipDecoder, 3300 JsonDecoder, 3301 JsonItemsDecoder, 3302 JsonlDecoder, 3303 IterableDecoder, 3304 XmlDecoder, 3305 ZipfileDecoder, 3306 CustomDecoder, 3307 ] 3308 ] = Field( 3309 None, 3310 description="Component decoding the download response so records can be extracted.", 3311 title="Download HTTP Response Format", 3312 ) 3313 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3314 3315 3316class BlockSimultaneousSyncsAction(BaseModel): 3317 type: Literal["BlockSimultaneousSyncsAction"] 3318 3319 3320class StreamGroup(BaseModel): 3321 streams: List[str] = Field( 3322 ..., 3323 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3324 title="Streams", 3325 ) 3326 action: BlockSimultaneousSyncsAction = Field( 3327 ..., 3328 description="The action to apply to streams in this group.", 3329 title="Action", 3330 ) 3331 3332 3333class SubstreamPartitionRouter(BaseModel): 3334 type: Literal["SubstreamPartitionRouter"] 3335 parent_stream_configs: List[ParentStreamConfig] = Field( 3336 ..., 3337 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3338 title="Parent Stream Configs", 3339 ) 3340 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3341 3342 3343class GroupingPartitionRouter(BaseModel): 3344 type: Literal["GroupingPartitionRouter"] 3345 group_size: int = Field( 3346 ..., 3347 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3348 examples=[10, 50], 3349 title="Group Size", 3350 ) 3351 underlying_partition_router: Union[ 3352 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3353 ] = Field( 3354 ..., 3355 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3356 title="Underlying Partition Router", 3357 ) 3358 deduplicate: Optional[bool] = Field( 3359 True, 3360 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3361 title="Deduplicate Partitions", 3362 ) 3363 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3364 3365 3366class HttpComponentsResolver(BaseModel): 3367 type: Literal["HttpComponentsResolver"] 3368 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3369 ..., 3370 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3371 title="Retriever", 3372 ) 3373 components_mapping: List[ComponentMappingDefinition] 3374 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") 3375 3376 3377class DynamicDeclarativeStream(BaseModel): 3378 type: Literal["DynamicDeclarativeStream"] 3379 name: Optional[str] = Field( 3380 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3381 ) 3382 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3383 ..., description="Reference to the stream template.", title="Stream Template" 3384 ) 3385 components_resolver: Union[ 3386 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3387 ] = Field( 3388 ..., 3389 description="Component resolve and populates stream templates with components values.", 3390 title="Components Resolver", 3391 ) 3392 use_parent_parameters: Optional[bool] = Field( 3393 True, 3394 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3395 title="Use Parent Parameters", 3396 ) 3397 3398 3399ComplexFieldType.update_forward_refs() 3400GzipDecoder.update_forward_refs() 3401CompositeErrorHandler.update_forward_refs() 3402DeclarativeSource1.update_forward_refs() 3403DeclarativeSource2.update_forward_refs() 3404SelectiveAuthenticator.update_forward_refs() 3405ConditionalStreams.update_forward_refs() 3406FileUploader.update_forward_refs() 3407DeclarativeStream.update_forward_refs() 3408SessionTokenAuthenticator.update_forward_refs() 3409HttpRequester.update_forward_refs() 3410DynamicSchemaLoader.update_forward_refs() 3411ParentStreamConfig.update_forward_refs() 3412PropertiesFromEndpoint.update_forward_refs() 3413SimpleRetriever.update_forward_refs() 3414AsyncRetriever.update_forward_refs()
class
AuthFlowType(enum.Enum):
oauth2_0 =
<AuthFlowType.oauth2_0: 'oauth2.0'>
oauth1_0 =
<AuthFlowType.oauth1_0: 'oauth1.0'>
class
ScopesJoinStrategy(enum.Enum):
space =
<ScopesJoinStrategy.space: 'space'>
comma =
<ScopesJoinStrategy.comma: 'comma'>
plus =
<ScopesJoinStrategy.plus: 'plus'>
class
BasicHttpAuthenticator(pydantic.v1.main.BaseModel):
28class BasicHttpAuthenticator(BaseModel): 29 type: Literal["BasicHttpAuthenticator"] 30 username: str = Field( 31 ..., 32 description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", 33 examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], 34 title="Username", 35 ) 36 password: Optional[str] = Field( 37 "", 38 description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", 39 examples=["{{ config['password'] }}", ""], 40 title="Password", 41 ) 42 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
BearerAuthenticator(pydantic.v1.main.BaseModel):
45class BearerAuthenticator(BaseModel): 46 type: Literal["BearerAuthenticator"] 47 api_token: str = Field( 48 ..., 49 description="Token to inject as request header for authenticating with the API.", 50 examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], 51 title="Bearer Token", 52 ) 53 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
DynamicStreamCheckConfig(pydantic.v1.main.BaseModel):
56class DynamicStreamCheckConfig(BaseModel): 57 type: Literal["DynamicStreamCheckConfig"] 58 dynamic_stream_name: str = Field( 59 ..., description="The dynamic stream name.", title="Dynamic Stream Name" 60 ) 61 stream_count: Optional[int] = Field( 62 None, 63 description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", 64 ge=1, 65 title="Stream Count", 66 )
class
CheckDynamicStream(pydantic.v1.main.BaseModel):
69class CheckDynamicStream(BaseModel): 70 type: Literal["CheckDynamicStream"] 71 stream_count: int = Field( 72 ..., 73 description="Numbers of the streams to try reading from when running a check operation.", 74 title="Stream Count", 75 ) 76 use_check_availability: Optional[bool] = Field( 77 True, 78 description="Enables stream check availability. This field is automatically set by the CDK.", 79 title="Use Check Availability", 80 )
class
ConcurrencyLevel(pydantic.v1.main.BaseModel):
83class ConcurrencyLevel(BaseModel): 84 type: Optional[Literal["ConcurrencyLevel"]] = None 85 default_concurrency: Union[int, str] = Field( 86 ..., 87 description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.", 88 examples=[10, "{{ config['num_workers'] or 10 }}"], 89 title="Default Concurrency", 90 ) 91 max_concurrency: Optional[int] = Field( 92 None, 93 description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.", 94 examples=[20, 100], 95 title="Max Concurrency", 96 ) 97 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConstantBackoffStrategy(pydantic.v1.main.BaseModel):
100class ConstantBackoffStrategy(BaseModel): 101 type: Literal["ConstantBackoffStrategy"] 102 backoff_time_in_seconds: Union[float, str] = Field( 103 ..., 104 description="Backoff time in seconds.", 105 examples=[30, 30.5, "{{ config['backoff_time'] }}"], 106 title="Backoff Time", 107 ) 108 jitter_range_in_seconds: Optional[float] = Field( 109 None, 110 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", 111 examples=[15], 112 ge=0, 113 title="Jitter Range", 114 ) 115 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CursorPagination(pydantic.v1.main.BaseModel):
118class CursorPagination(BaseModel): 119 type: Literal["CursorPagination"] 120 cursor_value: str = Field( 121 ..., 122 description="Value of the cursor defining the next page to fetch.", 123 examples=[ 124 "{{ headers.link.next.cursor }}", 125 "{{ last_record['key'] }}", 126 "{{ response['nextPage'] }}", 127 ], 128 title="Cursor Value", 129 ) 130 page_size: Optional[Union[int, str]] = Field( 131 None, 132 description="The number of records to include in each pages.", 133 examples=[100, "{{ config['page_size'] }}"], 134 title="Page Size", 135 ) 136 stop_condition: Optional[str] = Field( 137 None, 138 description="Template string evaluating when to stop paginating.", 139 examples=[ 140 "{{ response.data.has_more is false }}", 141 "{{ 'next' not in headers['link'] }}", 142 ], 143 title="Stop Condition", 144 ) 145 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator(pydantic.v1.main.BaseModel):
148class CustomAuthenticator(BaseModel): 149 class Config: 150 extra = Extra.allow 151 152 type: Literal["CustomAuthenticator"] 153 class_name: str = Field( 154 ..., 155 description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.", 156 examples=["source_railz.components.ShortLivedTokenAuthenticator"], 157 title="Class Name", 158 ) 159 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomAuthenticator.Config:
class
CustomBackoffStrategy(pydantic.v1.main.BaseModel):
162class CustomBackoffStrategy(BaseModel): 163 class Config: 164 extra = Extra.allow 165 166 type: Literal["CustomBackoffStrategy"] 167 class_name: str = Field( 168 ..., 169 description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.", 170 examples=["source_railz.components.MyCustomBackoffStrategy"], 171 title="Class Name", 172 ) 173 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomBackoffStrategy.Config:
class
CustomErrorHandler(pydantic.v1.main.BaseModel):
176class CustomErrorHandler(BaseModel): 177 class Config: 178 extra = Extra.allow 179 180 type: Literal["CustomErrorHandler"] 181 class_name: str = Field( 182 ..., 183 description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.", 184 examples=["source_railz.components.MyCustomErrorHandler"], 185 title="Class Name", 186 ) 187 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomErrorHandler.Config:
class
CustomPaginationStrategy(pydantic.v1.main.BaseModel):
190class CustomPaginationStrategy(BaseModel): 191 class Config: 192 extra = Extra.allow 193 194 type: Literal["CustomPaginationStrategy"] 195 class_name: str = Field( 196 ..., 197 description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.", 198 examples=["source_railz.components.MyCustomPaginationStrategy"], 199 title="Class Name", 200 ) 201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPaginationStrategy.Config:
class
CustomRecordExtractor(pydantic.v1.main.BaseModel):
204class CustomRecordExtractor(BaseModel): 205 class Config: 206 extra = Extra.allow 207 208 type: Literal["CustomRecordExtractor"] 209 class_name: str = Field( 210 ..., 211 description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.", 212 examples=["source_railz.components.MyCustomRecordExtractor"], 213 title="Class Name", 214 ) 215 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordExtractor.Config:
class
CustomRecordFilter(pydantic.v1.main.BaseModel):
218class CustomRecordFilter(BaseModel): 219 class Config: 220 extra = Extra.allow 221 222 type: Literal["CustomRecordFilter"] 223 class_name: str = Field( 224 ..., 225 description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.", 226 examples=["source_railz.components.MyCustomCustomRecordFilter"], 227 title="Class Name", 228 ) 229 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRecordFilter.Config:
class
CustomRequester(pydantic.v1.main.BaseModel):
232class CustomRequester(BaseModel): 233 class Config: 234 extra = Extra.allow 235 236 type: Literal["CustomRequester"] 237 class_name: str = Field( 238 ..., 239 description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.", 240 examples=["source_railz.components.MyCustomRecordExtractor"], 241 title="Class Name", 242 ) 243 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRequester.Config:
class
CustomRetriever(pydantic.v1.main.BaseModel):
246class CustomRetriever(BaseModel): 247 class Config: 248 extra = Extra.allow 249 250 type: Literal["CustomRetriever"] 251 class_name: str = Field( 252 ..., 253 description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.", 254 examples=["source_railz.components.MyCustomRetriever"], 255 title="Class Name", 256 ) 257 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomRetriever.Config:
class
CustomPartitionRouter(pydantic.v1.main.BaseModel):
260class CustomPartitionRouter(BaseModel): 261 class Config: 262 extra = Extra.allow 263 264 type: Literal["CustomPartitionRouter"] 265 class_name: str = Field( 266 ..., 267 description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.", 268 examples=["source_railz.components.MyCustomPartitionRouter"], 269 title="Class Name", 270 ) 271 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomPartitionRouter.Config:
class
CustomSchemaLoader(pydantic.v1.main.BaseModel):
274class CustomSchemaLoader(BaseModel): 275 class Config: 276 extra = Extra.allow 277 278 type: Literal["CustomSchemaLoader"] 279 class_name: str = Field( 280 ..., 281 description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.", 282 examples=["source_railz.components.MyCustomSchemaLoader"], 283 title="Class Name", 284 ) 285 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaLoader.Config:
class
CustomSchemaNormalization(pydantic.v1.main.BaseModel):
288class CustomSchemaNormalization(BaseModel): 289 class Config: 290 extra = Extra.allow 291 292 type: Literal["CustomSchemaNormalization"] 293 class_name: str = Field( 294 ..., 295 description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.", 296 examples=[ 297 "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer" 298 ], 299 title="Class Name", 300 ) 301 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomSchemaNormalization.Config:
class
CustomStateMigration(pydantic.v1.main.BaseModel):
304class CustomStateMigration(BaseModel): 305 class Config: 306 extra = Extra.allow 307 308 type: Literal["CustomStateMigration"] 309 class_name: str = Field( 310 ..., 311 description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.", 312 examples=["source_railz.components.MyCustomStateMigration"], 313 title="Class Name", 314 ) 315 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomStateMigration.Config:
class
CustomTransformation(pydantic.v1.main.BaseModel):
318class CustomTransformation(BaseModel): 319 class Config: 320 extra = Extra.allow 321 322 type: Literal["CustomTransformation"] 323 class_name: str = Field( 324 ..., 325 description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.", 326 examples=["source_railz.components.MyCustomTransformation"], 327 title="Class Name", 328 ) 329 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomTransformation.Config:
class
LegacyToPerPartitionStateMigration(pydantic.v1.main.BaseModel):
class
LegacyToPerPartitionStateMigration.Config:
class
Clamping(pydantic.v1.main.BaseModel):
class
Algorithm(enum.Enum):
349class Algorithm(Enum): 350 HS256 = "HS256" 351 HS384 = "HS384" 352 HS512 = "HS512" 353 ES256 = "ES256" 354 ES256K = "ES256K" 355 ES384 = "ES384" 356 ES512 = "ES512" 357 RS256 = "RS256" 358 RS384 = "RS384" 359 RS512 = "RS512" 360 PS256 = "PS256" 361 PS384 = "PS384" 362 PS512 = "PS512" 363 EdDSA = "EdDSA"
HS256 =
<Algorithm.HS256: 'HS256'>
HS384 =
<Algorithm.HS384: 'HS384'>
HS512 =
<Algorithm.HS512: 'HS512'>
ES256 =
<Algorithm.ES256: 'ES256'>
ES256K =
<Algorithm.ES256K: 'ES256K'>
ES384 =
<Algorithm.ES384: 'ES384'>
ES512 =
<Algorithm.ES512: 'ES512'>
RS256 =
<Algorithm.RS256: 'RS256'>
RS384 =
<Algorithm.RS384: 'RS384'>
RS512 =
<Algorithm.RS512: 'RS512'>
PS256 =
<Algorithm.PS256: 'PS256'>
PS384 =
<Algorithm.PS384: 'PS384'>
PS512 =
<Algorithm.PS512: 'PS512'>
EdDSA =
<Algorithm.EdDSA: 'EdDSA'>
class
JwtHeaders(pydantic.v1.main.BaseModel):
366class JwtHeaders(BaseModel): 367 class Config: 368 extra = Extra.forbid 369 370 kid: Optional[str] = Field( 371 None, 372 description="Private key ID for user account.", 373 examples=["{{ config['kid'] }}"], 374 title="Key Identifier", 375 ) 376 typ: Optional[str] = Field( 377 "JWT", 378 description="The media type of the complete JWT.", 379 examples=["JWT"], 380 title="Type", 381 ) 382 cty: Optional[str] = Field( 383 None, 384 description="Content type of JWT header.", 385 examples=["JWT"], 386 title="Content Type", 387 )
class
JwtHeaders.Config:
class
JwtPayload(pydantic.v1.main.BaseModel):
390class JwtPayload(BaseModel): 391 class Config: 392 extra = Extra.forbid 393 394 iss: Optional[str] = Field( 395 None, 396 description="The user/principal that issued the JWT. Commonly a value unique to the user.", 397 examples=["{{ config['iss'] }}"], 398 title="Issuer", 399 ) 400 sub: Optional[str] = Field( 401 None, 402 description="The subject of the JWT. Commonly defined by the API.", 403 title="Subject", 404 ) 405 aud: Optional[str] = Field( 406 None, 407 description="The recipient that the JWT is intended for. Commonly defined by the API.", 408 examples=["appstoreconnect-v1"], 409 title="Audience", 410 )
class
JwtPayload.Config:
class
RefreshTokenUpdater(pydantic.v1.main.BaseModel):
413class RefreshTokenUpdater(BaseModel): 414 refresh_token_name: Optional[str] = Field( 415 "refresh_token", 416 description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", 417 examples=["refresh_token"], 418 title="Refresh Token Property Name", 419 ) 420 access_token_config_path: Optional[List[str]] = Field( 421 ["credentials", "access_token"], 422 description="Config path to the access token. Make sure the field actually exists in the config.", 423 examples=[["credentials", "access_token"], ["access_token"]], 424 title="Config Path To Access Token", 425 ) 426 refresh_token_config_path: Optional[List[str]] = Field( 427 ["credentials", "refresh_token"], 428 description="Config path to the access token. Make sure the field actually exists in the config.", 429 examples=[["credentials", "refresh_token"], ["refresh_token"]], 430 title="Config Path To Refresh Token", 431 ) 432 token_expiry_date_config_path: Optional[List[str]] = Field( 433 ["credentials", "token_expiry_date"], 434 description="Config path to the expiry date. Make sure actually exists in the config.", 435 examples=[["credentials", "token_expiry_date"]], 436 title="Config Path To Expiry Date", 437 ) 438 refresh_token_error_status_codes: Optional[List[int]] = Field( 439 [], 440 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 441 examples=[[400, 500]], 442 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Status Codes", 443 ) 444 refresh_token_error_key: Optional[str] = Field( 445 "", 446 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 447 examples=["error"], 448 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Key", 449 ) 450 refresh_token_error_values: Optional[List[str]] = Field( 451 [], 452 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 453 examples=[["invalid_grant", "invalid_permissions"]], 454 title="(Deprecated - Use the same field on the OAuthAuthenticator level) Refresh Token Error Values", 455 )
class
Rate(pydantic.v1.main.BaseModel):
458class Rate(BaseModel): 459 class Config: 460 extra = Extra.allow 461 462 limit: Union[int, str] = Field( 463 ..., 464 description="The maximum number of calls allowed within the interval.", 465 title="Limit", 466 ) 467 interval: str = Field( 468 ..., 469 description="The time interval for the rate limit.", 470 examples=["PT1H", "P1D"], 471 title="Interval", 472 )
class
Rate.Config:
class
HttpRequestRegexMatcher(pydantic.v1.main.BaseModel):
475class HttpRequestRegexMatcher(BaseModel): 476 class Config: 477 extra = Extra.allow 478 479 method: Optional[str] = Field( 480 None, description="The HTTP method to match (e.g., GET, POST).", title="Method" 481 ) 482 url_base: Optional[str] = Field( 483 None, 484 description='The base URL (scheme and host, e.g. "https://api.example.com") to match.', 485 title="URL Base", 486 ) 487 url_path_pattern: Optional[str] = Field( 488 None, 489 description="A regular expression pattern to match the URL path.", 490 title="URL Path Pattern", 491 ) 492 params: Optional[Dict[str, Any]] = Field( 493 None, description="The query parameters to match.", title="Parameters" 494 ) 495 headers: Optional[Dict[str, Any]] = Field( 496 None, description="The headers to match.", title="Headers" 497 ) 498 weight: Optional[Union[int, str]] = Field( 499 None, 500 description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", 501 title="Weight", 502 )
class
HttpRequestRegexMatcher.Config:
class
ResponseToFileExtractor(pydantic.v1.main.BaseModel):
505class ResponseToFileExtractor(BaseModel): 506 type: Literal["ResponseToFileExtractor"] 507 preserve_na_values: Optional[bool] = Field( 508 False, 509 description='When enabled, string values such as "NA", "N/A", "NULL", "None" and "NaN" are kept as-is instead of being interpreted as missing and converted to null. Empty cells are still treated as null. Defaults to false to preserve historical behavior.', 510 title="Preserve NA Values", 511 ) 512 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
OnNoRecords(enum.Enum):
skip =
<OnNoRecords.skip: 'skip'>
emit_parent =
<OnNoRecords.emit_parent: 'emit_parent'>
class
ExponentialBackoffStrategy(pydantic.v1.main.BaseModel):
520class ExponentialBackoffStrategy(BaseModel): 521 type: Literal["ExponentialBackoffStrategy"] 522 factor: Optional[Union[float, str]] = Field( 523 5, 524 description="Multiplicative constant applied on each retry.", 525 examples=[5, 5.5, "10"], 526 title="Factor", 527 ) 528 jitter_range_in_seconds: Optional[float] = Field( 529 None, 530 description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", 531 examples=[2], 532 ge=0, 533 title="Jitter Range", 534 ) 535 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
GroupByKeyMergeStrategy(pydantic.v1.main.BaseModel):
538class GroupByKeyMergeStrategy(BaseModel): 539 type: Literal["GroupByKeyMergeStrategy"] 540 key: Union[str, List[str]] = Field( 541 ..., 542 description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.", 543 examples=["id", ["parent_id", "end_date"]], 544 title="Key", 545 ) 546 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SessionTokenRequestBearerAuthenticator(pydantic.v1.main.BaseModel):
class
HttpMethod(enum.Enum):
GET =
<HttpMethod.GET: 'GET'>
POST =
<HttpMethod.POST: 'POST'>
class
QuotaStatusSource(pydantic.v1.main.BaseModel):
558class QuotaStatusSource(BaseModel): 559 type: Literal["QuotaStatusSource"] 560 url: str = Field( 561 ..., 562 description="The full URL of the quota status endpoint.", 563 examples=[ 564 "https://api.github.com/rate_limit", 565 "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", 566 ], 567 title="URL", 568 ) 569 http_method: Optional[HttpMethod] = Field( 570 HttpMethod.GET, 571 description="The HTTP method used to fetch the quota status.", 572 title="HTTP Method", 573 ) 574 request_headers: Optional[Dict[str, str]] = Field( 575 None, 576 description="Additional headers to send with the quota status request.", 577 title="Request Headers", 578 ) 579 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
http_method: Optional[HttpMethod]
class
TokenQuota(pydantic.v1.main.BaseModel):
582class TokenQuota(BaseModel): 583 type: Literal["TokenQuota"] 584 name: str = Field( 585 ..., 586 description="Name of the quota pool.", 587 examples=["rest", "graphql"], 588 title="Name", 589 ) 590 remaining_path: List[str] = Field( 591 ..., 592 description="Path to the remaining call count for this pool in the quota status response.", 593 examples=[["resources", "core", "remaining"]], 594 title="Remaining Path", 595 ) 596 reset_path: List[str] = Field( 597 ..., 598 description="Path to the quota reset timestamp for this pool in the quota status response.", 599 examples=[["resources", "core", "reset"]], 600 title="Reset Path", 601 ) 602 limit_path: Optional[List[str]] = Field( 603 None, 604 description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", 605 examples=[["resources", "core", "limit"]], 606 title="Limit Path", 607 ) 608 matchers: Optional[List[HttpRequestRegexMatcher]] = Field( 609 None, 610 description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", 611 title="Matchers", 612 ) 613 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
matchers: Optional[List[HttpRequestRegexMatcher]]
class
RateLimitedMultipleTokenAuthenticator(pydantic.v1.main.BaseModel):
616class RateLimitedMultipleTokenAuthenticator(BaseModel): 617 type: Literal["RateLimitedMultipleTokenAuthenticator"] 618 tokens: Union[str, List[str]] = Field( 619 ..., 620 description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", 621 examples=[ 622 "{{ config['credentials']['personal_access_token'] }}", 623 ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], 624 ], 625 title="Tokens", 626 ) 627 token_delimiter: Optional[str] = Field( 628 ",", 629 description="Delimiter used to split a single token string into multiple tokens.", 630 title="Token Delimiter", 631 ) 632 auth_method: Optional[str] = Field( 633 "Bearer", 634 description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer <token>`).", 635 examples=["Bearer", "token"], 636 title="Auth Method", 637 ) 638 header: Optional[str] = Field( 639 "Authorization", 640 description="The name of the HTTP header in which to inject the token.", 641 title="Header Name", 642 ) 643 quota_status_source: QuotaStatusSource = Field( 644 ..., 645 description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", 646 title="Quota Status Source", 647 ) 648 quotas: List[TokenQuota] = Field( 649 ..., 650 description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", 651 min_items=1, 652 title="Quota Pools", 653 ) 654 max_wait_time: Optional[str] = Field( 655 "PT2H", 656 description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", 657 examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], 658 title="Maximum Wait Time", 659 ) 660 budget_reserve_fraction: Optional[float] = Field( 661 0.1, 662 description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", 663 title="Budget Reserve Fraction", 664 ) 665 budget_min_reserve: Optional[int] = Field( 666 50, 667 description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", 668 title="Budget Minimum Reserve", 669 ) 670 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
quota_status_source: QuotaStatusSource
quotas: List[TokenQuota]
class
Action(enum.Enum):
673class Action(Enum): 674 SUCCESS = "SUCCESS" 675 FAIL = "FAIL" 676 RETRY = "RETRY" 677 IGNORE = "IGNORE" 678 RESET_PAGINATION = "RESET_PAGINATION" 679 RATE_LIMITED = "RATE_LIMITED" 680 REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY"
SUCCESS =
<Action.SUCCESS: 'SUCCESS'>
FAIL =
<Action.FAIL: 'FAIL'>
RETRY =
<Action.RETRY: 'RETRY'>
IGNORE =
<Action.IGNORE: 'IGNORE'>
RESET_PAGINATION =
<Action.RESET_PAGINATION: 'RESET_PAGINATION'>
RATE_LIMITED =
<Action.RATE_LIMITED: 'RATE_LIMITED'>
REFRESH_TOKEN_THEN_RETRY =
<Action.REFRESH_TOKEN_THEN_RETRY: 'REFRESH_TOKEN_THEN_RETRY'>
class
FailureType(enum.Enum):
683class FailureType(Enum): 684 system_error = "system_error" 685 config_error = "config_error" 686 transient_error = "transient_error"
system_error =
<FailureType.system_error: 'system_error'>
config_error =
<FailureType.config_error: 'config_error'>
transient_error =
<FailureType.transient_error: 'transient_error'>
class
HttpResponseFilter(pydantic.v1.main.BaseModel):
689class HttpResponseFilter(BaseModel): 690 type: Literal["HttpResponseFilter"] 691 action: Optional[Action] = Field( 692 None, 693 description="Action to execute if a response matches the filter.", 694 examples=[ 695 "SUCCESS", 696 "FAIL", 697 "RETRY", 698 "IGNORE", 699 "RESET_PAGINATION", 700 "RATE_LIMITED", 701 "REFRESH_TOKEN_THEN_RETRY", 702 ], 703 title="Action", 704 ) 705 failure_type: Optional[FailureType] = Field( 706 None, 707 description="Failure type of traced exception if a response matches the filter.", 708 examples=["system_error", "config_error", "transient_error"], 709 title="Failure Type", 710 ) 711 error_message: Optional[str] = Field( 712 None, 713 description="Error Message to display if the response matches the filter.", 714 title="Error Message", 715 ) 716 error_message_contains: Optional[str] = Field( 717 None, 718 description="Match the response if its error message contains the substring.", 719 example=["This API operation is not enabled for this site"], 720 title="Error Message Substring", 721 ) 722 http_codes: Optional[List[int]] = Field( 723 None, 724 description="Match the response if its HTTP code is included in this list.", 725 examples=[[420, 429], [500]], 726 title="HTTP Codes", 727 unique_items=True, 728 ) 729 predicate: Optional[str] = Field( 730 None, 731 description="Match the response if the predicate evaluates to true.", 732 examples=[ 733 "{{ 'Too much requests' in response }}", 734 "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", 735 ], 736 title="Predicate", 737 ) 738 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
action: Optional[Action]
failure_type: Optional[FailureType]
class
ComplexFieldType(pydantic.v1.main.BaseModel):
741class ComplexFieldType(BaseModel): 742 field_type: str 743 items: Optional[Union[str, ComplexFieldType]] = None
items: Union[str, ComplexFieldType, NoneType]
class
TypesMap(pydantic.v1.main.BaseModel):
746class TypesMap(BaseModel): 747 target_type: Union[str, List[str], ComplexFieldType] 748 current_type: Union[str, List[str]] 749 condition: Optional[str] = None
target_type: Union[str, List[str], ComplexFieldType]
class
SchemaTypeIdentifier(pydantic.v1.main.BaseModel):
752class SchemaTypeIdentifier(BaseModel): 753 type: Optional[Literal["SchemaTypeIdentifier"]] = None 754 schema_pointer: Optional[List[str]] = Field( 755 [], 756 description="List of nested fields defining the schema field path to extract. Defaults to [].", 757 title="Schema Path", 758 ) 759 key_pointer: List[str] = Field( 760 ..., 761 description="List of potentially nested fields describing the full path of the field key to extract.", 762 title="Key Path", 763 ) 764 type_pointer: Optional[List[str]] = Field( 765 None, 766 description="List of potentially nested fields describing the full path of the field type to extract.", 767 title="Type Path", 768 ) 769 types_mapping: Optional[List[TypesMap]] = None 770 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
types_mapping: Optional[List[TypesMap]]
class
InlineSchemaLoader(pydantic.v1.main.BaseModel):
773class InlineSchemaLoader(BaseModel): 774 type: Literal["InlineSchemaLoader"] 775 schema_: Optional[Dict[str, Any]] = Field( 776 None, 777 alias="schema", 778 description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.', 779 title="Schema", 780 )
class
JsonFileSchemaLoader(pydantic.v1.main.BaseModel):
783class JsonFileSchemaLoader(BaseModel): 784 type: Literal["JsonFileSchemaLoader"] 785 file_path: Optional[str] = Field( 786 None, 787 description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", 788 example=["./schemas/users.json"], 789 title="File Path", 790 ) 791 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
JsonDecoder(pydantic.v1.main.BaseModel):
class
JsonItemsDecoder(pydantic.v1.main.BaseModel):
798class JsonItemsDecoder(BaseModel): 799 type: Literal["JsonItemsDecoder"] 800 items_path: str = Field( 801 ..., 802 description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", 803 title="Items Path", 804 ) 805 encoding: Optional[str] = Field( 806 "utf-8", 807 description="The character encoding of the JSON data. Defaults to UTF-8.", 808 title="Encoding", 809 )
class
JsonlDecoder(pydantic.v1.main.BaseModel):
class
KeysToLower(pydantic.v1.main.BaseModel):
class
KeysToSnakeCase(pydantic.v1.main.BaseModel):
class
FlattenFields(pydantic.v1.main.BaseModel):
826class FlattenFields(BaseModel): 827 type: Literal["FlattenFields"] 828 flatten_lists: Optional[bool] = Field( 829 True, 830 description="Whether to flatten lists or leave it as is. Default is True.", 831 title="Flatten Lists", 832 ) 833 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
KeyTransformation(pydantic.v1.main.BaseModel):
836class KeyTransformation(BaseModel): 837 type: Literal["KeyTransformation"] 838 prefix: Optional[str] = Field( 839 None, 840 description="Prefix to add for object keys. If not provided original keys remain unchanged.", 841 examples=["flattened_"], 842 title="Key Prefix", 843 ) 844 suffix: Optional[str] = Field( 845 None, 846 description="Suffix to add for object keys. If not provided original keys remain unchanged.", 847 examples=["_flattened"], 848 title="Key Suffix", 849 )
class
DpathFlattenFields(pydantic.v1.main.BaseModel):
852class DpathFlattenFields(BaseModel): 853 type: Literal["DpathFlattenFields"] 854 field_path: List[str] = Field( 855 ..., 856 description="A path to field that needs to be flattened.", 857 examples=[["data"], ["data", "*", "field"]], 858 title="Field Path", 859 ) 860 delete_origin_value: Optional[bool] = Field( 861 None, 862 description="Whether to delete the origin value or keep it. Default is False.", 863 title="Delete Origin Value", 864 ) 865 replace_record: Optional[bool] = Field( 866 None, 867 description="Whether to replace the origin record or not. Default is False.", 868 title="Replace Origin Record", 869 ) 870 key_transformation: Optional[KeyTransformation] = Field( 871 None, 872 description="Transformation for object keys. If not provided, original key will be used.", 873 title="Key transformation", 874 ) 875 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
key_transformation: Optional[KeyTransformation]
class
KeysReplace(pydantic.v1.main.BaseModel):
878class KeysReplace(BaseModel): 879 type: Literal["KeysReplace"] 880 old: str = Field( 881 ..., 882 description="Old value to replace.", 883 examples=[ 884 " ", 885 "{{ record.id }}", 886 "{{ config['id'] }}", 887 "{{ stream_slice['id'] }}", 888 ], 889 title="Old value", 890 ) 891 new: str = Field( 892 ..., 893 description="New value to set.", 894 examples=[ 895 "_", 896 "{{ record.id }}", 897 "{{ config['id'] }}", 898 "{{ stream_slice['id'] }}", 899 ], 900 title="New value", 901 ) 902 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
IterableDecoder(pydantic.v1.main.BaseModel):
class
XmlDecoder(pydantic.v1.main.BaseModel):
class
CustomDecoder(pydantic.v1.main.BaseModel):
913class CustomDecoder(BaseModel): 914 class Config: 915 extra = Extra.allow 916 917 type: Literal["CustomDecoder"] 918 class_name: str = Field( 919 ..., 920 description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.", 921 examples=["source_amazon_ads.components.GzipJsonlDecoder"], 922 title="Class Name", 923 ) 924 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
CustomDecoder.Config:
class
MinMaxDatetime(pydantic.v1.main.BaseModel):
927class MinMaxDatetime(BaseModel): 928 type: Literal["MinMaxDatetime"] 929 datetime: str = Field( 930 ..., 931 description="Datetime value.", 932 examples=[ 933 "2021-01-01", 934 "2021-01-01T00:00:00Z", 935 "{{ config['start_time'] }}", 936 "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}", 937 ], 938 title="Datetime", 939 ) 940 datetime_format: Optional[str] = Field( 941 "", 942 description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', 943 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], 944 title="Datetime Format", 945 ) 946 max_datetime: Optional[str] = Field( 947 None, 948 description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", 949 examples=["2021-01-01T00:00:00Z", "2021-01-01"], 950 title="Max Datetime", 951 ) 952 min_datetime: Optional[str] = Field( 953 None, 954 description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", 955 examples=["2010-01-01T00:00:00Z", "2010-01-01"], 956 title="Min Datetime", 957 ) 958 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
NoAuth(pydantic.v1.main.BaseModel):
class
NoPagination(pydantic.v1.main.BaseModel):
class
State(pydantic.v1.main.BaseModel):
class
State.Config:
class
OAuthScope(pydantic.v1.main.BaseModel):
class
OAuthScope.Config:
class
OauthConnectorInputSpecification(pydantic.v1.main.BaseModel):
988class OauthConnectorInputSpecification(BaseModel): 989 class Config: 990 extra = Extra.allow 991 992 consent_url: str = Field( 993 ..., 994 description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.", 995 examples=[ 996 "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}", 997 "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}", 998 ], 999 title="Consent URL", 1000 ) 1001 scope: Optional[str] = Field( 1002 None, 1003 description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.", 1004 examples=["user:read user:read_orders workspaces:read"], 1005 title="Scopes", 1006 ) 1007 # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the 1008 # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. 1009 # The CDK schema defines the manifest contract; the platform reads these fields 1010 # during the OAuth consent flow to build the authorization URL. 1011 scopes: Optional[List[OAuthScope]] = Field( 1012 None, 1013 description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", 1014 examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], 1015 title="Scopes", 1016 ) 1017 optional_scopes: Optional[List[OAuthScope]] = Field( 1018 None, 1019 description="Optional OAuth scope objects that may or may not be granted.", 1020 examples=[[{"scope": "admin:read"}]], 1021 title="Optional Scopes", 1022 ) 1023 scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( 1024 ScopesJoinStrategy.space, 1025 description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", 1026 title="Scopes Join Strategy", 1027 ) 1028 access_token_url: str = Field( 1029 ..., 1030 description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", 1031 examples=[ 1032 "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}" 1033 ], 1034 title="Access Token URL", 1035 ) 1036 access_token_headers: Optional[Dict[str, Any]] = Field( 1037 None, 1038 description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.", 1039 examples=[ 1040 { 1041 "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}" 1042 } 1043 ], 1044 title="Access Token Headers", 1045 ) 1046 access_token_params: Optional[Dict[str, Any]] = Field( 1047 None, 1048 description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.", 1049 examples=[ 1050 { 1051 "{{ auth_code_key }}": "{{ auth_code_value }}", 1052 "{{ client_id_key }}": "{{ client_id_value }}", 1053 "{{ client_secret_key }}": "{{ client_secret_value }}", 1054 } 1055 ], 1056 title="Access Token Query Params (Json Encoded)", 1057 ) 1058 extract_output: Optional[List[str]] = Field( 1059 None, 1060 description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.", 1061 examples=[["access_token", "refresh_token", "other_field"]], 1062 title="Extract Output", 1063 ) 1064 state: Optional[State] = Field( 1065 None, 1066 description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.", 1067 examples=[{"min": 7, "max": 128}], 1068 title="Configurable State Query Param", 1069 ) 1070 client_id_key: Optional[str] = Field( 1071 None, 1072 description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.", 1073 examples=["my_custom_client_id_key_name"], 1074 title="Client ID Key Override", 1075 ) 1076 client_secret_key: Optional[str] = Field( 1077 None, 1078 description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.", 1079 examples=["my_custom_client_secret_key_name"], 1080 title="Client Secret Key Override", 1081 ) 1082 scope_key: Optional[str] = Field( 1083 None, 1084 description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.", 1085 examples=["my_custom_scope_key_key_name"], 1086 title="Scopes Key Override", 1087 ) 1088 state_key: Optional[str] = Field( 1089 None, 1090 description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.", 1091 examples=["my_custom_state_key_key_name"], 1092 title="State Key Override", 1093 ) 1094 auth_code_key: Optional[str] = Field( 1095 None, 1096 description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.", 1097 examples=["my_custom_auth_code_key_name"], 1098 title="Auth Code Key Override", 1099 ) 1100 redirect_uri_key: Optional[str] = Field( 1101 None, 1102 description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.", 1103 examples=["my_custom_redirect_uri_key_name"], 1104 title="Redirect URI Key Override", 1105 )
scopes: Optional[List[OAuthScope]]
optional_scopes: Optional[List[OAuthScope]]
scopes_join_strategy: Optional[ScopesJoinStrategy]
state: Optional[State]
class
OauthConnectorInputSpecification.Config:
class
OAuthConfigSpecification(pydantic.v1.main.BaseModel):
1108class OAuthConfigSpecification(BaseModel): 1109 class Config: 1110 extra = Extra.allow 1111 1112 oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( 1113 None, 1114 description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", 1115 examples=[ 1116 {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, 1117 { 1118 "app_id": { 1119 "type": "string", 1120 "path_in_connector_config": ["info", "app_id"], 1121 } 1122 }, 1123 ], 1124 title="OAuth user input", 1125 ) 1126 oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( 1127 None, 1128 description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', 1129 title="DeclarativeOAuth Connector Specification", 1130 ) 1131 complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( 1132 None, 1133 description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", 1134 examples=[ 1135 { 1136 "refresh_token": { 1137 "type": "string,", 1138 "path_in_connector_config": ["credentials", "refresh_token"], 1139 } 1140 } 1141 ], 1142 title="OAuth output specification", 1143 ) 1144 complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( 1145 None, 1146 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", 1147 examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], 1148 title="OAuth input specification", 1149 ) 1150 complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( 1151 None, 1152 description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", 1153 examples=[ 1154 { 1155 "client_id": { 1156 "type": "string,", 1157 "path_in_connector_config": ["credentials", "client_id"], 1158 }, 1159 "client_secret": { 1160 "type": "string,", 1161 "path_in_connector_config": ["credentials", "client_secret"], 1162 }, 1163 } 1164 ], 1165 title="OAuth server output specification", 1166 )
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification]
class
OAuthConfigSpecification.Config:
class
OffsetIncrement(pydantic.v1.main.BaseModel):
1169class OffsetIncrement(BaseModel): 1170 type: Literal["OffsetIncrement"] 1171 page_size: Optional[Union[int, str]] = Field( 1172 None, 1173 description="The number of records to include in each pages.", 1174 examples=[100, "{{ config['page_size'] }}"], 1175 title="Limit", 1176 ) 1177 inject_on_first_request: Optional[bool] = Field( 1178 False, 1179 description="Using the `offset` with value `0` during the first request", 1180 title="Inject Offset on First Request", 1181 ) 1182 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PageIncrement(pydantic.v1.main.BaseModel):
1185class PageIncrement(BaseModel): 1186 type: Literal["PageIncrement"] 1187 page_size: Optional[Union[int, str]] = Field( 1188 None, 1189 description="The number of records to include in each pages.", 1190 examples=[100, "100", "{{ config['page_size'] }}"], 1191 title="Page Size", 1192 ) 1193 start_from_page: Optional[int] = Field( 1194 0, 1195 description="Index of the first page to request.", 1196 examples=[0, 1], 1197 title="Start From Page", 1198 ) 1199 inject_on_first_request: Optional[bool] = Field( 1200 False, 1201 description="Using the `page number` with value defined by `start_from_page` during the first request", 1202 title="Inject Page Number on First Request", 1203 ) 1204 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
PrimaryKey(pydantic.v1.main.BaseModel):
1207class PrimaryKey(BaseModel): 1208 __root__: Union[str, List[str], List[List[str]]] = Field( 1209 ..., 1210 description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", 1211 examples=["id", ["code", "type"]], 1212 title="Primary Key", 1213 )
class
PropertyLimitType(enum.Enum):
1216class PropertyLimitType(Enum): 1217 characters = "characters" 1218 property_count = "property_count"
characters =
<PropertyLimitType.characters: 'characters'>
property_count =
<PropertyLimitType.property_count: 'property_count'>
class
PropertyChunking(pydantic.v1.main.BaseModel):
1221class PropertyChunking(BaseModel): 1222 type: Literal["PropertyChunking"] 1223 property_limit_type: PropertyLimitType = Field( 1224 ..., 1225 description="The type used to determine the maximum number of properties per chunk", 1226 title="Property Limit Type", 1227 ) 1228 property_limit: Optional[int] = Field( 1229 None, 1230 description="The maximum amount of properties that can be retrieved per request according to the limit type.", 1231 title="Property Limit", 1232 ) 1233 record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field( 1234 None, 1235 description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination", 1236 title="Record Merge Strategy", 1237 ) 1238 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_limit_type: PropertyLimitType
record_merge_strategy: Optional[GroupByKeyMergeStrategy]
class
RecordFilter(pydantic.v1.main.BaseModel):
1241class RecordFilter(BaseModel): 1242 type: Literal["RecordFilter"] 1243 condition: Optional[str] = Field( 1244 "", 1245 description="The predicate to filter a record. Records will be removed if evaluated to False.", 1246 examples=[ 1247 "{{ record['created_at'] >= stream_interval['start_time'] }}", 1248 "{{ record.status in ['active', 'expired'] }}", 1249 ], 1250 title="Condition", 1251 ) 1252 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SchemaNormalization(enum.Enum):
Default =
<SchemaNormalization.Default: 'Default'>
None_ =
<SchemaNormalization.None_: 'None'>
class
RemoveFields(pydantic.v1.main.BaseModel):
1260class RemoveFields(BaseModel): 1261 type: Literal["RemoveFields"] 1262 condition: Optional[str] = Field( 1263 "", 1264 description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", 1265 examples=[ 1266 "{{ property|string == '' }}", 1267 "{{ property is integer }}", 1268 "{{ property|length > 5 }}", 1269 "{{ property == 'some_string_to_match' }}", 1270 ], 1271 ) 1272 field_pointers: List[List[str]] = Field( 1273 ..., 1274 description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", 1275 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1276 title="Field Paths", 1277 )
class
RequestPath(pydantic.v1.main.BaseModel):
class
InjectInto(enum.Enum):
1284class InjectInto(Enum): 1285 request_parameter = "request_parameter" 1286 header = "header" 1287 body_data = "body_data" 1288 body_json = "body_json"
request_parameter =
<InjectInto.request_parameter: 'request_parameter'>
header =
<InjectInto.header: 'header'>
body_data =
<InjectInto.body_data: 'body_data'>
body_json =
<InjectInto.body_json: 'body_json'>
class
RequestOption(pydantic.v1.main.BaseModel):
1291class RequestOption(BaseModel): 1292 type: Literal["RequestOption"] 1293 inject_into: InjectInto = Field( 1294 ..., 1295 description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", 1296 examples=["request_parameter", "header", "body_data", "body_json"], 1297 title="Inject Into", 1298 ) 1299 field_name: Optional[str] = Field( 1300 None, 1301 description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.", 1302 examples=["segment_id"], 1303 title="Field Name", 1304 ) 1305 field_path: Optional[List[str]] = Field( 1306 None, 1307 description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)", 1308 examples=[["data", "viewer", "id"]], 1309 title="Field Path", 1310 )
inject_into: InjectInto
class
Schemas(pydantic.v1.main.BaseModel):
class
Schemas.Config:
class
LegacySessionTokenAuthenticator(pydantic.v1.main.BaseModel):
1320class LegacySessionTokenAuthenticator(BaseModel): 1321 type: Literal["LegacySessionTokenAuthenticator"] 1322 header: str = Field( 1323 ..., 1324 description="The name of the session token header that will be injected in the request", 1325 examples=["X-Session"], 1326 title="Session Request Header", 1327 ) 1328 login_url: str = Field( 1329 ..., 1330 description="Path of the login URL (do not include the base URL)", 1331 examples=["session"], 1332 title="Login Path", 1333 ) 1334 session_token: Optional[str] = Field( 1335 None, 1336 description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", 1337 example=["{{ config['session_token'] }}"], 1338 title="Session Token", 1339 ) 1340 session_token_response_key: str = Field( 1341 ..., 1342 description="Name of the key of the session token to be extracted from the response", 1343 examples=["id"], 1344 title="Response Token Response Key", 1345 ) 1346 username: Optional[str] = Field( 1347 None, 1348 description="Username used to authenticate and obtain a session token", 1349 examples=[" {{ config['username'] }}"], 1350 title="Username", 1351 ) 1352 password: Optional[str] = Field( 1353 "", 1354 description="Password used to authenticate and obtain a session token", 1355 examples=["{{ config['password'] }}", ""], 1356 title="Password", 1357 ) 1358 validate_session_url: str = Field( 1359 ..., 1360 description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", 1361 examples=["user/current"], 1362 title="Validate Session Path", 1363 ) 1364 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
Action1(enum.Enum):
SPLIT_USING_CURSOR =
<Action1.SPLIT_USING_CURSOR: 'SPLIT_USING_CURSOR'>
RESET =
<Action1.RESET: 'RESET'>
class
PaginationResetLimits(pydantic.v1.main.BaseModel):
class
CsvDecoder(pydantic.v1.main.BaseModel):
class
AsyncJobStatusMap(pydantic.v1.main.BaseModel):
class
ValueType(enum.Enum):
1393class ValueType(Enum): 1394 string = "string" 1395 number = "number" 1396 integer = "integer" 1397 boolean = "boolean"
string =
<ValueType.string: 'string'>
number =
<ValueType.number: 'number'>
integer =
<ValueType.integer: 'integer'>
boolean =
<ValueType.boolean: 'boolean'>
class
WaitTimeFromHeader(pydantic.v1.main.BaseModel):
1400class WaitTimeFromHeader(BaseModel): 1401 type: Literal["WaitTimeFromHeader"] 1402 header: str = Field( 1403 ..., 1404 description="The name of the response header defining how long to wait before retrying.", 1405 examples=["Retry-After"], 1406 title="Response Header Name", 1407 ) 1408 regex: Optional[str] = Field( 1409 None, 1410 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1411 examples=["([-+]?\\d+)"], 1412 title="Extraction Regex", 1413 ) 1414 max_waiting_time_in_seconds: Optional[float] = Field( 1415 None, 1416 description="Given the value extracted from the header is greater than this value, stop the stream.", 1417 examples=[3600], 1418 title="Max Waiting Time in Seconds", 1419 ) 1420 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
WaitUntilTimeFromHeader(pydantic.v1.main.BaseModel):
1423class WaitUntilTimeFromHeader(BaseModel): 1424 type: Literal["WaitUntilTimeFromHeader"] 1425 header: str = Field( 1426 ..., 1427 description="The name of the response header defining how long to wait before retrying.", 1428 examples=["wait_time"], 1429 title="Response Header", 1430 ) 1431 min_wait: Optional[Union[float, str]] = Field( 1432 None, 1433 description="Minimum time to wait before retrying.", 1434 examples=[10, "60"], 1435 title="Minimum Wait Time", 1436 ) 1437 regex: Optional[str] = Field( 1438 None, 1439 description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", 1440 examples=["([-+]?\\d+)"], 1441 title="Extraction Regex", 1442 ) 1443 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ComponentMappingDefinition(pydantic.v1.main.BaseModel):
1446class ComponentMappingDefinition(BaseModel): 1447 type: Literal["ComponentMappingDefinition"] 1448 field_path: List[str] = Field( 1449 ..., 1450 description="A list of potentially nested fields indicating the full path where value will be added or updated.", 1451 examples=[ 1452 ["name"], 1453 ["retriever", "requester", "url"], 1454 ["retriever", "requester", "{{ components_values.field }}"], 1455 ["*", "**", "name"], 1456 ], 1457 title="Field Path", 1458 ) 1459 value: str = Field( 1460 ..., 1461 description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.", 1462 examples=[ 1463 "{{ components_values['updates'] }}", 1464 "{{ components_values['MetaData']['LastUpdatedTime'] }}", 1465 "{{ config['segment_id'] }}", 1466 "{{ stream_slice['parent_id'] }}", 1467 "{{ stream_slice['extra_fields']['name'] }}", 1468 ], 1469 title="Value", 1470 ) 1471 value_type: Optional[ValueType] = Field( 1472 None, 1473 description="The expected data type of the value. If omitted, the type will be inferred from the value provided.", 1474 title="Value Type", 1475 ) 1476 create_or_update: Optional[bool] = Field( 1477 False, 1478 description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.", 1479 title="Create or Update", 1480 ) 1481 condition: Optional[str] = Field( 1482 None, 1483 description="A condition that must be met for the mapping to be applied. This property is only supported for `ConfigComponentsResolver`.", 1484 examples=[ 1485 "{{ components_values.get('cursor_field', None) }}", 1486 "{{ '_incremental' in components_values.get('stream_name', '') }}", 1487 ], 1488 title="Condition", 1489 ) 1490 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
StreamConfig(pydantic.v1.main.BaseModel):
1493class StreamConfig(BaseModel): 1494 type: Literal["StreamConfig"] 1495 configs_pointer: List[str] = Field( 1496 ..., 1497 description="A list of potentially nested fields indicating the full path in source config file where streams configs located.", 1498 examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]], 1499 title="Configs Pointer", 1500 ) 1501 default_values: Optional[List[Dict[str, Any]]] = Field( 1502 None, 1503 description="A list of default values, each matching the structure expected from the parsed component value.", 1504 title="Default Values", 1505 ) 1506 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
ConfigComponentsResolver(pydantic.v1.main.BaseModel):
1509class ConfigComponentsResolver(BaseModel): 1510 type: Literal["ConfigComponentsResolver"] 1511 stream_config: Union[List[StreamConfig], StreamConfig] 1512 components_mapping: List[ComponentMappingDefinition] 1513 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_config: Union[List[StreamConfig], StreamConfig]
components_mapping: List[ComponentMappingDefinition]
class
StreamParametersDefinition(pydantic.v1.main.BaseModel):
1516class StreamParametersDefinition(BaseModel): 1517 type: Literal["StreamParametersDefinition"] 1518 list_of_parameters_for_stream: List[Dict[str, Any]] = Field( 1519 ..., 1520 description="A list of object of parameters for stream, each object in the list represents params for one stream.", 1521 examples=[ 1522 [ 1523 { 1524 "name": "test stream", 1525 "$parameters": {"entity": "test entity"}, 1526 "primary_key": "test key", 1527 } 1528 ] 1529 ], 1530 title="Stream Parameters", 1531 )
class
ParametrizedComponentsResolver(pydantic.v1.main.BaseModel):
1534class ParametrizedComponentsResolver(BaseModel): 1535 type: Literal["ParametrizedComponentsResolver"] 1536 stream_parameters: StreamParametersDefinition 1537 components_mapping: List[ComponentMappingDefinition] 1538 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream_parameters: StreamParametersDefinition
components_mapping: List[ComponentMappingDefinition]
class
RequestBodyPlainText(pydantic.v1.main.BaseModel):
class
RequestBodyUrlEncodedForm(pydantic.v1.main.BaseModel):
class
RequestBodyJsonObject(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery(pydantic.v1.main.BaseModel):
class
RequestBodyGraphQlQuery.Config:
class
ValidateAdheresToSchema(pydantic.v1.main.BaseModel):
1563class ValidateAdheresToSchema(BaseModel): 1564 type: Literal["ValidateAdheresToSchema"] 1565 base_schema: Union[str, Dict[str, Any]] = Field( 1566 ..., 1567 description="The base JSON schema against which the user-provided schema will be validated.", 1568 examples=[ 1569 "{{ config['report_validation_schema'] }}", 1570 '\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n', 1571 { 1572 "$schema": "http://json-schema.org/draft-07/schema#", 1573 "title": "Person", 1574 "type": "object", 1575 "properties": { 1576 "name": {"type": "string", "description": "The person's name"}, 1577 "age": { 1578 "type": "integer", 1579 "minimum": 0, 1580 "description": "The person's age", 1581 }, 1582 }, 1583 "required": ["name", "age"], 1584 }, 1585 ], 1586 title="Base JSON Schema", 1587 )
class
CustomValidationStrategy(pydantic.v1.main.BaseModel):
1590class CustomValidationStrategy(BaseModel): 1591 class Config: 1592 extra = Extra.allow 1593 1594 type: Literal["CustomValidationStrategy"] 1595 class_name: str = Field( 1596 ..., 1597 description="Fully-qualified name of the class that will be implementing the custom validation strategy. Has to be a sub class of ValidationStrategy. The format is `source_<name>.<package>.<class_name>`.", 1598 examples=["source_declarative_manifest.components.MyCustomValidationStrategy"], 1599 title="Class Name", 1600 )
class
CustomValidationStrategy.Config:
class
ConfigRemapField(pydantic.v1.main.BaseModel):
1603class ConfigRemapField(BaseModel): 1604 type: Literal["ConfigRemapField"] 1605 map: Union[Dict[str, Any], str] = Field( 1606 ..., 1607 description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.", 1608 examples=[ 1609 {"pending": "in_progress", "done": "completed", "cancelled": "terminated"}, 1610 "{{ config['status_mapping'] }}", 1611 ], 1612 title="Value Mapping", 1613 ) 1614 field_path: List[str] = Field( 1615 ..., 1616 description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.", 1617 examples=[ 1618 ["status"], 1619 ["data", "status"], 1620 ["data", "{{ config.name }}", "status"], 1621 ["data", "*", "status"], 1622 ], 1623 title="Field Path", 1624 )
class
ConfigRemoveFields(pydantic.v1.main.BaseModel):
1627class ConfigRemoveFields(BaseModel): 1628 type: Literal["ConfigRemoveFields"] 1629 field_pointers: List[List[str]] = Field( 1630 ..., 1631 description="A list of field pointers to be removed from the config.", 1632 examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], 1633 title="Field Pointers", 1634 ) 1635 condition: Optional[str] = Field( 1636 "", 1637 description="Fields will be removed if expression is evaluated to True.", 1638 examples=[ 1639 "{{ config['environemnt'] == 'sandbox' }}", 1640 "{{ property is integer }}", 1641 "{{ property|length > 5 }}", 1642 "{{ property == 'some_string_to_match' }}", 1643 ], 1644 )
class
CustomConfigTransformation(pydantic.v1.main.BaseModel):
1647class CustomConfigTransformation(BaseModel): 1648 type: Literal["CustomConfigTransformation"] 1649 class_name: str = Field( 1650 ..., 1651 description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", 1652 examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], 1653 ) 1654 parameters: Optional[Dict[str, Any]] = Field( 1655 None, 1656 alias="$parameters", 1657 description="Additional parameters to be passed to the custom config transformation.", 1658 )
class
AddedFieldDefinition(pydantic.v1.main.BaseModel):
1661class AddedFieldDefinition(BaseModel): 1662 type: Literal["AddedFieldDefinition"] 1663 path: List[str] = Field( 1664 ..., 1665 description="List of strings defining the path where to add the value on the record.", 1666 examples=[["segment_id"], ["metadata", "segment_id"]], 1667 title="Path", 1668 ) 1669 value: str = Field( 1670 ..., 1671 description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", 1672 examples=[ 1673 "{{ record['updates'] }}", 1674 "{{ record['MetaData']['LastUpdatedTime'] }}", 1675 "{{ stream_partition['segment_id'] }}", 1676 ], 1677 title="Value", 1678 ) 1679 value_type: Optional[ValueType] = Field( 1680 None, 1681 description="Type of the value. If not specified, the type will be inferred from the value.", 1682 title="Value Type", 1683 ) 1684 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
value_type: Optional[ValueType]
class
AddFields(pydantic.v1.main.BaseModel):
1687class AddFields(BaseModel): 1688 type: Literal["AddFields"] 1689 fields: List[AddedFieldDefinition] = Field( 1690 ..., 1691 description="List of transformations (path and corresponding value) that will be added to the record.", 1692 title="Fields", 1693 ) 1694 condition: Optional[str] = Field( 1695 "", 1696 description="Fields will be added if expression is evaluated to True.", 1697 examples=[ 1698 "{{ property|string == '' }}", 1699 "{{ property is integer }}", 1700 "{{ property|length > 5 }}", 1701 "{{ property == 'some_string_to_match' }}", 1702 ], 1703 ) 1704 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
fields: List[AddedFieldDefinition]
class
ApiKeyAuthenticator(pydantic.v1.main.BaseModel):
1707class ApiKeyAuthenticator(BaseModel): 1708 type: Literal["ApiKeyAuthenticator"] 1709 api_token: Optional[str] = Field( 1710 None, 1711 description="The API key to inject in the request. Fill it in the user inputs.", 1712 examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], 1713 title="API Key", 1714 ) 1715 header: Optional[str] = Field( 1716 None, 1717 description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", 1718 examples=["Authorization", "Api-Token", "X-Auth-Token"], 1719 title="Header Name", 1720 ) 1721 inject_into: Optional[RequestOption] = Field( 1722 None, 1723 description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", 1724 examples=[ 1725 {"inject_into": "header", "field_name": "Authorization"}, 1726 {"inject_into": "request_parameter", "field_name": "authKey"}, 1727 ], 1728 title="Inject API Key Into Outgoing HTTP Request", 1729 ) 1730 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
inject_into: Optional[RequestOption]
class
AuthFlow(pydantic.v1.main.BaseModel):
1733class AuthFlow(BaseModel): 1734 auth_flow_type: Optional[AuthFlowType] = Field( 1735 None, description="The type of auth to use", title="Auth flow type" 1736 ) 1737 predicate_key: Optional[List[str]] = Field( 1738 None, 1739 description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", 1740 examples=[["credentials", "auth_type"]], 1741 title="Predicate key", 1742 ) 1743 predicate_value: Optional[str] = Field( 1744 None, 1745 description="Value of the predicate_key fields for the advanced auth to be applicable.", 1746 examples=["Oauth"], 1747 title="Predicate value", 1748 ) 1749 oauth_config_specification: Optional[OAuthConfigSpecification] = None
auth_flow_type: Optional[AuthFlowType]
oauth_config_specification: Optional[OAuthConfigSpecification]
class
CheckStream(pydantic.v1.main.BaseModel):
1752class CheckStream(BaseModel): 1753 type: Literal["CheckStream"] 1754 stream_names: Optional[List[str]] = Field( 1755 None, 1756 description="Names of the streams to try reading from when running a check operation.", 1757 examples=[["users"], ["users", "contacts"]], 1758 title="Stream Names", 1759 ) 1760 dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]]
class
IncrementingCountCursor(pydantic.v1.main.BaseModel):
1763class IncrementingCountCursor(BaseModel): 1764 type: Literal["IncrementingCountCursor"] 1765 cursor_field: str = Field( 1766 ..., 1767 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1768 examples=["created_at", "{{ config['record_cursor'] }}"], 1769 title="Cursor Field", 1770 ) 1771 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1772 None, 1773 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1774 title="Allow Catalog Defined Cursor Field", 1775 ) 1776 start_value: Optional[Union[str, int]] = Field( 1777 None, 1778 description="The value that determines the earliest record that should be synced.", 1779 examples=[0, "{{ config['start_value'] }}"], 1780 title="Start Value", 1781 ) 1782 start_value_option: Optional[RequestOption] = Field( 1783 None, 1784 description="Optionally configures how the start value will be sent in requests to the source API.", 1785 title="Inject Start Value Into Outgoing HTTP Request", 1786 ) 1787 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
start_value_option: Optional[RequestOption]
class
DatetimeBasedCursor(pydantic.v1.main.BaseModel):
1790class DatetimeBasedCursor(BaseModel): 1791 type: Literal["DatetimeBasedCursor"] 1792 clamping: Optional[Clamping] = Field( 1793 None, 1794 description="This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)", 1795 title="Date Range Clamping", 1796 ) 1797 cursor_field: str = Field( 1798 ..., 1799 description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", 1800 examples=["created_at", "{{ config['record_cursor'] }}"], 1801 title="Cursor Field", 1802 ) 1803 allow_catalog_defined_cursor_field: Optional[bool] = Field( 1804 None, 1805 description="Whether the cursor allows users to override the default cursor_field when configuring their connection. The user defined cursor field will be specified from within the configured catalog.", 1806 title="Allow Catalog Defined Cursor Field", 1807 ) 1808 cursor_datetime_formats: Optional[List[str]] = Field( 1809 None, 1810 description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the Outgoing Datetime Format will be used.\nUse placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1811 examples=[ 1812 "%Y-%m-%d", 1813 "%Y-%m-%d %H:%M:%S", 1814 "%Y-%m-%dT%H:%M:%S", 1815 "%Y-%m-%dT%H:%M:%SZ", 1816 "%Y-%m-%dT%H:%M:%S%z", 1817 "%Y-%m-%dT%H:%M:%S.%fZ", 1818 "%Y-%m-%dT%H:%M:%S.%f%z", 1819 "%Y-%m-%d %H:%M:%S.%f+00:00", 1820 "%s", 1821 "%ms", 1822 ], 1823 title="Cursor Datetime Formats", 1824 ) 1825 start_datetime: Union[MinMaxDatetime, str] = Field( 1826 ..., 1827 description="The datetime that determines the earliest record that should be synced.", 1828 examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], 1829 title="Start Datetime", 1830 ) 1831 start_time_option: Optional[RequestOption] = Field( 1832 None, 1833 description="Optionally configures how the start datetime will be sent in requests to the source API.", 1834 title="Inject Start Time Into Outgoing HTTP Request", 1835 ) 1836 end_datetime: Optional[Union[MinMaxDatetime, str]] = Field( 1837 None, 1838 description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", 1839 examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], 1840 title="End Datetime", 1841 ) 1842 end_time_option: Optional[RequestOption] = Field( 1843 None, 1844 description="Optionally configures how the end datetime will be sent in requests to the source API.", 1845 title="Inject End Time Into Outgoing HTTP Request", 1846 ) 1847 datetime_format: str = Field( 1848 ..., 1849 description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", 1850 examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], 1851 title="Outgoing Datetime Format", 1852 ) 1853 cursor_granularity: Optional[str] = Field( 1854 None, 1855 description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should\nbe P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.\n * **PT0.000001S**: 1 microsecond\n * **PT0.001S**: 1 millisecond\n * **PT1S**: 1 second\n * **PT1M**: 1 minute\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n", 1856 examples=["PT1S"], 1857 title="Cursor Granularity", 1858 ) 1859 is_data_feed: Optional[bool] = Field( 1860 None, 1861 description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", 1862 title="Data Feed API", 1863 ) 1864 is_client_side_incremental: Optional[bool] = Field( 1865 None, 1866 description="Set to True if the target API endpoint does not take cursor values to filter records and returns all records anyway. This will cause the connector to filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", 1867 title="Client-side Incremental Filtering", 1868 ) 1869 is_compare_strictly: Optional[bool] = Field( 1870 False, 1871 description="Set to True if the target API does not accept queries where the start time equal the end time. This will cause those requests to be skipped.", 1872 title="Strict Start-End Time Comparison", 1873 ) 1874 global_substream_cursor: Optional[bool] = Field( 1875 False, 1876 description="Setting to True causes the connector to store the cursor as one value, instead of per-partition. This setting optimizes performance when the parent stream has thousands of partitions. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).", 1877 title="Global Substream Cursor", 1878 ) 1879 lookback_window: Optional[str] = Field( 1880 None, 1881 description="Time interval (ISO8601 duration) before the start_datetime to read data for, e.g. P1M for looking back one month.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1882 examples=["P1D", "P{{ config['lookback_days'] }}D"], 1883 title="Lookback Window", 1884 ) 1885 partition_field_end: Optional[str] = Field( 1886 None, 1887 description="Name of the partition start time field.", 1888 examples=["ending_time"], 1889 title="Partition Field End", 1890 ) 1891 partition_field_start: Optional[str] = Field( 1892 None, 1893 description="Name of the partition end time field.", 1894 examples=["starting_time"], 1895 title="Partition Field Start", 1896 ) 1897 step: Optional[str] = Field( 1898 None, 1899 description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 1900 examples=["P1W", "{{ config['step_increment'] }}"], 1901 title="Step", 1902 ) 1903 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
clamping: Optional[Clamping]
start_datetime: Union[MinMaxDatetime, str]
start_time_option: Optional[RequestOption]
end_datetime: Union[MinMaxDatetime, str, NoneType]
end_time_option: Optional[RequestOption]
class
JwtAuthenticator(pydantic.v1.main.BaseModel):
1906class JwtAuthenticator(BaseModel): 1907 type: Literal["JwtAuthenticator"] 1908 secret_key: str = Field( 1909 ..., 1910 description="Secret used to sign the JSON web token.", 1911 examples=["{{ config['secret_key'] }}"], 1912 title="Secret Key", 1913 ) 1914 base64_encode_secret_key: Optional[bool] = Field( 1915 False, 1916 description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', 1917 title="Base64-encode Secret Key", 1918 ) 1919 algorithm: Algorithm = Field( 1920 ..., 1921 description="Algorithm used to sign the JSON web token.", 1922 examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], 1923 title="Algorithm", 1924 ) 1925 token_duration: Optional[int] = Field( 1926 1200, 1927 description="The amount of time in seconds a JWT token can be valid after being issued.", 1928 examples=[1200, 3600], 1929 title="Token Duration", 1930 ) 1931 header_prefix: Optional[str] = Field( 1932 None, 1933 description="The prefix to be used within the Authentication header.", 1934 examples=["Bearer", "Basic"], 1935 title="Header Prefix", 1936 ) 1937 jwt_headers: Optional[JwtHeaders] = Field( 1938 None, 1939 description="JWT headers used when signing JSON web token.", 1940 title="JWT Headers", 1941 ) 1942 additional_jwt_headers: Optional[Dict[str, Any]] = Field( 1943 None, 1944 description="Additional headers to be included with the JWT headers object.", 1945 title="Additional JWT Headers", 1946 ) 1947 jwt_payload: Optional[JwtPayload] = Field( 1948 None, 1949 description="JWT Payload used when signing JSON web token.", 1950 title="JWT Payload", 1951 ) 1952 additional_jwt_payload: Optional[Dict[str, Any]] = Field( 1953 None, 1954 description="Additional properties to be added to the JWT payload.", 1955 title="Additional JWT Payload Properties", 1956 ) 1957 passphrase: Optional[str] = Field( 1958 None, 1959 description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.", 1960 examples=["{{ config['passphrase'] }}"], 1961 title="Passphrase", 1962 ) 1963 request_option: Optional[RequestOption] = Field( 1964 None, 1965 description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.", 1966 title="Request Option", 1967 ) 1968 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
algorithm: Algorithm
jwt_headers: Optional[JwtHeaders]
jwt_payload: Optional[JwtPayload]
request_option: Optional[RequestOption]
class
OAuthAuthenticator(pydantic.v1.main.BaseModel):
1971class OAuthAuthenticator(BaseModel): 1972 type: Literal["OAuthAuthenticator"] 1973 client_id_name: Optional[str] = Field( 1974 "client_id", 1975 description="The name of the property to use to refresh the `access_token`.", 1976 examples=["custom_app_id"], 1977 title="Client ID Property Name", 1978 ) 1979 client_id: Optional[str] = Field( 1980 None, 1981 description="The OAuth client ID. Fill it in the user inputs.", 1982 examples=[ 1983 "{{ config['client_id'] }}", 1984 "{{ config['credentials']['client_id }}", 1985 ], 1986 title="Client ID", 1987 ) 1988 client_secret_name: Optional[str] = Field( 1989 "client_secret", 1990 description="The name of the property to use to refresh the `access_token`.", 1991 examples=["custom_app_secret"], 1992 title="Client Secret Property Name", 1993 ) 1994 client_secret: Optional[str] = Field( 1995 None, 1996 description="The OAuth client secret. Fill it in the user inputs.", 1997 examples=[ 1998 "{{ config['client_secret'] }}", 1999 "{{ config['credentials']['client_secret }}", 2000 ], 2001 title="Client Secret", 2002 ) 2003 refresh_token_name: Optional[str] = Field( 2004 "refresh_token", 2005 description="The name of the property to use to refresh the `access_token`.", 2006 examples=["custom_app_refresh_value"], 2007 title="Refresh Token Property Name", 2008 ) 2009 refresh_token: Optional[str] = Field( 2010 None, 2011 description="Credential artifact used to get a new access token.", 2012 examples=[ 2013 "{{ config['refresh_token'] }}", 2014 "{{ config['credentials]['refresh_token'] }}", 2015 ], 2016 title="Refresh Token", 2017 ) 2018 token_refresh_endpoint: Optional[str] = Field( 2019 None, 2020 description="The full URL to call to obtain a new access token.", 2021 examples=["https://connect.squareup.com/oauth2/token"], 2022 title="Token Refresh Endpoint", 2023 ) 2024 access_token_name: Optional[str] = Field( 2025 "access_token", 2026 description="The name of the property which contains the access token in the response from the token refresh endpoint.", 2027 examples=["access_token"], 2028 title="Access Token Property Name", 2029 ) 2030 access_token_value: Optional[str] = Field( 2031 None, 2032 description="The value of the access_token to bypass the token refreshing using `refresh_token`.", 2033 examples=["secret_access_token_value"], 2034 title="Access Token Value", 2035 ) 2036 expires_in_name: Optional[str] = Field( 2037 "expires_in", 2038 description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", 2039 examples=["expires_in"], 2040 title="Token Expiry Property Name", 2041 ) 2042 grant_type_name: Optional[str] = Field( 2043 "grant_type", 2044 description="The name of the property to use to refresh the `access_token`.", 2045 examples=["custom_grant_type"], 2046 title="Grant Type Property Name", 2047 ) 2048 grant_type: Optional[str] = Field( 2049 "refresh_token", 2050 description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", 2051 examples=["refresh_token", "client_credentials"], 2052 title="Grant Type", 2053 ) 2054 refresh_request_body: Optional[Dict[str, Any]] = Field( 2055 None, 2056 description="Body of the request sent to get a new access token.", 2057 examples=[ 2058 { 2059 "applicationId": "{{ config['application_id'] }}", 2060 "applicationSecret": "{{ config['application_secret'] }}", 2061 "token": "{{ config['token'] }}", 2062 } 2063 ], 2064 title="Refresh Request Body", 2065 ) 2066 refresh_request_headers: Optional[Dict[str, Any]] = Field( 2067 None, 2068 description="Headers of the request sent to get a new access token.", 2069 examples=[ 2070 { 2071 "Authorization": "<AUTH_TOKEN>", 2072 "Content-Type": "application/x-www-form-urlencoded", 2073 } 2074 ], 2075 title="Refresh Request Headers", 2076 ) 2077 send_refresh_request_as_query_params: Optional[bool] = Field( 2078 False, 2079 description="When set to true, the standard OAuth refresh args (`grant_type`, `refresh_token`, client credentials when not in an `Authorization` header, scopes, plus any `refresh_request_body` extras) are sent on the URL query string and the request body is emitted empty. Use this for OAuth providers like Gong that document their refresh endpoint with refresh args on the URL query string.", 2080 examples=[True], 2081 title="Send Refresh Request As Query Params", 2082 ) 2083 scopes: Optional[List[str]] = Field( 2084 None, 2085 description="List of scopes that should be granted to the access token.", 2086 examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], 2087 title="Scopes", 2088 ) 2089 token_expiry_date: Optional[str] = Field( 2090 None, 2091 description="The access token expiry date.", 2092 examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], 2093 title="Token Expiry Date", 2094 ) 2095 token_expiry_date_format: Optional[str] = Field( 2096 None, 2097 description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", 2098 examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], 2099 title="Token Expiry Date Format", 2100 ) 2101 refresh_token_error_status_codes: Optional[List[int]] = Field( 2102 None, 2103 description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", 2104 examples=[[400, 500]], 2105 title="Refresh Token Error Status Codes", 2106 ) 2107 refresh_token_error_key: Optional[str] = Field( 2108 None, 2109 description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", 2110 examples=["error"], 2111 title="Refresh Token Error Key", 2112 ) 2113 refresh_token_error_values: Optional[List[str]] = Field( 2114 None, 2115 description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', 2116 examples=[["invalid_grant", "invalid_permissions"]], 2117 title="Refresh Token Error Values", 2118 ) 2119 refresh_token_updater: Optional[RefreshTokenUpdater] = Field( 2120 None, 2121 description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", 2122 title="Refresh Token Updater", 2123 ) 2124 profile_assertion: Optional[JwtAuthenticator] = Field( 2125 None, 2126 description="The authenticator being used to authenticate the client authenticator.", 2127 title="Profile Assertion", 2128 ) 2129 use_profile_assertion: Optional[bool] = Field( 2130 False, 2131 description="Enable using profile assertion as a flow for OAuth authorization.", 2132 title="Use Profile Assertion", 2133 ) 2134 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
refresh_token_updater: Optional[RefreshTokenUpdater]
profile_assertion: Optional[JwtAuthenticator]
class
FixedWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2137class FixedWindowCallRatePolicy(BaseModel): 2138 class Config: 2139 extra = Extra.allow 2140 2141 type: Literal["FixedWindowCallRatePolicy"] 2142 period: str = Field( 2143 ..., description="The time interval for the rate limit window.", title="Period" 2144 ) 2145 call_limit: int = Field( 2146 ..., 2147 description="The maximum number of calls allowed within the period.", 2148 title="Call Limit", 2149 ) 2150 matchers: List[HttpRequestRegexMatcher] = Field( 2151 ..., 2152 description="List of matchers that define which requests this policy applies to.", 2153 title="Matchers", 2154 )
matchers: List[HttpRequestRegexMatcher]
class
FixedWindowCallRatePolicy.Config:
class
MovingWindowCallRatePolicy(pydantic.v1.main.BaseModel):
2157class MovingWindowCallRatePolicy(BaseModel): 2158 class Config: 2159 extra = Extra.allow 2160 2161 type: Literal["MovingWindowCallRatePolicy"] 2162 rates: List[Rate] = Field( 2163 ..., 2164 description="List of rates that define the call limits for different time intervals.", 2165 title="Rates", 2166 ) 2167 matchers: List[HttpRequestRegexMatcher] = Field( 2168 ..., 2169 description="List of matchers that define which requests this policy applies to.", 2170 title="Matchers", 2171 )
rates: List[Rate]
matchers: List[HttpRequestRegexMatcher]
class
MovingWindowCallRatePolicy.Config:
class
UnlimitedCallRatePolicy(pydantic.v1.main.BaseModel):
2174class UnlimitedCallRatePolicy(BaseModel): 2175 class Config: 2176 extra = Extra.allow 2177 2178 type: Literal["UnlimitedCallRatePolicy"] 2179 matchers: List[HttpRequestRegexMatcher] = Field( 2180 ..., 2181 description="List of matchers that define which requests this policy applies to.", 2182 title="Matchers", 2183 )
matchers: List[HttpRequestRegexMatcher]
class
UnlimitedCallRatePolicy.Config:
class
DefaultErrorHandler(pydantic.v1.main.BaseModel):
2186class DefaultErrorHandler(BaseModel): 2187 type: Literal["DefaultErrorHandler"] 2188 backoff_strategies: Optional[ 2189 List[ 2190 Union[ 2191 ConstantBackoffStrategy, 2192 ExponentialBackoffStrategy, 2193 WaitTimeFromHeader, 2194 WaitUntilTimeFromHeader, 2195 CustomBackoffStrategy, 2196 ] 2197 ] 2198 ] = Field( 2199 None, 2200 description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", 2201 title="Backoff Strategies", 2202 ) 2203 max_retries: Optional[Union[int, str]] = Field( 2204 5, 2205 description="The maximum number of times to retry a retryable request before giving up and failing. Can be a hardcoded integer or a string interpolated from the connector config.", 2206 examples=[5, 0, 10, "{{ config['max_retries_on_throttle'] }}"], 2207 title="Max Retry Count", 2208 ) 2209 response_filters: Optional[List[HttpResponseFilter]] = Field( 2210 None, 2211 description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", 2212 title="Response Filters", 2213 ) 2214 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
backoff_strategies: Optional[List[Union[ConstantBackoffStrategy, ExponentialBackoffStrategy, WaitTimeFromHeader, WaitUntilTimeFromHeader, CustomBackoffStrategy]]]
response_filters: Optional[List[HttpResponseFilter]]
class
DefaultPaginator(pydantic.v1.main.BaseModel):
2217class DefaultPaginator(BaseModel): 2218 type: Literal["DefaultPaginator"] 2219 pagination_strategy: Union[ 2220 PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy 2221 ] = Field( 2222 ..., 2223 description="Strategy defining how records are paginated.", 2224 title="Pagination Strategy", 2225 ) 2226 page_size_option: Optional[RequestOption] = Field( 2227 None, title="Inject Page Size Into Outgoing HTTP Request" 2228 ) 2229 page_token_option: Optional[Union[RequestOption, RequestPath]] = Field( 2230 None, 2231 description="Inject the page token into the outgoing HTTP requests by inserting it into either the request URL path or a field on the request.", 2232 title="Inject Page Token Into Outgoing HTTP Request", 2233 ) 2234 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
pagination_strategy: Union[PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy]
page_size_option: Optional[RequestOption]
page_token_option: Union[RequestOption, RequestPath, NoneType]
class
RecordExpander(pydantic.v1.main.BaseModel):
2237class RecordExpander(BaseModel): 2238 type: Literal["RecordExpander"] 2239 expand_records_from_field: List[str] = Field( 2240 ..., 2241 description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", 2242 examples=[ 2243 ["lines", "data"], 2244 ["items"], 2245 ["nested", "array"], 2246 ["sections", "*", "items"], 2247 ], 2248 title="Expand Records From Field", 2249 ) 2250 remain_original_record: Optional[bool] = Field( 2251 False, 2252 description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', 2253 title="Remain Original Record", 2254 ) 2255 on_no_records: Optional[OnNoRecords] = Field( 2256 OnNoRecords.skip, 2257 description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', 2258 title="On No Records", 2259 ) 2260 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
on_no_records: Optional[OnNoRecords]
class
SessionTokenRequestApiKeyAuthenticator(pydantic.v1.main.BaseModel):
2263class SessionTokenRequestApiKeyAuthenticator(BaseModel): 2264 type: Literal["ApiKey"] 2265 inject_into: RequestOption = Field( 2266 ..., 2267 description="Configure how the API Key will be sent in requests to the source API.", 2268 examples=[ 2269 {"inject_into": "header", "field_name": "Authorization"}, 2270 {"inject_into": "request_parameter", "field_name": "authKey"}, 2271 ], 2272 title="Inject API Key Into Outgoing HTTP Request", 2273 ) 2274 api_token: Optional[str] = Field( 2275 "{{ session_token }}", 2276 description='A template for the token value to inject. Use {{ session_token }} to reference the session token. For example, use "Token {{ session_token }}" for APIs that expect "Authorization: Token <token>".', 2277 examples=[ 2278 "{{ session_token }}", 2279 "Token {{ session_token }}", 2280 "Bearer {{ session_token }}", 2281 ], 2282 title="API Token Template", 2283 )
inject_into: RequestOption
class
JsonSchemaPropertySelector(pydantic.v1.main.BaseModel):
2286class JsonSchemaPropertySelector(BaseModel): 2287 type: Literal["JsonSchemaPropertySelector"] 2288 transformations: Optional[ 2289 List[ 2290 Union[ 2291 AddFields, 2292 RemoveFields, 2293 KeysToLower, 2294 KeysToSnakeCase, 2295 FlattenFields, 2296 DpathFlattenFields, 2297 KeysReplace, 2298 CustomTransformation, 2299 ] 2300 ] 2301 ] = Field( 2302 None, 2303 description="A list of transformations to be applied on the customer configured schema that will be used to filter out unselected fields when specifying query properties for API requests.", 2304 title="Transformations", 2305 ) 2306 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
class
ListPartitionRouter(pydantic.v1.main.BaseModel):
2309class ListPartitionRouter(BaseModel): 2310 type: Literal["ListPartitionRouter"] 2311 cursor_field: str = Field( 2312 ..., 2313 description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', 2314 examples=["section", "{{ config['section_key'] }}"], 2315 title="Current Partition Value Identifier", 2316 ) 2317 values: Union[str, List[str]] = Field( 2318 ..., 2319 description="The list of attributes being iterated over and used as input for the requests made to the source API.", 2320 examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], 2321 title="Partition Values", 2322 ) 2323 request_option: Optional[RequestOption] = Field( 2324 None, 2325 description="A request option describing where the list value should be injected into and under what field name if applicable.", 2326 title="Inject Partition Value Into Outgoing HTTP Request", 2327 ) 2328 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
request_option: Optional[RequestOption]
class
PaginationReset(pydantic.v1.main.BaseModel):
2331class PaginationReset(BaseModel): 2332 type: Literal["PaginationReset"] 2333 action: Action1 2334 limits: Optional[PaginationResetLimits] = None
action: Action1
limits: Optional[PaginationResetLimits]
class
GzipDecoder(pydantic.v1.main.BaseModel):
2337class GzipDecoder(BaseModel): 2338 type: Literal["GzipDecoder"] 2339 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
RequestBodyGraphQL(pydantic.v1.main.BaseModel):
2342class RequestBodyGraphQL(BaseModel): 2343 type: Literal["RequestBodyGraphQL"] 2344 value: RequestBodyGraphQlQuery
value: RequestBodyGraphQlQuery
class
DpathValidator(pydantic.v1.main.BaseModel):
2347class DpathValidator(BaseModel): 2348 type: Literal["DpathValidator"] 2349 field_path: List[str] = Field( 2350 ..., 2351 description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.', 2352 examples=[ 2353 ["data"], 2354 ["data", "records"], 2355 ["data", "{{ parameters.name }}"], 2356 ["data", "*", "record"], 2357 ], 2358 title="Field Path", 2359 ) 2360 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2361 ..., 2362 description="The condition that the specified config value will be evaluated against", 2363 title="Validation Strategy", 2364 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
PredicateValidator(pydantic.v1.main.BaseModel):
2367class PredicateValidator(BaseModel): 2368 type: Literal["PredicateValidator"] 2369 value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field( 2370 ..., 2371 description="The value to be validated. Can be a literal value or interpolated from configuration.", 2372 examples=[ 2373 "test-value", 2374 "{{ config['api_version'] }}", 2375 "{{ config['tenant_id'] }}", 2376 123, 2377 ], 2378 title="Value", 2379 ) 2380 validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( 2381 ..., 2382 description="The validation strategy to apply to the value.", 2383 title="Validation Strategy", 2384 )
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy]
class
ConfigAddFields(pydantic.v1.main.BaseModel):
2387class ConfigAddFields(BaseModel): 2388 type: Literal["ConfigAddFields"] 2389 fields: List[AddedFieldDefinition] = Field( 2390 ..., 2391 description="A list of transformations (path and corresponding value) that will be added to the config.", 2392 title="Fields", 2393 ) 2394 condition: Optional[str] = Field( 2395 "", 2396 description="Fields will be added if expression is evaluated to True.", 2397 examples=[ 2398 "{{ config['environemnt'] == 'sandbox' }}", 2399 "{{ property is integer }}", 2400 "{{ property|length > 5 }}", 2401 "{{ property == 'some_string_to_match' }}", 2402 ], 2403 )
fields: List[AddedFieldDefinition]
class
CompositeErrorHandler(pydantic.v1.main.BaseModel):
2406class CompositeErrorHandler(BaseModel): 2407 type: Literal["CompositeErrorHandler"] 2408 error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( 2409 Field( 2410 ..., 2411 description="List of error handlers to iterate on to determine how to handle a failed response.", 2412 title="Error Handlers", 2413 ) 2414 ) 2415 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]]
class
HTTPAPIBudget(pydantic.v1.main.BaseModel):
2418class HTTPAPIBudget(BaseModel): 2419 class Config: 2420 extra = Extra.allow 2421 2422 type: Literal["HTTPAPIBudget"] 2423 policies: List[ 2424 Union[ 2425 FixedWindowCallRatePolicy, 2426 MovingWindowCallRatePolicy, 2427 UnlimitedCallRatePolicy, 2428 ] 2429 ] = Field( 2430 ..., 2431 description="List of call rate policies that define how many calls are allowed.", 2432 title="Policies", 2433 ) 2434 ratelimit_reset_header: Optional[str] = Field( 2435 "ratelimit-reset", 2436 description="The HTTP response header name that indicates when the rate limit resets.", 2437 title="Rate Limit Reset Header", 2438 ) 2439 ratelimit_remaining_header: Optional[str] = Field( 2440 "ratelimit-remaining", 2441 description="The HTTP response header name that indicates the number of remaining allowed calls.", 2442 title="Rate Limit Remaining Header", 2443 ) 2444 status_codes_for_ratelimit_hit: Optional[List[int]] = Field( 2445 [429], 2446 description="List of HTTP status codes that indicate a rate limit has been hit.", 2447 title="Status Codes for Rate Limit Hit", 2448 )
policies: List[Union[FixedWindowCallRatePolicy, MovingWindowCallRatePolicy, UnlimitedCallRatePolicy]]
class
HTTPAPIBudget.Config:
class
DpathExtractor(pydantic.v1.main.BaseModel):
2451class DpathExtractor(BaseModel): 2452 type: Literal["DpathExtractor"] 2453 field_path: List[str] = Field( 2454 ..., 2455 description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', 2456 examples=[ 2457 ["data"], 2458 ["data", "records"], 2459 ["data", "{{ parameters.name }}"], 2460 ["data", "*", "record"], 2461 ], 2462 title="Field Path", 2463 ) 2464 record_expander: Optional[RecordExpander] = Field( 2465 None, 2466 description="Optional component to expand records by extracting items from nested array fields.", 2467 title="Record Expander", 2468 ) 2469 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_expander: Optional[RecordExpander]
class
ZipfileDecoder(pydantic.v1.main.BaseModel):
2472class ZipfileDecoder(BaseModel): 2473 class Config: 2474 extra = Extra.allow 2475 2476 type: Literal["ZipfileDecoder"] 2477 decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( 2478 ..., 2479 description="Parser to parse the decompressed data from the zipfile(s).", 2480 title="Parser", 2481 )
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder]
class
ZipfileDecoder.Config:
class
RecordSelector(pydantic.v1.main.BaseModel):
2484class RecordSelector(BaseModel): 2485 type: Literal["RecordSelector"] 2486 extractor: Union[DpathExtractor, CustomRecordExtractor] 2487 record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 2488 None, 2489 description="Responsible for filtering records to be emitted by the Source.", 2490 title="Record Filter", 2491 ) 2492 schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( 2493 None, 2494 description="Responsible for normalization according to the schema.", 2495 title="Schema Normalization", 2496 ) 2497 transform_before_filtering: Optional[bool] = Field( 2498 None, 2499 description="If true, transformation will be applied before record filtering.", 2500 title="Transform Before Filtering", 2501 ) 2502 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
extractor: Union[DpathExtractor, CustomRecordExtractor]
record_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_normalization: Union[SchemaNormalization, CustomSchemaNormalization, NoneType]
class
ConfigMigration(pydantic.v1.main.BaseModel):
2505class ConfigMigration(BaseModel): 2506 type: Literal["ConfigMigration"] 2507 description: Optional[str] = Field( 2508 None, description="The description/purpose of the config migration." 2509 ) 2510 transformations: List[ 2511 Union[ 2512 ConfigRemapField, 2513 ConfigAddFields, 2514 ConfigRemoveFields, 2515 CustomConfigTransformation, 2516 ] 2517 ] = Field( 2518 ..., 2519 description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", 2520 title="Transformations", 2521 )
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
class
ConfigNormalizationRules(pydantic.v1.main.BaseModel):
2524class ConfigNormalizationRules(BaseModel): 2525 class Config: 2526 extra = Extra.forbid 2527 2528 type: Literal["ConfigNormalizationRules"] 2529 config_migrations: Optional[List[ConfigMigration]] = Field( 2530 [], 2531 description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.", 2532 title="Config Migrations", 2533 ) 2534 transformations: Optional[ 2535 List[ 2536 Union[ 2537 ConfigRemapField, 2538 ConfigAddFields, 2539 ConfigRemoveFields, 2540 CustomConfigTransformation, 2541 ] 2542 ] 2543 ] = Field( 2544 [], 2545 description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.", 2546 title="Transformations", 2547 ) 2548 validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field( 2549 [], 2550 description="The list of validations that will be performed on the incoming config at the start of each sync.", 2551 title="Validations", 2552 )
config_migrations: Optional[List[ConfigMigration]]
transformations: Optional[List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]]
validations: Optional[List[Union[DpathValidator, PredicateValidator]]]
class
ConfigNormalizationRules.Config:
class
Spec(pydantic.v1.main.BaseModel):
2555class Spec(BaseModel): 2556 type: Literal["Spec"] 2557 connection_specification: Dict[str, Any] = Field( 2558 ..., 2559 description="A connection specification describing how a the connector can be configured.", 2560 title="Connection Specification", 2561 ) 2562 documentation_url: Optional[str] = Field( 2563 None, 2564 description="URL of the connector's documentation page.", 2565 examples=["https://docs.airbyte.com/integrations/sources/dremio"], 2566 title="Documentation URL", 2567 ) 2568 advanced_auth: Optional[AuthFlow] = Field( 2569 None, 2570 description="Advanced specification for configuring the authentication flow.", 2571 title="Advanced Auth", 2572 ) 2573 config_normalization_rules: Optional[ConfigNormalizationRules] = Field( 2574 None, title="Config Normalization Rules" 2575 )
advanced_auth: Optional[AuthFlow]
config_normalization_rules: Optional[ConfigNormalizationRules]
class
DeclarativeSource1(pydantic.v1.main.BaseModel):
2578class DeclarativeSource1(BaseModel): 2579 class Config: 2580 extra = Extra.forbid 2581 2582 type: Literal["DeclarativeSource"] 2583 check: Union[CheckStream, CheckDynamicStream] 2584 streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] 2585 dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None 2586 version: str = Field( 2587 ..., 2588 description="The version of the Airbyte CDK used to build and test the source.", 2589 ) 2590 schemas: Optional[Schemas] = None 2591 definitions: Optional[Dict[str, Any]] = None 2592 spec: Optional[Spec] = None 2593 concurrency_level: Optional[ConcurrencyLevel] = None 2594 api_budget: Optional[HTTPAPIBudget] = None 2595 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2596 None, 2597 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2598 title="Stream Groups", 2599 ) 2600 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2601 None, 2602 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2603 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2604 title="Maximum Concurrent Asynchronous Jobs", 2605 ) 2606 metadata: Optional[Dict[str, Any]] = Field( 2607 None, 2608 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2609 ) 2610 description: Optional[str] = Field( 2611 None, 2612 description="A description of the connector. It will be presented on the Source documentation page.", 2613 )
check: Union[CheckStream, CheckDynamicStream]
streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
dynamic_streams: Optional[List[DynamicDeclarativeStream]]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource1.Config:
class
DeclarativeSource2(pydantic.v1.main.BaseModel):
2616class DeclarativeSource2(BaseModel): 2617 class Config: 2618 extra = Extra.forbid 2619 2620 type: Literal["DeclarativeSource"] 2621 check: Union[CheckStream, CheckDynamicStream] 2622 streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( 2623 None 2624 ) 2625 dynamic_streams: List[DynamicDeclarativeStream] 2626 version: str = Field( 2627 ..., 2628 description="The version of the Airbyte CDK used to build and test the source.", 2629 ) 2630 schemas: Optional[Schemas] = None 2631 definitions: Optional[Dict[str, Any]] = None 2632 spec: Optional[Spec] = None 2633 concurrency_level: Optional[ConcurrencyLevel] = None 2634 api_budget: Optional[HTTPAPIBudget] = None 2635 stream_groups: Optional[Dict[str, StreamGroup]] = Field( 2636 None, 2637 description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", 2638 title="Stream Groups", 2639 ) 2640 max_concurrent_async_job_count: Optional[Union[int, str]] = Field( 2641 None, 2642 description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.", 2643 examples=[3, "{{ config['max_concurrent_async_job_count'] }}"], 2644 title="Maximum Concurrent Asynchronous Jobs", 2645 ) 2646 metadata: Optional[Dict[str, Any]] = Field( 2647 None, 2648 description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", 2649 ) 2650 description: Optional[str] = Field( 2651 None, 2652 description="A description of the connector. It will be presented on the Source documentation page.", 2653 )
check: Union[CheckStream, CheckDynamicStream]
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]]
dynamic_streams: List[DynamicDeclarativeStream]
schemas: Optional[Schemas]
spec: Optional[Spec]
concurrency_level: Optional[ConcurrencyLevel]
api_budget: Optional[HTTPAPIBudget]
stream_groups: Optional[Dict[str, StreamGroup]]
class
DeclarativeSource2.Config:
class
DeclarativeSource(pydantic.v1.main.BaseModel):
2656class DeclarativeSource(BaseModel): 2657 class Config: 2658 extra = Extra.forbid 2659 2660 __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field( 2661 ..., 2662 description="An API source that extracts data according to its declarative components.", 2663 title="DeclarativeSource", 2664 )
class
DeclarativeSource.Config:
class
SelectiveAuthenticator(pydantic.v1.main.BaseModel):
2667class SelectiveAuthenticator(BaseModel): 2668 class Config: 2669 extra = Extra.allow 2670 2671 type: Literal["SelectiveAuthenticator"] 2672 authenticator_selection_path: List[str] = Field( 2673 ..., 2674 description="Path of the field in config with selected authenticator name", 2675 examples=[["auth"], ["auth", "type"]], 2676 title="Authenticator Selection Path", 2677 ) 2678 authenticators: Dict[ 2679 str, 2680 Union[ 2681 ApiKeyAuthenticator, 2682 BasicHttpAuthenticator, 2683 BearerAuthenticator, 2684 OAuthAuthenticator, 2685 JwtAuthenticator, 2686 SessionTokenAuthenticator, 2687 LegacySessionTokenAuthenticator, 2688 CustomAuthenticator, 2689 NoAuth, 2690 RateLimitedMultipleTokenAuthenticator, 2691 ], 2692 ] = Field( 2693 ..., 2694 description="Authenticators to select from.", 2695 examples=[ 2696 { 2697 "authenticators": { 2698 "token": "#/definitions/ApiKeyAuthenticator", 2699 "oauth": "#/definitions/OAuthAuthenticator", 2700 "jwt": "#/definitions/JwtAuthenticator", 2701 } 2702 } 2703 ], 2704 title="Authenticators", 2705 ) 2706 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
class
SelectiveAuthenticator.Config:
class
ConditionalStreams(pydantic.v1.main.BaseModel):
2709class ConditionalStreams(BaseModel): 2710 type: Literal["ConditionalStreams"] 2711 condition: str = Field( 2712 ..., 2713 description="Condition that will be evaluated to determine if a set of streams should be available.", 2714 examples=["{{ config['is_sandbox'] }}"], 2715 title="Condition", 2716 ) 2717 streams: List[DeclarativeStream] = Field( 2718 ..., 2719 description="Streams that will be used during an operation based on the condition.", 2720 title="Streams", 2721 ) 2722 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
streams: List[DeclarativeStream]
class
FileUploader(pydantic.v1.main.BaseModel):
2725class FileUploader(BaseModel): 2726 type: Literal["FileUploader"] 2727 requester: Union[HttpRequester, CustomRequester] = Field( 2728 ..., 2729 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 2730 ) 2731 download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 2732 ..., 2733 description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", 2734 ) 2735 file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 2736 None, 2737 description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", 2738 ) 2739 filename_extractor: Optional[str] = Field( 2740 None, 2741 description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.", 2742 examples=[ 2743 "{{ record.id }}/{{ record.file_name }}/", 2744 "{{ record.id }}_{{ record.file_name }}/", 2745 ], 2746 ) 2747 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor]
file_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
class
DeclarativeStream(pydantic.v1.main.BaseModel):
2750class DeclarativeStream(BaseModel): 2751 class Config: 2752 extra = Extra.allow 2753 2754 type: Literal["DeclarativeStream"] 2755 name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") 2756 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 2757 ..., 2758 description="Component used to coordinate how records are extracted across stream slices and request pages.", 2759 title="Retriever", 2760 ) 2761 incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( 2762 None, 2763 description="Component used to fetch data incrementally based on a time field in the data.", 2764 title="Incremental Sync", 2765 ) 2766 primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") 2767 schema_loader: Optional[ 2768 Union[ 2769 InlineSchemaLoader, 2770 DynamicSchemaLoader, 2771 JsonFileSchemaLoader, 2772 List[ 2773 Union[ 2774 InlineSchemaLoader, 2775 DynamicSchemaLoader, 2776 JsonFileSchemaLoader, 2777 CustomSchemaLoader, 2778 ] 2779 ], 2780 CustomSchemaLoader, 2781 ] 2782 ] = Field( 2783 None, 2784 description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.", 2785 title="Schema Loader", 2786 ) 2787 transformations: Optional[ 2788 List[ 2789 Union[ 2790 AddFields, 2791 RemoveFields, 2792 KeysToLower, 2793 KeysToSnakeCase, 2794 FlattenFields, 2795 DpathFlattenFields, 2796 KeysReplace, 2797 CustomTransformation, 2798 ] 2799 ] 2800 ] = Field( 2801 None, 2802 description="A list of transformations to be applied to each output record.", 2803 title="Transformations", 2804 ) 2805 state_migrations: Optional[ 2806 List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] 2807 ] = Field( 2808 [], 2809 description="Array of state migrations to be applied on the input state", 2810 title="State Migrations", 2811 ) 2812 file_uploader: Optional[FileUploader] = Field( 2813 None, 2814 description="(experimental) Describes how to fetch a file", 2815 title="File Uploader", 2816 ) 2817 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
incremental_sync: Union[DatetimeBasedCursor, IncrementingCountCursor, NoneType]
primary_key: Optional[PrimaryKey]
schema_loader: Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, List[Union[InlineSchemaLoader, DynamicSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]], CustomSchemaLoader, NoneType]
transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]]
file_uploader: Optional[FileUploader]
class
DeclarativeStream.Config:
class
SessionTokenAuthenticator(pydantic.v1.main.BaseModel):
2820class SessionTokenAuthenticator(BaseModel): 2821 type: Literal["SessionTokenAuthenticator"] 2822 login_requester: HttpRequester = Field( 2823 ..., 2824 description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", 2825 examples=[ 2826 { 2827 "type": "HttpRequester", 2828 "url_base": "https://my_api.com", 2829 "path": "/login", 2830 "authenticator": { 2831 "type": "BasicHttpAuthenticator", 2832 "username": "{{ config.username }}", 2833 "password": "{{ config.password }}", 2834 }, 2835 } 2836 ], 2837 title="Login Requester", 2838 ) 2839 session_token_path: List[str] = Field( 2840 ..., 2841 description="The path in the response body returned from the login requester to the session token.", 2842 examples=[["access_token"], ["result", "token"]], 2843 title="Session Token Path", 2844 ) 2845 expiration_duration: Optional[str] = Field( 2846 None, 2847 description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n", 2848 examples=["PT1H", "P1D"], 2849 title="Expiration Duration", 2850 ) 2851 request_authentication: Union[ 2852 SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator 2853 ] = Field( 2854 ..., 2855 description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", 2856 title="Data Request Authentication", 2857 ) 2858 decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field( 2859 None, description="Component used to decode the response.", title="Decoder" 2860 ) 2861 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
login_requester: HttpRequester
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator]
decoder: Union[JsonDecoder, XmlDecoder, NoneType]
class
HttpRequester(airbyte_cdk.sources.declarative.models.base_model_with_deprecations.BaseModelWithDeprecations):
2864class HttpRequester(BaseModelWithDeprecations): 2865 type: Literal["HttpRequester"] 2866 url_base: Optional[str] = Field( 2867 None, 2868 deprecated=True, 2869 deprecation_message="Use `url` field instead.", 2870 description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2871 examples=[ 2872 "https://connect.squareup.com/v2", 2873 "{{ config['base_url'] or 'https://app.posthog.com'}}/api", 2874 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2875 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2876 ], 2877 title="API Base URL", 2878 ) 2879 url: Optional[str] = Field( 2880 None, 2881 description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2882 examples=[ 2883 "https://connect.squareup.com/v2", 2884 "{{ config['url'] or 'https://app.posthog.com'}}/api", 2885 "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2886 "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", 2887 ], 2888 title="API Endpoint URL", 2889 ) 2890 path: Optional[str] = Field( 2891 None, 2892 deprecated=True, 2893 deprecation_message="Use `url` field instead.", 2894 description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", 2895 examples=[ 2896 "/products", 2897 "/quotes/{{ stream_partition['id'] }}/quote_line_groups", 2898 "/trades/{{ config['symbol_id'] }}/history", 2899 ], 2900 title="URL Path", 2901 ) 2902 http_method: Optional[HttpMethod] = Field( 2903 HttpMethod.GET, 2904 description="The HTTP method used to fetch data from the source (can be GET or POST).", 2905 examples=["GET", "POST"], 2906 title="HTTP Method", 2907 ) 2908 authenticator: Optional[ 2909 Union[ 2910 ApiKeyAuthenticator, 2911 BasicHttpAuthenticator, 2912 BearerAuthenticator, 2913 OAuthAuthenticator, 2914 JwtAuthenticator, 2915 SessionTokenAuthenticator, 2916 SelectiveAuthenticator, 2917 CustomAuthenticator, 2918 NoAuth, 2919 LegacySessionTokenAuthenticator, 2920 RateLimitedMultipleTokenAuthenticator, 2921 ] 2922 ] = Field( 2923 None, 2924 description="Authentication method to use for requests sent to the API.", 2925 title="Authenticator", 2926 ) 2927 fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field( 2928 None, 2929 deprecated=True, 2930 deprecation_message="Use `query_properties` field instead.", 2931 description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", 2932 title="Fetch Properties from Endpoint", 2933 ) 2934 query_properties: Optional[QueryProperties] = Field( 2935 None, 2936 description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", 2937 title="Query Properties", 2938 ) 2939 request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( 2940 None, 2941 description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", 2942 examples=[ 2943 {"unit": "day"}, 2944 { 2945 "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' 2946 }, 2947 {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, 2948 {"sort_by[asc]": "updated_at"}, 2949 ], 2950 title="Query Parameters", 2951 ) 2952 request_headers: Optional[Union[Dict[str, str], str]] = Field( 2953 None, 2954 description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", 2955 examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], 2956 title="Request Headers", 2957 ) 2958 request_body_data: Optional[Union[Dict[str, str], str]] = Field( 2959 None, 2960 deprecated=True, 2961 deprecation_message="Use `request_body` field instead.", 2962 description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", 2963 examples=[ 2964 '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' 2965 ], 2966 title="Request Body Payload (Non-JSON)", 2967 ) 2968 request_body_json: Optional[Union[Dict[str, Any], str]] = Field( 2969 None, 2970 deprecated=True, 2971 deprecation_message="Use `request_body` field instead.", 2972 description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", 2973 examples=[ 2974 {"sort_order": "ASC", "sort_field": "CREATED_AT"}, 2975 {"key": "{{ config['value'] }}"}, 2976 {"sort": {"field": "updated_at", "order": "ascending"}}, 2977 ], 2978 title="Request Body JSON Payload", 2979 ) 2980 request_body: Optional[ 2981 Union[ 2982 RequestBodyPlainText, 2983 RequestBodyUrlEncodedForm, 2984 RequestBodyJsonObject, 2985 RequestBodyGraphQL, 2986 ] 2987 ] = Field( 2988 None, 2989 description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", 2990 title="Request Body", 2991 ) 2992 error_handler: Optional[ 2993 Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] 2994 ] = Field( 2995 None, 2996 description="Error handler component that defines how to handle errors.", 2997 title="Error Handler", 2998 ) 2999 use_cache: Optional[bool] = Field( 3000 False, 3001 description="Enables stream requests caching. When set to true, repeated requests to the same URL will return cached responses. Parent streams automatically have caching enabled. Only set this to false if you are certain that caching should be disabled, as it may negatively impact performance when the same data is needed multiple times (e.g., for scroll-based pagination APIs where caching causes duplicate records).", 3002 title="Use Cache", 3003 ) 3004 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
Pydantic BaseModel that warns when deprecated fields are accessed. The deprecation message is stored in the field's extra attributes. This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized.
The _deprecation_logs attribute is stored in the model itself.
The collected deprecation warnings are further propagated to the Airbyte log messages,
during the component creation process, in model_to_component._collect_model_deprecations().
The component implementation is not responsible for handling the deprecation warnings, since the deprecation warnings are already handled in the model itself.
http_method: Optional[HttpMethod]
authenticator: Union[ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, OAuthAuthenticator, JwtAuthenticator, SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, NoAuth, LegacySessionTokenAuthenticator, RateLimitedMultipleTokenAuthenticator, NoneType]
fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint]
query_properties: Optional[QueryProperties]
request_parameters: Union[Dict[str, Union[str, QueryProperties]], str, NoneType]
request_body: Union[RequestBodyPlainText, RequestBodyUrlEncodedForm, RequestBodyJsonObject, RequestBodyGraphQL, NoneType]
error_handler: Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler, NoneType]
class
DynamicSchemaLoader(pydantic.v1.main.BaseModel):
3007class DynamicSchemaLoader(BaseModel): 3008 type: Literal["DynamicSchemaLoader"] 3009 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3010 ..., 3011 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3012 title="Retriever", 3013 ) 3014 schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( 3015 None, 3016 description="Responsible for filtering fields to be added to json schema.", 3017 title="Schema Filter", 3018 ) 3019 schema_transformations: Optional[ 3020 List[ 3021 Union[ 3022 AddFields, 3023 RemoveFields, 3024 KeysToLower, 3025 KeysToSnakeCase, 3026 FlattenFields, 3027 DpathFlattenFields, 3028 KeysReplace, 3029 CustomTransformation, 3030 ] 3031 ] 3032 ] = Field( 3033 None, 3034 description="A list of transformations to be applied to the schema.", 3035 title="Schema Transformations", 3036 ) 3037 schema_type_identifier: SchemaTypeIdentifier 3038 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
schema_filter: Union[RecordFilter, CustomRecordFilter, NoneType]
schema_transformations: Optional[List[Union[AddFields, RemoveFields, KeysToLower, KeysToSnakeCase, FlattenFields, DpathFlattenFields, KeysReplace, CustomTransformation]]]
schema_type_identifier: SchemaTypeIdentifier
class
ParentStreamConfig(pydantic.v1.main.BaseModel):
3041class ParentStreamConfig(BaseModel): 3042 type: Literal["ParentStreamConfig"] 3043 stream: Union[DeclarativeStream, StateDelegatingStream] = Field( 3044 ..., description="Reference to the parent stream.", title="Parent Stream" 3045 ) 3046 parent_key: str = Field( 3047 ..., 3048 description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", 3049 examples=["id", "{{ config['parent_record_id'] }}"], 3050 title="Parent Key", 3051 ) 3052 partition_field: str = Field( 3053 ..., 3054 description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", 3055 examples=["parent_id", "{{ config['parent_partition_field'] }}"], 3056 title="Current Parent Key Value Identifier", 3057 ) 3058 request_option: Optional[RequestOption] = Field( 3059 None, 3060 description="A request option describing where the parent key value should be injected into and under what field name if applicable.", 3061 title="Request Option", 3062 ) 3063 incremental_dependency: Optional[bool] = Field( 3064 False, 3065 description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", 3066 title="Incremental Dependency", 3067 ) 3068 lazy_read_pointer: Optional[List[str]] = Field( 3069 [], 3070 description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.", 3071 title="Lazy Read Pointer", 3072 ) 3073 extra_fields: Optional[List[List[str]]] = Field( 3074 None, 3075 description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.", 3076 title="Extra Fields", 3077 ) 3078 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
stream: Union[DeclarativeStream, StateDelegatingStream]
request_option: Optional[RequestOption]
class
PropertiesFromEndpoint(pydantic.v1.main.BaseModel):
3081class PropertiesFromEndpoint(BaseModel): 3082 type: Literal["PropertiesFromEndpoint"] 3083 property_field_path: List[str] = Field( 3084 ..., 3085 description="Describes the path to the field that should be extracted", 3086 examples=[["name"]], 3087 ) 3088 retriever: Union[SimpleRetriever, CustomRetriever] = Field( 3089 ..., 3090 description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", 3091 ) 3092 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, CustomRetriever]
class
QueryProperties(pydantic.v1.main.BaseModel):
3095class QueryProperties(BaseModel): 3096 type: Literal["QueryProperties"] 3097 property_list: Union[List[str], PropertiesFromEndpoint] = Field( 3098 ..., 3099 description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint", 3100 title="Property List", 3101 ) 3102 always_include_properties: Optional[List[str]] = Field( 3103 None, 3104 description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.", 3105 title="Always Include Properties", 3106 ) 3107 property_chunking: Optional[PropertyChunking] = Field( 3108 None, 3109 description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.", 3110 title="Property Chunking", 3111 ) 3112 property_selector: Optional[JsonSchemaPropertySelector] = Field( 3113 None, 3114 description="Defines where to look for and which query properties that should be sent in outbound API requests. For example, you can specify that only the selected columns of a stream should be in the request.", 3115 title="Property Selector", 3116 ) 3117 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
property_list: Union[List[str], PropertiesFromEndpoint]
property_chunking: Optional[PropertyChunking]
property_selector: Optional[JsonSchemaPropertySelector]
class
StateDelegatingStream(pydantic.v1.main.BaseModel):
3120class StateDelegatingStream(BaseModel): 3121 type: Literal["StateDelegatingStream"] 3122 name: str = Field(..., description="The stream name.", example=["Users"], title="Name") 3123 full_refresh_stream: DeclarativeStream = Field( 3124 ..., 3125 description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", 3126 title="Full Refresh Stream", 3127 ) 3128 incremental_stream: DeclarativeStream = Field( 3129 ..., 3130 description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.", 3131 title="Incremental Stream", 3132 ) 3133 api_retention_period: Optional[str] = Field( 3134 None, 3135 description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", 3136 examples=["P30D", "P90D", "P1Y"], 3137 title="API Retention Period", 3138 ) 3139 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
full_refresh_stream: DeclarativeStream
incremental_stream: DeclarativeStream
class
SimpleRetriever(pydantic.v1.main.BaseModel):
3142class SimpleRetriever(BaseModel): 3143 type: Literal["SimpleRetriever"] 3144 requester: Union[HttpRequester, CustomRequester] = Field( 3145 ..., 3146 description="Requester component that describes how to prepare HTTP requests to send to the source API.", 3147 ) 3148 decoder: Optional[ 3149 Union[ 3150 JsonDecoder, 3151 JsonItemsDecoder, 3152 XmlDecoder, 3153 CsvDecoder, 3154 JsonlDecoder, 3155 GzipDecoder, 3156 IterableDecoder, 3157 ZipfileDecoder, 3158 CustomDecoder, 3159 ] 3160 ] = Field( 3161 None, 3162 description="Component decoding the response so records can be extracted.", 3163 title="HTTP Response Format", 3164 ) 3165 record_selector: RecordSelector = Field( 3166 ..., 3167 description="Component that describes how to extract records from a HTTP response.", 3168 ) 3169 paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3170 None, 3171 description="Paginator component that describes how to navigate through the API's pages.", 3172 ) 3173 pagination_reset: Optional[PaginationReset] = Field( 3174 None, 3175 description="Describes what triggers pagination reset and how to handle it.", 3176 ) 3177 ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( 3178 False, 3179 description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", 3180 ) 3181 partition_router: Optional[ 3182 Union[ 3183 SubstreamPartitionRouter, 3184 ListPartitionRouter, 3185 GroupingPartitionRouter, 3186 CustomPartitionRouter, 3187 List[ 3188 Union[ 3189 SubstreamPartitionRouter, 3190 ListPartitionRouter, 3191 GroupingPartitionRouter, 3192 CustomPartitionRouter, 3193 ] 3194 ], 3195 ] 3196 ] = Field( 3197 None, 3198 description="Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.", 3199 title="Partition Router", 3200 ) 3201 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
requester: Union[HttpRequester, CustomRequester]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
record_selector: RecordSelector
paginator: Union[DefaultPaginator, NoPagination, NoneType]
pagination_reset: Optional[PaginationReset]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
class
AsyncRetriever(pydantic.v1.main.BaseModel):
3204class AsyncRetriever(BaseModel): 3205 type: Literal["AsyncRetriever"] 3206 record_selector: RecordSelector = Field( 3207 ..., 3208 description="Component that describes how to extract records from a HTTP response.", 3209 ) 3210 status_mapping: AsyncJobStatusMap = Field( 3211 ..., description="Async Job Status to Airbyte CDK Async Job Status mapping." 3212 ) 3213 status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( 3214 ..., description="Responsible for fetching the actual status of the async job." 3215 ) 3216 download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( 3217 None, 3218 description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", 3219 ) 3220 download_extractor: Optional[ 3221 Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] 3222 ] = Field(None, description="Responsible for fetching the records from provided urls.") 3223 creation_requester: Union[HttpRequester, CustomRequester] = Field( 3224 ..., 3225 description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", 3226 ) 3227 polling_requester: Union[HttpRequester, CustomRequester] = Field( 3228 ..., 3229 description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.", 3230 ) 3231 polling_job_timeout: Optional[Union[int, str]] = Field( 3232 None, 3233 description="The time in minutes after which the single Async Job should be considered as Timed Out.", 3234 ) 3235 failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( 3236 None, 3237 description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", 3238 ge=1, 3239 ) 3240 download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3241 None, 3242 description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.", 3243 ) 3244 download_requester: Union[HttpRequester, CustomRequester] = Field( 3245 ..., 3246 description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.", 3247 ) 3248 download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( 3249 None, 3250 description="Paginator component that describes how to navigate through the API's pages during download.", 3251 ) 3252 abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3253 None, 3254 description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.", 3255 ) 3256 delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( 3257 None, 3258 description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.", 3259 ) 3260 partition_router: Optional[ 3261 Union[ 3262 ListPartitionRouter, 3263 SubstreamPartitionRouter, 3264 GroupingPartitionRouter, 3265 CustomPartitionRouter, 3266 List[ 3267 Union[ 3268 ListPartitionRouter, 3269 SubstreamPartitionRouter, 3270 GroupingPartitionRouter, 3271 CustomPartitionRouter, 3272 ] 3273 ], 3274 ] 3275 ] = Field( 3276 [], 3277 description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", 3278 title="Partition Router", 3279 ) 3280 decoder: Optional[ 3281 Union[ 3282 CsvDecoder, 3283 GzipDecoder, 3284 JsonDecoder, 3285 JsonItemsDecoder, 3286 JsonlDecoder, 3287 IterableDecoder, 3288 XmlDecoder, 3289 ZipfileDecoder, 3290 CustomDecoder, 3291 ] 3292 ] = Field( 3293 None, 3294 description="Component decoding the response so records can be extracted.", 3295 title="HTTP Response Format", 3296 ) 3297 download_decoder: Optional[ 3298 Union[ 3299 CsvDecoder, 3300 GzipDecoder, 3301 JsonDecoder, 3302 JsonItemsDecoder, 3303 JsonlDecoder, 3304 IterableDecoder, 3305 XmlDecoder, 3306 ZipfileDecoder, 3307 CustomDecoder, 3308 ] 3309 ] = Field( 3310 None, 3311 description="Component decoding the download response so records can be extracted.", 3312 title="Download HTTP Response Format", 3313 ) 3314 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
record_selector: RecordSelector
status_mapping: AsyncJobStatusMap
status_extractor: Union[DpathExtractor, CustomRecordExtractor]
download_target_extractor: Union[DpathExtractor, CustomRecordExtractor, NoneType]
download_extractor: Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor, NoneType]
creation_requester: Union[HttpRequester, CustomRequester]
polling_requester: Union[HttpRequester, CustomRequester]
download_target_requester: Union[HttpRequester, CustomRequester, NoneType]
download_requester: Union[HttpRequester, CustomRequester]
download_paginator: Union[DefaultPaginator, NoPagination, NoneType]
abort_requester: Union[HttpRequester, CustomRequester, NoneType]
delete_requester: Union[HttpRequester, CustomRequester, NoneType]
partition_router: Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter, List[Union[SubstreamPartitionRouter, ListPartitionRouter, GroupingPartitionRouter, CustomPartitionRouter]], NoneType]
decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
download_decoder: Union[JsonDecoder, JsonItemsDecoder, XmlDecoder, CsvDecoder, JsonlDecoder, GzipDecoder, IterableDecoder, ZipfileDecoder, CustomDecoder, NoneType]
class
BlockSimultaneousSyncsAction(pydantic.v1.main.BaseModel):
class
StreamGroup(pydantic.v1.main.BaseModel):
3321class StreamGroup(BaseModel): 3322 streams: List[str] = Field( 3323 ..., 3324 description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', 3325 title="Streams", 3326 ) 3327 action: BlockSimultaneousSyncsAction = Field( 3328 ..., 3329 description="The action to apply to streams in this group.", 3330 title="Action", 3331 )
action: BlockSimultaneousSyncsAction
class
SubstreamPartitionRouter(pydantic.v1.main.BaseModel):
3334class SubstreamPartitionRouter(BaseModel): 3335 type: Literal["SubstreamPartitionRouter"] 3336 parent_stream_configs: List[ParentStreamConfig] = Field( 3337 ..., 3338 description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", 3339 title="Parent Stream Configs", 3340 ) 3341 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
parent_stream_configs: List[ParentStreamConfig]
class
GroupingPartitionRouter(pydantic.v1.main.BaseModel):
3344class GroupingPartitionRouter(BaseModel): 3345 type: Literal["GroupingPartitionRouter"] 3346 group_size: int = Field( 3347 ..., 3348 description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.", 3349 examples=[10, 50], 3350 title="Group Size", 3351 ) 3352 underlying_partition_router: Union[ 3353 ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter 3354 ] = Field( 3355 ..., 3356 description="The partition router whose output will be grouped. This can be any valid partition router component.", 3357 title="Underlying Partition Router", 3358 ) 3359 deduplicate: Optional[bool] = Field( 3360 True, 3361 description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.", 3362 title="Deduplicate Partitions", 3363 ) 3364 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
underlying_partition_router: Union[ListPartitionRouter, SubstreamPartitionRouter, CustomPartitionRouter]
class
HttpComponentsResolver(pydantic.v1.main.BaseModel):
3367class HttpComponentsResolver(BaseModel): 3368 type: Literal["HttpComponentsResolver"] 3369 retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( 3370 ..., 3371 description="Component used to coordinate how records are extracted across stream slices and request pages.", 3372 title="Retriever", 3373 ) 3374 components_mapping: List[ComponentMappingDefinition] 3375 parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever]
components_mapping: List[ComponentMappingDefinition]
class
DynamicDeclarativeStream(pydantic.v1.main.BaseModel):
3378class DynamicDeclarativeStream(BaseModel): 3379 type: Literal["DynamicDeclarativeStream"] 3380 name: Optional[str] = Field( 3381 "", description="The dynamic stream name.", example=["Tables"], title="Name" 3382 ) 3383 stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field( 3384 ..., description="Reference to the stream template.", title="Stream Template" 3385 ) 3386 components_resolver: Union[ 3387 HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver 3388 ] = Field( 3389 ..., 3390 description="Component resolve and populates stream templates with components values.", 3391 title="Components Resolver", 3392 ) 3393 use_parent_parameters: Optional[bool] = Field( 3394 True, 3395 description="Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.", 3396 title="Use Parent Parameters", 3397 )
stream_template: Union[DeclarativeStream, StateDelegatingStream]
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver]