airbyte.constants

  1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
  2"""Constants shared across the PyAirbyte codebase."""
  3
  4from __future__ import annotations
  5
  6import logging
  7import os
  8from pathlib import Path
  9from typing import overload
 10
 11
 12logger = logging.getLogger("airbyte")
 13
 14
 15DEBUG_MODE = False  # Set to True to enable additional debug logging.
 16
 17AB_EXTRACTED_AT_COLUMN = "_airbyte_extracted_at"
 18"""A column that stores the timestamp when the record was extracted."""
 19
 20AB_META_COLUMN = "_airbyte_meta"
 21"""A column that stores metadata about the record."""
 22
 23AB_RAW_ID_COLUMN = "_airbyte_raw_id"
 24"""A column that stores a unique identifier for each row in the source data.
 25
 26Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations.
 27In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In PyAirbyte,
 28this column is simply used as a unique identifier for each record as it is received.
 29
 30PyAirbyte uses ULIDs for this column, which are identifiers that can be sorted by time
 31received. This allows us to determine the debug the order of records as they are received, even if
 32the source provides records that are tied or received out of order from the perspective of their
 33`emitted_at` (`_airbyte_extracted_at`) timestamps.
 34"""
 35
 36AB_INTERNAL_COLUMNS = {
 37    AB_RAW_ID_COLUMN,
 38    AB_EXTRACTED_AT_COLUMN,
 39    AB_META_COLUMN,
 40}
 41"""A set of internal columns that are reserved for PyAirbyte's internal use."""
 42
 43
 44def _try_create_dir_if_missing(path: Path, desc: str = "specified") -> Path:
 45    """Try to create a directory if it does not exist."""
 46    resolved_path = path.expanduser().resolve()
 47    try:
 48        if resolved_path.exists():
 49            if not resolved_path.is_dir():
 50                logger.warning(
 51                    "The %s path exists but is not a directory: '%s'", desc, resolved_path
 52                )
 53            return resolved_path
 54        resolved_path.mkdir(parents=True, exist_ok=True)
 55    except Exception as ex:
 56        logger.warning(
 57            "Could not auto-create missing %s directory at '%s': %s", desc, resolved_path, ex
 58        )
 59    return resolved_path
 60
 61
 62DEFAULT_PROJECT_DIR: Path = _try_create_dir_if_missing(
 63    Path(os.getenv("AIRBYTE_PROJECT_DIR", "") or Path.cwd()).expanduser().absolute(),
 64    desc="project",
 65)
 66"""Default project directory.
 67
 68Can be overridden by setting the `AIRBYTE_PROJECT_DIR` environment variable.
 69
 70If not set, defaults to the current working directory.
 71
 72This serves as the parent directory for both cache and install directories when not explicitly
 73configured.
 74
 75If a path is specified that does not yet exist, PyAirbyte will attempt to create it.
 76"""
 77
 78
 79DEFAULT_INSTALL_DIR: Path = _try_create_dir_if_missing(
 80    Path(os.getenv("AIRBYTE_INSTALL_DIR", "") or DEFAULT_PROJECT_DIR).expanduser().absolute(),
 81    desc="install",
 82)
 83"""Default install directory for connectors.
 84
 85If not set, defaults to `DEFAULT_PROJECT_DIR` (`AIRBYTE_PROJECT_DIR` env var) or the current
 86working directory if neither is set.
 87
 88If a path is specified that does not yet exist, PyAirbyte will attempt to create it.
 89"""
 90
 91
 92DEFAULT_CACHE_ROOT: Path = (
 93    (Path(os.getenv("AIRBYTE_CACHE_ROOT", "") or (DEFAULT_PROJECT_DIR / ".cache")))
 94    .expanduser()
 95    .absolute()
 96)
 97"""Default cache root is `.cache` in the current working directory.
 98
 99The default location can be overridden by setting the `AIRBYTE_CACHE_ROOT` environment variable.
100
101Overriding this can be useful if you always want to store cache files in a specific location.
102For example, in ephemeral environments like Google Colab, you might want to store cache files in
103your mounted Google Drive by setting this to a path like `/content/drive/MyDrive/Airbyte/cache`.
104"""
105
106DEFAULT_CACHE_SCHEMA_NAME = "airbyte_raw"
107"""The default schema name to use for caches.
108
109Specific caches may override this value with a different schema name.
110"""
111
112DEFAULT_GOOGLE_DRIVE_MOUNT_PATH = "/content/drive"
113"""Default path to mount Google Drive in Google Colab environments."""
114
115DEFAULT_ARROW_MAX_CHUNK_SIZE = 100_000
116"""The default number of records to include in each batch of an Arrow dataset."""
117
118
119_TRUE_STR_VALUES: frozenset[str] = frozenset({"1", "true", "t", "yes", "y", "on"})
120"""String values that mean `True` in environment variables and config values."""
121
122_FALSE_STR_VALUES: frozenset[str] = frozenset({"0", "false", "f", "no", "n", "off"})
123"""String values that mean `False` in environment variables and config values."""
124
125
126@overload
127def _str_to_bool(value: str | None, *, default: bool) -> bool: ...
128
129
130@overload
131def _str_to_bool(value: str | None, *, default: None = None) -> bool | None: ...
132
133
134def _str_to_bool(value: str | None, *, default: bool | None = None) -> bool | None:
135    """Convert an environment variable or config value to a boolean.
136
137    Matching is case-insensitive and ignores surrounding whitespace. A value that is
138    unset, blank, or unrecognized yields `default`, which is `None` unless the caller
139    says otherwise, so "no value" stays distinguishable from `False`.
140    """
141    normalized = (value or "").strip().lower()
142    if normalized in _TRUE_STR_VALUES:
143        return True
144    if normalized in _FALSE_STR_VALUES:
145        return False
146    return default
147
148
149TEMP_DIR_OVERRIDE: Path | None = (
150    Path(os.environ["AIRBYTE_TEMP_DIR"]) if os.getenv("AIRBYTE_TEMP_DIR") else None
151)
152"""The directory to use for temporary files.
153
154This value is read from the `AIRBYTE_TEMP_DIR` environment variable. If the variable is not set,
155Tempfile will use the system's default temporary directory.
156
157This can be useful if you want to store temporary files in a specific location (or) when you
158need your temporary files to exist in user level directories, and not in system level
159directories for permissions reasons.
160"""
161
162TEMP_FILE_CLEANUP = _str_to_bool(
163    os.getenv(key="AIRBYTE_TEMP_FILE_CLEANUP"),
164    default=True,
165)
166"""Whether to clean up temporary files after use.
167
168This value is read from the `AIRBYTE_TEMP_FILE_CLEANUP` environment variable. If the variable is
169not set, the default value is `True`.
170"""
171
172AIRBYTE_OFFLINE_MODE = _str_to_bool(
173    os.getenv(key="AIRBYTE_OFFLINE_MODE"),
174    default=False,
175)
176"""Enable or disable offline mode.
177
178When offline mode is enabled, PyAirbyte will attempt to fetch metadata for connectors from the
179Airbyte registry but will not raise an error if the registry is unavailable. This can be useful in
180environments without internet access or with air-gapped networks.
181
182Offline mode also disables telemetry, similar to a `DO_NOT_TRACK` setting, ensuring no usage data
183is sent from your environment. You may also specify a custom registry URL via the`_REGISTRY_ENV_VAR`
184environment variable if you prefer to use a different registry source for metadata.
185
186This setting helps you make informed choices about data privacy and operation in restricted and
187air-gapped environments.
188"""
189
190AIRBYTE_PRINT_FULL_ERROR_LOGS: bool = _str_to_bool(
191    os.getenv(key="AIRBYTE_PRINT_FULL_ERROR_LOGS", default=os.getenv("CI")),
192    default=False,
193)
194"""Whether to print full error logs when an error occurs.
195This setting helps in debugging by providing detailed logs when errors occur. This is especially
196helpful in ephemeral environments like CI/CD pipelines where log files may not be persisted after
197the pipeline run.
198
199If not set, the default value is `False` for non-CI environments.
200If running in a CI environment ("CI" env var is set), then the default value is `True`.
201"""
202
203NO_UV: bool = os.getenv("AIRBYTE_NO_UV", "").lower() in {"1", "true", "yes"}
204"""Whether to disable uv and use pip for Python package management.
205
206This value is determined by the `AIRBYTE_NO_UV` environment variable. When `AIRBYTE_NO_UV`
207is set to "1", "true", or "yes", pip will be used instead of uv.
208
209If the variable is not set or set to any other value, uv will be used by default. Set this
210variable to opt out of uv and use pip instead.
211"""
212
213SECRETS_HYDRATION_PREFIX = "secret_reference::"
214"""Use this prefix to indicate a secret reference in configuration.
215
216For example, this snippet will populate the `personal_access_token` field with the value of the
217secret named `GITHUB_PERSONAL_ACCESS_TOKEN`, for instance from an environment variable.
218
219```json
220{
221  "credentials": {
222    "personal_access_token": "secret_reference::GITHUB_PERSONAL_ACCESS_TOKEN"
223  }
224}
225```
226
227For more information, see the `airbyte.secrets` module documentation.
228"""
229
230# Cloud Constants
231
232CLOUD_CLIENT_ID_ENV_VAR: str = "AIRBYTE_CLOUD_CLIENT_ID"
233"""The environment variable name for the Airbyte Cloud client ID."""
234
235CLOUD_CLIENT_SECRET_ENV_VAR: str = "AIRBYTE_CLOUD_CLIENT_SECRET"
236"""The environment variable name for the Airbyte Cloud client secret."""
237
238CLOUD_API_ROOT_ENV_VAR: str = "AIRBYTE_CLOUD_API_URL"
239"""The environment variable name for the Airbyte Cloud API URL."""
240
241CLOUD_CONFIG_API_ROOT_ENV_VAR: str = "AIRBYTE_CLOUD_CONFIG_API_URL"
242"""The environment variable name for the Airbyte Cloud Config API URL.
243
244The Config API is a separate internal API used for certain operations like
245connector builder projects and custom source definitions. This environment
246variable allows overriding the default Config API URL, which is useful when
247the public API URL has been overridden and the Config API cannot be derived
248from it automatically.
249"""
250
251CLOUD_WORKSPACE_ID_ENV_VAR: str = "AIRBYTE_CLOUD_WORKSPACE_ID"
252"""The environment variable name for the Airbyte Cloud workspace ID."""
253
254CLOUD_ORGANIZATION_ID_ENV_VAR: str = "AIRBYTE_CLOUD_ORGANIZATION_ID"
255"""The environment variable name for the Airbyte Cloud organization ID."""
256
257CLOUD_BEARER_TOKEN_ENV_VAR: str = "AIRBYTE_CLOUD_BEARER_TOKEN"
258"""The environment variable name for the Airbyte Cloud bearer token.
259
260When set, this bearer token will be used for authentication instead of
261client credentials (client_id + client_secret). This is useful when you
262already have a valid bearer token and want to skip the OAuth2 token exchange.
263"""
264
265CLOUD_API_ROOT: str = "https://api.airbyte.com/v1"
266"""The Airbyte Cloud API root URL.
267
268This is the root URL for the Airbyte Cloud API. It is used to interact with the Airbyte Cloud API
269and is the default API root for the `CloudWorkspace` class.
270- https://reference.airbyte.com/reference/getting-started
271"""
272
273CLOUD_CONFIG_API_ROOT: str = "https://cloud.airbyte.com/api/v1"
274"""Internal-Use API Root, aka Airbyte "Config API".
275
276Documentation:
277- https://docs.airbyte.com/api-documentation#configuration-api-deprecated
278- https://github.com/airbytehq/airbyte-platform-internal/blob/master/oss/airbyte-api/server-api/src/main/openapi/config.yaml
279"""
280
281# MCP (Model Context Protocol) Constants
282
283_HOSTED_MCP_MODE_ENABLED: bool = False
284"""Whether the process is serving MCP over hosted HTTP transport."""
285
286
287def set_hosted_mcp_mode() -> None:
288    """Set the flag indicating the process serves MCP over hosted HTTP transport."""
289    global _HOSTED_MCP_MODE_ENABLED
290    _HOSTED_MCP_MODE_ENABLED = True
291
292
293def is_hosted_mcp_mode() -> bool:
294    """Return True if the process serves MCP over hosted HTTP transport."""
295    return _HOSTED_MCP_MODE_ENABLED
296
297
298MCP_READONLY_MODE_ENV_VAR: str = "AIRBYTE_CLOUD_MCP_READONLY_MODE"
299"""Environment variable to enable read-only mode for the MCP server.
300
301When set to "1" or "true", only tools with readOnlyHint=True will be available.
302"""
303
304MCP_DOMAINS_DISABLED_ENV_VAR: str = "AIRBYTE_MCP_DOMAINS_DISABLED"
305"""Environment variable to disable specific MCP tool domains.
306
307Accepts a comma-separated list of domain names (e.g., "local,registry").
308Tools from these domains will not be advertised by the MCP server.
309"""
310
311MCP_DOMAINS_ENV_VAR: str = "AIRBYTE_MCP_DOMAINS"
312"""Environment variable to enable specific MCP tool domains.
313
314Accepts a comma-separated list of domain names (e.g., "cloud,registry").
315If set, only tools from these domains will be advertised by the MCP server.
316"""
317
318MCP_TRUSTED_EXECUTION_ENV_VAR: str = "AIRBYTE_MCP_TRUSTED_EXECUTION"
319"""Environment variable that enables trusted (local) execution for the MCP server.
320
321When set to `1`/`true`/`yes`, the server may use its trusted-machine capabilities: local
322filesystem access, local connector installation/execution, and server-side secret
323resolution. It defaults to *off* on every transport and is permanently unavailable over
324the HTTP transport (a hosted deployment can never enable it). This gate is server-owned
325and is deliberately never read from a request header, because it *widens* the surface and
326so must never be caller-controllable.
327"""
328
329MCP_WORKSPACE_ID_HEADER: str = "X-Airbyte-Workspace-Id"
330"""HTTP header key for passing workspace ID to the MCP server.
331
332This allows per-request workspace ID configuration when using HTTP transport.
333"""
334
335MCP_ORGANIZATION_ID_HEADER: str = "X-Airbyte-Organization-Id"
336"""HTTP header key for passing organization ID to the MCP server.
337
338This allows per-request organization ID configuration when using HTTP transport, for the
339tools that scope a listing to an organization rather than a workspace.
340"""
341
342MCP_INSIDERS_MODULES: frozenset[str] = frozenset({"agents"})
343"""MCP tool modules that are hidden unless insiders mode is enabled.
344
345Enable them with `AIRBYTE_MCP_INSIDERS` / `X-MCP-Insiders`, or by naming the module in
346the include list.
347"""
348
349MCP_INSIDERS_ENV_VAR: str = "AIRBYTE_MCP_INSIDERS"
350"""Environment variable that advertises insiders MCP tools. Off by default.
351
352Set to `1`/`true`/`yes` to advertise the tools in `MCP_INSIDERS_MODULES` to every
353caller, or to `0`/`false`/`no` to hide them from every caller. Either value overrides
354`MCP_INSIDERS_HEADER`; any other value, including an empty string, leaves the decision
355to that header.
356"""
357
358MCP_INSIDERS_HEADER: str = "X-MCP-Insiders"
359"""HTTP header key that advertises insiders MCP tools, per request.
360
361Set to `1`/`true`/`yes` to add the tools in `MCP_INSIDERS_MODULES` to the advertised
362tool surface. This selects which tools are advertised and is not an access-control
363boundary: every insiders tool authorizes each call against the Airbyte API.
364`MCP_INSIDERS_ENV_VAR` overrides this header when explicitly set.
365"""
366
367# MCP Config Arg Names (used with get_mcp_config)
368
369MCP_CONFIG_READONLY_MODE: str = "airbyte_readonly_mode"
370"""Config arg name for the legacy AIRBYTE_CLOUD_MCP_READONLY_MODE setting."""
371
372MCP_CONFIG_EXCLUDE_MODULES: str = "airbyte_exclude_modules"
373"""Config arg name for the legacy AIRBYTE_MCP_DOMAINS_DISABLED setting."""
374
375MCP_CONFIG_INCLUDE_MODULES: str = "airbyte_include_modules"
376"""Config arg name for the legacy AIRBYTE_MCP_DOMAINS setting."""
377
378MCP_CONFIG_WORKSPACE_ID: str = "workspace_id"
379"""Config arg name for the workspace ID setting."""
380
381MCP_CONFIG_ORGANIZATION_ID: str = "organization_id"
382"""Config arg name for the organization ID setting."""
383
384MCP_CONFIG_INSIDERS: str = "insiders"
385"""Config arg name for the insiders tools gate."""
386
387MCP_CONFIG_BEARER_TOKEN: str = "bearer_token"
388"""Config arg name for the bearer token setting."""
389
390MCP_CONFIG_CLIENT_ID: str = "client_id"
391"""Config arg name for the client ID setting."""
392
393MCP_CONFIG_CLIENT_SECRET: str = "client_secret"
394"""Config arg name for the client secret setting."""
395
396MCP_CONFIG_API_URL: str = "api_url"
397"""Config arg name for the API URL setting."""
398
399MCP_CONFIG_CONFIG_API_URL: str = "config_api_url"
400"""Config arg name for the Config API URL setting."""
401
402# MCP HTTP Header Keys for credentials
403
404MCP_BEARER_TOKEN_HEADER: str = "Authorization"
405"""HTTP header key for bearer token (standard Authorization header)."""
406
407MCP_EXTENSIONS_HEADER: str = "X-MCP-Extensions"
408"""HTTP header key for client-declared MCP extension IDs."""
409
410# Security Note: The API root and Config API root are intentionally NOT exposed as HTTP
411# headers. Each hosted MCP deployment is paired to a single backend, so allowing
412# a caller to override these URLs per-request would let them redirect the
413# server's credentialed requests to an arbitrary host and exfiltrate secrets.
414# These base URLs remain configurable via env var for local (stdio) use only.