airbyte.constants

  1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
  2"""Constants shared across the PyAirbyte codebase."""
  3
  4from __future__ import annotations
  5
  6import logging
  7import os
  8from pathlib import Path
  9
 10
 11logger = logging.getLogger("airbyte")
 12
 13
 14DEBUG_MODE = False  # Set to True to enable additional debug logging.
 15
 16AB_EXTRACTED_AT_COLUMN = "_airbyte_extracted_at"
 17"""A column that stores the timestamp when the record was extracted."""
 18
 19AB_META_COLUMN = "_airbyte_meta"
 20"""A column that stores metadata about the record."""
 21
 22AB_RAW_ID_COLUMN = "_airbyte_raw_id"
 23"""A column that stores a unique identifier for each row in the source data.
 24
 25Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations.
 26In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In PyAirbyte,
 27this column is simply used as a unique identifier for each record as it is received.
 28
 29PyAirbyte uses ULIDs for this column, which are identifiers that can be sorted by time
 30received. This allows us to determine the debug the order of records as they are received, even if
 31the source provides records that are tied or received out of order from the perspective of their
 32`emitted_at` (`_airbyte_extracted_at`) timestamps.
 33"""
 34
 35AB_INTERNAL_COLUMNS = {
 36    AB_RAW_ID_COLUMN,
 37    AB_EXTRACTED_AT_COLUMN,
 38    AB_META_COLUMN,
 39}
 40"""A set of internal columns that are reserved for PyAirbyte's internal use."""
 41
 42
 43def _try_create_dir_if_missing(path: Path, desc: str = "specified") -> Path:
 44    """Try to create a directory if it does not exist."""
 45    resolved_path = path.expanduser().resolve()
 46    try:
 47        if resolved_path.exists():
 48            if not resolved_path.is_dir():
 49                logger.warning(
 50                    "The %s path exists but is not a directory: '%s'", desc, resolved_path
 51                )
 52            return resolved_path
 53        resolved_path.mkdir(parents=True, exist_ok=True)
 54    except Exception as ex:
 55        logger.warning(
 56            "Could not auto-create missing %s directory at '%s': %s", desc, resolved_path, ex
 57        )
 58    return resolved_path
 59
 60
 61DEFAULT_PROJECT_DIR: Path = _try_create_dir_if_missing(
 62    Path(os.getenv("AIRBYTE_PROJECT_DIR", "") or Path.cwd()).expanduser().absolute(),
 63    desc="project",
 64)
 65"""Default project directory.
 66
 67Can be overridden by setting the `AIRBYTE_PROJECT_DIR` environment variable.
 68
 69If not set, defaults to the current working directory.
 70
 71This serves as the parent directory for both cache and install directories when not explicitly
 72configured.
 73
 74If a path is specified that does not yet exist, PyAirbyte will attempt to create it.
 75"""
 76
 77
 78DEFAULT_INSTALL_DIR: Path = _try_create_dir_if_missing(
 79    Path(os.getenv("AIRBYTE_INSTALL_DIR", "") or DEFAULT_PROJECT_DIR).expanduser().absolute(),
 80    desc="install",
 81)
 82"""Default install directory for connectors.
 83
 84If not set, defaults to `DEFAULT_PROJECT_DIR` (`AIRBYTE_PROJECT_DIR` env var) or the current
 85working directory if neither is set.
 86
 87If a path is specified that does not yet exist, PyAirbyte will attempt to create it.
 88"""
 89
 90
 91DEFAULT_CACHE_ROOT: Path = (
 92    (Path(os.getenv("AIRBYTE_CACHE_ROOT", "") or (DEFAULT_PROJECT_DIR / ".cache")))
 93    .expanduser()
 94    .absolute()
 95)
 96"""Default cache root is `.cache` in the current working directory.
 97
 98The default location can be overridden by setting the `AIRBYTE_CACHE_ROOT` environment variable.
 99
100Overriding this can be useful if you always want to store cache files in a specific location.
101For example, in ephemeral environments like Google Colab, you might want to store cache files in
102your mounted Google Drive by setting this to a path like `/content/drive/MyDrive/Airbyte/cache`.
103"""
104
105DEFAULT_CACHE_SCHEMA_NAME = "airbyte_raw"
106"""The default schema name to use for caches.
107
108Specific caches may override this value with a different schema name.
109"""
110
111DEFAULT_GOOGLE_DRIVE_MOUNT_PATH = "/content/drive"
112"""Default path to mount Google Drive in Google Colab environments."""
113
114DEFAULT_ARROW_MAX_CHUNK_SIZE = 100_000
115"""The default number of records to include in each batch of an Arrow dataset."""
116
117
118def _str_to_bool(value: str) -> bool:
119    """Convert a string value of an environment values to a boolean value."""
120    return bool(value) and value.lower() not in {"", "0", "false", "f", "no", "n", "off"}
121
122
123TEMP_DIR_OVERRIDE: Path | None = (
124    Path(os.environ["AIRBYTE_TEMP_DIR"]) if os.getenv("AIRBYTE_TEMP_DIR") else None
125)
126"""The directory to use for temporary files.
127
128This value is read from the `AIRBYTE_TEMP_DIR` environment variable. If the variable is not set,
129Tempfile will use the system's default temporary directory.
130
131This can be useful if you want to store temporary files in a specific location (or) when you
132need your temporary files to exist in user level directories, and not in system level
133directories for permissions reasons.
134"""
135
136TEMP_FILE_CLEANUP = _str_to_bool(
137    os.getenv(
138        key="AIRBYTE_TEMP_FILE_CLEANUP",
139        default="true",
140    )
141)
142"""Whether to clean up temporary files after use.
143
144This value is read from the `AIRBYTE_TEMP_FILE_CLEANUP` environment variable. If the variable is
145not set, the default value is `True`.
146"""
147
148AIRBYTE_OFFLINE_MODE = _str_to_bool(
149    os.getenv(
150        key="AIRBYTE_OFFLINE_MODE",
151        default="false",
152    )
153)
154"""Enable or disable offline mode.
155
156When offline mode is enabled, PyAirbyte will attempt to fetch metadata for connectors from the
157Airbyte registry but will not raise an error if the registry is unavailable. This can be useful in
158environments without internet access or with air-gapped networks.
159
160Offline mode also disables telemetry, similar to a `DO_NOT_TRACK` setting, ensuring no usage data
161is sent from your environment. You may also specify a custom registry URL via the`_REGISTRY_ENV_VAR`
162environment variable if you prefer to use a different registry source for metadata.
163
164This setting helps you make informed choices about data privacy and operation in restricted and
165air-gapped environments.
166"""
167
168AIRBYTE_PRINT_FULL_ERROR_LOGS: bool = _str_to_bool(
169    os.getenv(
170        key="AIRBYTE_PRINT_FULL_ERROR_LOGS",
171        default=os.getenv("CI", "false"),
172    )
173)
174"""Whether to print full error logs when an error occurs.
175This setting helps in debugging by providing detailed logs when errors occur. This is especially
176helpful in ephemeral environments like CI/CD pipelines where log files may not be persisted after
177the pipeline run.
178
179If not set, the default value is `False` for non-CI environments.
180If running in a CI environment ("CI" env var is set), then the default value is `True`.
181"""
182
183NO_UV: bool = os.getenv("AIRBYTE_NO_UV", "").lower() in {"1", "true", "yes"}
184"""Whether to disable uv and use pip for Python package management.
185
186This value is determined by the `AIRBYTE_NO_UV` environment variable. When `AIRBYTE_NO_UV`
187is set to "1", "true", or "yes", pip will be used instead of uv.
188
189If the variable is not set or set to any other value, uv will be used by default. Set this
190variable to opt out of uv and use pip instead.
191"""
192
193SECRETS_HYDRATION_PREFIX = "secret_reference::"
194"""Use this prefix to indicate a secret reference in configuration.
195
196For example, this snippet will populate the `personal_access_token` field with the value of the
197secret named `GITHUB_PERSONAL_ACCESS_TOKEN`, for instance from an environment variable.
198
199```json
200{
201  "credentials": {
202    "personal_access_token": "secret_reference::GITHUB_PERSONAL_ACCESS_TOKEN"
203  }
204}
205```
206
207For more information, see the `airbyte.secrets` module documentation.
208"""
209
210# Cloud Constants
211
212CLOUD_CLIENT_ID_ENV_VAR: str = "AIRBYTE_CLOUD_CLIENT_ID"
213"""The environment variable name for the Airbyte Cloud client ID."""
214
215CLOUD_CLIENT_SECRET_ENV_VAR: str = "AIRBYTE_CLOUD_CLIENT_SECRET"
216"""The environment variable name for the Airbyte Cloud client secret."""
217
218CLOUD_API_ROOT_ENV_VAR: str = "AIRBYTE_CLOUD_API_URL"
219"""The environment variable name for the Airbyte Cloud API URL."""
220
221CLOUD_CONFIG_API_ROOT_ENV_VAR: str = "AIRBYTE_CLOUD_CONFIG_API_URL"
222"""The environment variable name for the Airbyte Cloud Config API URL.
223
224The Config API is a separate internal API used for certain operations like
225connector builder projects and custom source definitions. This environment
226variable allows overriding the default Config API URL, which is useful when
227the public API URL has been overridden and the Config API cannot be derived
228from it automatically.
229"""
230
231CLOUD_WORKSPACE_ID_ENV_VAR: str = "AIRBYTE_CLOUD_WORKSPACE_ID"
232"""The environment variable name for the Airbyte Cloud workspace ID."""
233
234CLOUD_ORGANIZATION_ID_ENV_VAR: str = "AIRBYTE_CLOUD_ORGANIZATION_ID"
235"""The environment variable name for the Airbyte Cloud organization ID."""
236
237CLOUD_BEARER_TOKEN_ENV_VAR: str = "AIRBYTE_CLOUD_BEARER_TOKEN"
238"""The environment variable name for the Airbyte Cloud bearer token.
239
240When set, this bearer token will be used for authentication instead of
241client credentials (client_id + client_secret). This is useful when you
242already have a valid bearer token and want to skip the OAuth2 token exchange.
243"""
244
245CLOUD_API_ROOT: str = "https://api.airbyte.com/v1"
246"""The Airbyte Cloud API root URL.
247
248This is the root URL for the Airbyte Cloud API. It is used to interact with the Airbyte Cloud API
249and is the default API root for the `CloudWorkspace` class.
250- https://reference.airbyte.com/reference/getting-started
251"""
252
253CLOUD_CONFIG_API_ROOT: str = "https://cloud.airbyte.com/api/v1"
254"""Internal-Use API Root, aka Airbyte "Config API".
255
256Documentation:
257- https://docs.airbyte.com/api-documentation#configuration-api-deprecated
258- https://github.com/airbytehq/airbyte-platform-internal/blob/master/oss/airbyte-api/server-api/src/main/openapi/config.yaml
259"""
260
261# MCP (Model Context Protocol) Constants
262
263_HOSTED_MCP_MODE_ENABLED: bool = False
264"""Whether the process is serving MCP over hosted HTTP transport."""
265
266
267def set_hosted_mcp_mode() -> None:
268    """Set the flag indicating the process serves MCP over hosted HTTP transport."""
269    global _HOSTED_MCP_MODE_ENABLED
270    _HOSTED_MCP_MODE_ENABLED = True
271
272
273def is_hosted_mcp_mode() -> bool:
274    """Return True if the process serves MCP over hosted HTTP transport."""
275    return _HOSTED_MCP_MODE_ENABLED
276
277
278MCP_READONLY_MODE_ENV_VAR: str = "AIRBYTE_CLOUD_MCP_READONLY_MODE"
279"""Environment variable to enable read-only mode for the MCP server.
280
281When set to "1" or "true", only tools with readOnlyHint=True will be available.
282"""
283
284MCP_DOMAINS_DISABLED_ENV_VAR: str = "AIRBYTE_MCP_DOMAINS_DISABLED"
285"""Environment variable to disable specific MCP tool domains.
286
287Accepts a comma-separated list of domain names (e.g., "local,registry").
288Tools from these domains will not be advertised by the MCP server.
289"""
290
291MCP_DOMAINS_ENV_VAR: str = "AIRBYTE_MCP_DOMAINS"
292"""Environment variable to enable specific MCP tool domains.
293
294Accepts a comma-separated list of domain names (e.g., "cloud,registry").
295If set, only tools from these domains will be advertised by the MCP server.
296"""
297
298MCP_TRUSTED_EXECUTION_ENV_VAR: str = "AIRBYTE_MCP_TRUSTED_EXECUTION"
299"""Environment variable that enables trusted (local) execution for the MCP server.
300
301When set to `1`/`true`/`yes`, the server may use its trusted-machine capabilities: local
302filesystem access, local connector installation/execution, and server-side secret
303resolution. It defaults to *off* on every transport and is permanently unavailable over
304the HTTP transport (a hosted deployment can never enable it). This gate is server-owned
305and is deliberately never read from a request header, because it *widens* the surface and
306so must never be caller-controllable.
307"""
308
309MCP_WORKSPACE_ID_HEADER: str = "X-Airbyte-Workspace-Id"
310"""HTTP header key for passing workspace ID to the MCP server.
311
312This allows per-request workspace ID configuration when using HTTP transport.
313"""
314
315# MCP Config Arg Names (used with get_mcp_config)
316
317MCP_CONFIG_READONLY_MODE: str = "airbyte_readonly_mode"
318"""Config arg name for the legacy AIRBYTE_CLOUD_MCP_READONLY_MODE setting."""
319
320MCP_CONFIG_EXCLUDE_MODULES: str = "airbyte_exclude_modules"
321"""Config arg name for the legacy AIRBYTE_MCP_DOMAINS_DISABLED setting."""
322
323MCP_CONFIG_INCLUDE_MODULES: str = "airbyte_include_modules"
324"""Config arg name for the legacy AIRBYTE_MCP_DOMAINS setting."""
325
326MCP_CONFIG_WORKSPACE_ID: str = "workspace_id"
327"""Config arg name for the workspace ID setting."""
328
329MCP_CONFIG_BEARER_TOKEN: str = "bearer_token"
330"""Config arg name for the bearer token setting."""
331
332MCP_CONFIG_CLIENT_ID: str = "client_id"
333"""Config arg name for the client ID setting."""
334
335MCP_CONFIG_CLIENT_SECRET: str = "client_secret"
336"""Config arg name for the client secret setting."""
337
338MCP_CONFIG_API_URL: str = "api_url"
339"""Config arg name for the API URL setting."""
340
341MCP_CONFIG_CONFIG_API_URL: str = "config_api_url"
342"""Config arg name for the Config API URL setting."""
343
344# MCP HTTP Header Keys for credentials
345
346MCP_BEARER_TOKEN_HEADER: str = "Authorization"
347"""HTTP header key for bearer token (standard Authorization header)."""
348
349MCP_EXTENSIONS_HEADER: str = "X-MCP-Extensions"
350"""HTTP header key for client-declared MCP extension IDs."""
351
352MCP_CLIENT_ID_HEADER: str = "X-Airbyte-Cloud-Client-Id"
353"""HTTP header key for client ID."""
354
355MCP_CLIENT_SECRET_HEADER: str = "X-Airbyte-Cloud-Client-Secret"
356"""HTTP header key for client secret."""
357
358# Security Note: The API root and Config API root are intentionally NOT exposed as HTTP
359# headers. Each hosted MCP deployment is paired to a single backend, so allowing
360# a caller to override these URLs per-request would let them redirect the
361# server's credentialed requests to an arbitrary host and exfiltrate secrets.
362# These base URLs remain configurable via env var for local (stdio) use only.