airbyte_ops_mcp.cli.secrets

  1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
  2"""CLI commands for connector integration-test secrets.
  3
  4Commands:
  5    airbyte-ops secrets create   - Create a connector secret in GSM
  6    airbyte-ops secrets fetch   - Download connector secrets from GSM into `./secrets/`
  7    airbyte-ops secrets list    - List connector secrets visible in GSM
  8    airbyte-ops secrets update  - Add a version and disable previous enabled versions
  9    airbyte-ops secrets ci-mask - Emit GitHub Actions `::add-mask::` lines
 10
 11These commands operate on the connector integration-test secrets stored in
 12Google Secret Manager (project `dataline-integration-testing` by default).
 13They are intended to replace the `airbyte-cdk secrets` group in a subsequent
 14phase; the CDK commands remain functional until that migration lands.
 15
 16> **Note:** `airbyte-ops devin secret-*` is a different subsystem for
 17> Devin-internal 1Password on-demand secrets and is unrelated to this group.
 18
 19## CLI reference
 20
 21The commands below are regenerated by `poe docs-generate` via cyclopts's
 22programmatic docs API; see `docs/generate_cli.py`.
 23
 24.. include:: ../../../docs/generated/cli/secrets.md
 25   :start-line: 2
 26"""
 27
 28from __future__ import annotations
 29
 30__all__: list[str] = []
 31
 32from pathlib import Path
 33from typing import Annotated
 34
 35from airbyte_cdk.utils.connector_paths import resolve_connector_name_and_directory
 36from cyclopts import Parameter
 37from rich.console import Console
 38from rich.table import Table
 39
 40from airbyte_ops_mcp.cli._base import App, app
 41from airbyte_ops_mcp.connector_secrets import (
 42    DEFAULT_GCP_PROJECT_ID,
 43    ConnectorSecretWithNoValidVersionsError,
 44    create_secret,
 45    extract_gcp_secret_name,
 46    fetch_secret_handles,
 47    get_connector_secret,
 48    get_gcp_secret_url,
 49    get_gsm_secrets_client,
 50    get_gsm_writer_client,
 51    get_secret_filepath,
 52    get_secrets_dir,
 53    update_secret,
 54    write_secret_file,
 55)
 56from airbyte_ops_mcp.connector_secrets.ci_masks import (
 57    is_running_in_ci,
 58    print_ci_secrets_masks,
 59)
 60
 61error_console = Console(stderr=True)
 62
 63secrets_app = App(
 64    name="secrets", help="Manage connector integration-test secrets in GSM."
 65)
 66app.command(secrets_app)
 67
 68_CONNECTOR_HELP = (
 69    "Connector name (e.g. 'source-pokeapi') or path to a connector directory. "
 70    "If omitted, the current working directory is used and must be a connector "
 71    "directory."
 72)
 73_WRITE_CONNECTOR_HELP = (
 74    "Required connector name (e.g. 'source-pokeapi') or path to a connector directory."
 75)
 76_GCP_PROJECT_ID_HELP = (
 77    "GCP project ID for retrieving integration-test credentials. Defaults to "
 78    "the `GCP_PROJECT_ID` environment variable, or 'dataline-integration-testing' "
 79    "when unset."
 80)
 81_SECRET_NAME_HELP = (
 82    "GSM secret name. New names must match "
 83    "`SECRET_<CONNECTOR_NAME_UPPERCASE>__<SUFFIX>`."
 84)
 85_CONFIG_FILE_HELP = "Path to the JSON config file to write as the new secret version."
 86
 87
 88def _resolve_connector_for_write(
 89    connector: str,
 90    config_file: Path | None,
 91) -> tuple[str, Path | None]:
 92    """Resolve a connector name and directory, accepting bare names with a config file."""
 93    if (
 94        config_file is not None
 95        and "/" not in connector
 96        and "\\" not in connector
 97        and not Path(connector).is_dir()
 98    ):
 99        return connector, None
100    connector_name, connector_directory = resolve_connector_name_and_directory(
101        connector
102    )
103    return connector_name, connector_directory
104
105
106def _require_config_file(config_file: Path) -> str:
107    """Read a config file or raise a clear path-only error."""
108    if not config_file.is_file():
109        raise FileNotFoundError(f"Config file not found: {config_file}")
110    try:
111        return config_file.read_bytes().decode("utf-8")
112    except UnicodeDecodeError:
113        raise ValueError(f"Config file is not valid UTF-8: {config_file}") from None
114
115
116@secrets_app.command(name="create")
117def create(
118    connector: Annotated[str, Parameter(help=_WRITE_CONNECTOR_HELP)],
119    *,
120    secret_name: Annotated[
121        str, Parameter(name="--secret-name", help=_SECRET_NAME_HELP)
122    ],
123    filename: Annotated[
124        str | None,
125        Parameter(
126            name="--filename",
127            help="Optional lowercase filename label used under the connector's `secrets/` directory.",
128        ),
129    ] = None,
130    config_file: Annotated[
131        Path | None, Parameter(name="--config-file", help=_CONFIG_FILE_HELP)
132    ] = None,
133    gcp_project_id: Annotated[
134        str, Parameter(help=_GCP_PROJECT_ID_HELP)
135    ] = DEFAULT_GCP_PROJECT_ID,
136) -> None:
137    """Create a new connector integration-test secret in GSM.
138
139    The command fails if the secret already exists or if its resolved
140    `secrets/<filename>.json` path conflicts with another secret. Secret names
141    must follow `SECRET_<CONNECTOR_NAME_UPPERCASE>__<SUFFIX>`. Credentials are
142    read from `GCP_GSM_WRITER_CREDENTIALS` (or ADC); payloads must be non-empty
143    JSON objects.
144    """
145    connector_name, connector_directory = _resolve_connector_for_write(
146        connector, config_file
147    )
148    if config_file is None:
149        if connector_directory is None:
150            raise ValueError(
151                "A connector directory is required when `--config-file` is not set."
152            )
153        secrets_dir = get_secrets_dir(
154            connector_directory=connector_directory,
155            ensure_exists=False,
156        )
157        config_file = secrets_dir / f"{filename or 'config'}.json"
158
159    value = _require_config_file(config_file)
160    client = get_gsm_writer_client()
161    version_name = create_secret(
162        connector_name=connector_name,
163        value=value,
164        secret_name=secret_name,
165        filename=filename,
166        gcp_project_id=gcp_project_id,
167        client=client,
168    )
169    secret_url = get_gcp_secret_url(secret_name, gcp_project_id)
170    error_console.print(
171        f"{secret_name} version {version_name.rsplit('/', 1)[-1]}: {secret_url}"
172    )
173
174
175@secrets_app.command(name="update")
176def update(
177    connector: Annotated[str, Parameter(help=_WRITE_CONNECTOR_HELP)],
178    *,
179    secret_name: Annotated[
180        str, Parameter(name="--secret-name", help=_SECRET_NAME_HELP)
181    ],
182    config_file: Annotated[
183        Path | None, Parameter(name="--config-file", help=_CONFIG_FILE_HELP)
184    ] = None,
185    gcp_project_id: Annotated[
186        str, Parameter(help=_GCP_PROJECT_ID_HELP)
187    ] = DEFAULT_GCP_PROJECT_ID,
188) -> None:
189    """Add a new version to an existing connector integration-test secret.
190
191    The command fails if the secret is missing or its connector label does not
192    match. After adding a new version, it disables versions that were previously
193    enabled. If disabling fails, the new version remains latest and the error
194    lists versions that still need disabling. Credentials are read from
195    `GCP_GSM_WRITER_CREDENTIALS` (or ADC); payloads must be non-empty JSON
196    objects.
197    """
198    connector_name, connector_directory = _resolve_connector_for_write(
199        connector, config_file
200    )
201    client = get_gsm_writer_client()
202    secret = get_connector_secret(
203        connector_name=connector_name,
204        secret_name=secret_name,
205        gcp_project_id=gcp_project_id,
206        client=client,
207    )
208    if config_file is None:
209        if connector_directory is None:
210            raise ValueError(
211                "A connector directory is required when `--config-file` is not set."
212            )
213        secrets_dir = get_secrets_dir(
214            connector_directory=connector_directory,
215            ensure_exists=False,
216        )
217        config_file = get_secret_filepath(secrets_dir=secrets_dir, secret=secret)
218
219    value = _require_config_file(config_file)
220    version_name = update_secret(
221        connector_name=connector_name,
222        value=value,
223        secret_name=secret_name,
224        gcp_project_id=gcp_project_id,
225        client=client,
226    )
227    secret_url = get_gcp_secret_url(secret_name, gcp_project_id)
228    error_console.print(
229        f"{secret_name} version {version_name.rsplit('/', 1)[-1]}: {secret_url}"
230    )
231
232
233@secrets_app.command(name="fetch")
234def fetch(
235    connector: Annotated[str | None, Parameter(help=_CONNECTOR_HELP)] = None,
236    *,
237    gcp_project_id: Annotated[
238        str, Parameter(help=_GCP_PROJECT_ID_HELP)
239    ] = DEFAULT_GCP_PROJECT_ID,
240    print_ci_secrets_masks_flag: Annotated[
241        bool | None,
242        Parameter(
243            name=["--print-ci-secrets-masks"],
244            negative=["--no-print-ci-secrets-masks"],
245            help=(
246                "Emit GitHub Actions `::add-mask::` lines for fetched secrets. "
247                "Ignored outside CI (requires the `CI` env var). Defaults to "
248                "auto-detection from `CI`."
249            ),
250        ),
251    ] = None,
252) -> None:
253    """Fetch connector secrets from GSM into the connector's `secrets/` directory.
254
255    Equivalent to the old `airbyte-cdk secrets fetch`. Runs in three steps:
256
257    1. Resolve the connector (from `[CONNECTOR]`, or the cwd).
258    2. List GSM secrets labeled `connector=<name>` in `--gcp-project-id`.
259    3. Write each secret's latest enabled version to `secrets/<filename>.json`
260       (mode `0o600`). The filename comes from the secret's `filename` label,
261       or defaults to `config.json`.
262
263    When `--print-ci-secrets-masks` is set (or auto-detected in CI), emits
264    GitHub Actions masking commands after writing.
265    """
266    error_console.print("Fetching secrets...")
267
268    client = get_gsm_secrets_client()
269    connector_name, connector_directory = resolve_connector_name_and_directory(
270        connector
271    )
272    secrets_dir = get_secrets_dir(
273        connector_directory=connector_directory,
274        ensure_exists=True,
275    )
276    secrets = fetch_secret_handles(
277        connector_name=connector_name,
278        gcp_project_id=gcp_project_id,
279        client=client,
280    )
281
282    secret_count = 0
283    exceptions: list[ConnectorSecretWithNoValidVersionsError] = []
284
285    for secret in secrets:
286        secret_file_path = get_secret_filepath(secrets_dir=secrets_dir, secret=secret)
287        try:
288            write_secret_file(
289                secret=secret,
290                client=client,
291                file_path=secret_file_path,
292                connector_name=connector_name,
293                gcp_project_id=gcp_project_id,
294            )
295        except ConnectorSecretWithNoValidVersionsError as err:
296            exceptions.append(err)
297            error_console.print(
298                f"Failed to retrieve secret '{err.secret_name}': No enabled version found"
299            )
300            continue
301
302        error_console.print(f"Secret written to: {secret_file_path.absolute()!s}")
303        secret_count += 1
304
305    if secret_count == 0 and not exceptions:
306        error_console.print(f"No secrets found for connector: '{connector_name}'")
307
308    if exceptions:
309        error_console.print(
310            f"[red]Failed to retrieve {len(exceptions)} secret(s)[/red]"
311        )
312        if secret_count == 0:
313            raise exceptions[0]
314
315    emit_masks = print_ci_secrets_masks_flag
316    if emit_masks is None:
317        emit_masks = is_running_in_ci()
318
319    if emit_masks:
320        print_ci_secrets_masks(
321            secrets_dir=secrets_dir,
322            strict_ci_env_check=True,
323        )
324
325
326@secrets_app.command(name="list")
327def list_(
328    connector: Annotated[str | None, Parameter(help=_CONNECTOR_HELP)] = None,
329    *,
330    gcp_project_id: Annotated[
331        str, Parameter(help=_GCP_PROJECT_ID_HELP)
332    ] = DEFAULT_GCP_PROJECT_ID,
333) -> None:
334    """List connector secrets visible in GSM without downloading any values.
335
336    Prints a Rich table with the secret name (linked to its GCP console page),
337    labels, and creation time. Only metadata is read — no secret payloads are
338    fetched.
339    """
340    error_console.print("Scanning secrets...")
341
342    if connector and "/" not in connector and "\\" not in connector:
343        connector_name = connector
344    else:
345        connector_name, _ = resolve_connector_name_and_directory(connector)
346
347    secrets = fetch_secret_handles(
348        connector_name=connector_name,
349        gcp_project_id=gcp_project_id,
350    )
351    if not secrets:
352        error_console.print(f"No secrets found for connector: '{connector_name}'")
353        return
354
355    console = Console()
356    console.print(
357        f"[green]Secrets for connector '{connector_name}' in project "
358        f"'{gcp_project_id}':[/green]"
359    )
360    table = Table(title=f"'{connector_name}' Secrets")
361    table.add_column("Name", justify="left", style="cyan", overflow="fold")
362    table.add_column("Labels", justify="left", style="magenta", overflow="fold")
363    table.add_column("Created", justify="left", style="blue", overflow="fold")
364    for secret in secrets:
365        secret_name = extract_gcp_secret_name(secret.name)
366        secret_url = get_gcp_secret_url(secret_name, gcp_project_id)
367        table.add_row(
368            f"[link={secret_url}]{secret_name}[/link]",
369            "\n".join(f"{k}={v}" for k, v in secret.labels.items()),
370            str(secret.create_time),
371        )
372    console.print(table)
373
374
375@secrets_app.command(name="ci-mask")
376def ci_mask(
377    connector: Annotated[str | None, Parameter(help=_CONNECTOR_HELP)] = None,
378) -> None:
379    """Emit GitHub Actions `::add-mask::` lines for a connector's fetched secrets.
380
381    Reads every `*.json` file in the connector's local `secrets/` directory and
382    prints a mask command for each secret-valued property (matched against the
383    canonical
384    [`specs_secrets_mask.yaml`](https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml)
385    list). Intended to run inside a GitHub Actions job; refuses to emit output
386    when `CI` is unset to prevent accidentally printing secret values locally.
387
388    This subcommand is equivalent to running `airbyte-ops secrets fetch
389    --print-ci-secrets-masks` without re-downloading secrets.
390    """
391    _, connector_directory = resolve_connector_name_and_directory(connector)
392    secrets_dir = get_secrets_dir(
393        connector_directory=connector_directory,
394        ensure_exists=False,
395    )
396    if not secrets_dir.is_dir():
397        error_console.print(
398            f"No secrets directory found at {secrets_dir!s}; nothing to mask."
399        )
400        return
401
402    print_ci_secrets_masks(
403        secrets_dir=secrets_dir,
404        strict_ci_env_check=True,
405    )