airbyte_ops_mcp.connection_config_retriever

Connection config retriever module.

This module provides functionality to retrieve unmasked connection configuration from Airbyte Cloud's internal database, including secret resolution from GCP Secret Manager and audit logging to GCP Cloud Logging.

Refactored from: live_tests/_connection_retriever Original source: airbyte-platform-internal/tools/connection-retriever

 1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 2"""Connection config retriever module.
 3
 4This module provides functionality to retrieve unmasked connection configuration
 5from Airbyte Cloud's internal database, including secret resolution from GCP
 6Secret Manager and audit logging to GCP Cloud Logging.
 7
 8Refactored from: live_tests/_connection_retriever
 9Original source: airbyte-platform-internal/tools/connection-retriever
10"""
11
12from airbyte_ops_mcp.connection_config_retriever.retrieval import (
13    ConnectionNotFoundError,
14    RetrievalMetadata,
15    TestingCandidate,
16    get_connection_dataplane_group,
17    get_connection_residency,
18    residency_for_dataplane_group,
19    retrieve_objects,
20)
21from airbyte_ops_mcp.constants import ConnectionObject
22
23__all__ = [
24    "ConnectionNotFoundError",
25    "ConnectionObject",
26    "RetrievalMetadata",
27    "TestingCandidate",
28    "get_connection_dataplane_group",
29    "get_connection_residency",
30    "residency_for_dataplane_group",
31    "retrieve_objects",
32]
class ConnectionNotFoundError(builtins.Exception):
191class ConnectionNotFoundError(Exception):
192    """Raised when a connection cannot be found."""
193
194    pass

Raised when a connection cannot be found.

class ConnectionObject(enum.Enum):
227class ConnectionObject(Enum):
228    """Types of connection objects that can be retrieved."""
229
230    CONNECTION = "connection"
231    SOURCE_ID = "source-id"
232    DESTINATION_ID = "destination-id"
233    DESTINATION_CONFIG = "destination-config"
234    SOURCE_CONFIG = "source-config"
235    CATALOG = "catalog"
236    CONFIGURED_CATALOG = "configured-catalog"
237    STATE = "state"
238    WORKSPACE_ID = "workspace-id"
239    DESTINATION_DOCKER_IMAGE = "destination-docker-image"
240    SOURCE_DOCKER_IMAGE = "source-docker-image"

Types of connection objects that can be retrieved.

CONNECTION = <ConnectionObject.CONNECTION: 'connection'>
SOURCE_ID = <ConnectionObject.SOURCE_ID: 'source-id'>
DESTINATION_ID = <ConnectionObject.DESTINATION_ID: 'destination-id'>
DESTINATION_CONFIG = <ConnectionObject.DESTINATION_CONFIG: 'destination-config'>
SOURCE_CONFIG = <ConnectionObject.SOURCE_CONFIG: 'source-config'>
CATALOG = <ConnectionObject.CATALOG: 'catalog'>
CONFIGURED_CATALOG = <ConnectionObject.CONFIGURED_CATALOG: 'configured-catalog'>
STATE = <ConnectionObject.STATE: 'state'>
WORKSPACE_ID = <ConnectionObject.WORKSPACE_ID: 'workspace-id'>
DESTINATION_DOCKER_IMAGE = <ConnectionObject.DESTINATION_DOCKER_IMAGE: 'destination-docker-image'>
SOURCE_DOCKER_IMAGE = <ConnectionObject.SOURCE_DOCKER_IMAGE: 'source-docker-image'>
@dataclass
class RetrievalMetadata:
145@dataclass
146class RetrievalMetadata:
147    """Metadata about a retrieval operation for audit logging."""
148
149    connection_id: str
150    connection_object: ConnectionObject
151    retrieval_reason: str

Metadata about a retrieval operation for audit logging.

RetrievalMetadata( connection_id: str, connection_object: ConnectionObject, retrieval_reason: str)
connection_id: str
connection_object: ConnectionObject
retrieval_reason: str
@dataclass
class TestingCandidate:
154@dataclass
155class TestingCandidate:
156    """A connection candidate for testing."""
157
158    connection_id: str
159    connection_url: str | None = None
160    stream_count: int | None = None
161    last_attempt_duration_in_microseconds: int | None = None
162    is_internal: bool | None = None
163    streams_with_data: list[str] | None = None
164
165    # ConnectionObject fields
166    connection: str | None = None
167    source_id: str | None = None
168    destination_id: str | None = None
169
170    destination_config: Mapping | None = None
171    source_config: Mapping | None = None
172    catalog: Mapping | None = None
173    configured_catalog: Mapping | None = None
174    state: list[Mapping] | None = None
175
176    workspace_id: str | None = None
177    destination_docker_image: str | None = None
178    source_docker_image: str | None = None
179
180    def update(self, **kwargs: Any) -> None:
181        """Update fields from keyword arguments."""
182        for key, value in kwargs.items():
183            if hasattr(self, key):
184                setattr(self, key, value)
185            else:
186                raise AttributeError(
187                    f"{key} is not a valid field of {self.__class__.__name__}"
188                )

A connection candidate for testing.

TestingCandidate( connection_id: str, connection_url: str | None = None, stream_count: int | None = None, last_attempt_duration_in_microseconds: int | None = None, is_internal: bool | None = None, streams_with_data: list[str] | None = None, connection: str | None = None, source_id: str | None = None, destination_id: str | None = None, destination_config: Optional[Mapping] = None, source_config: Optional[Mapping] = None, catalog: Optional[Mapping] = None, configured_catalog: Optional[Mapping] = None, state: list[typing.Mapping] | None = None, workspace_id: str | None = None, destination_docker_image: str | None = None, source_docker_image: str | None = None)
connection_id: str
connection_url: str | None = None
stream_count: int | None = None
last_attempt_duration_in_microseconds: int | None = None
is_internal: bool | None = None
streams_with_data: list[str] | None = None
connection: str | None = None
source_id: str | None = None
destination_id: str | None = None
destination_config: Optional[Mapping] = None
source_config: Optional[Mapping] = None
catalog: Optional[Mapping] = None
configured_catalog: Optional[Mapping] = None
state: list[typing.Mapping] | None = None
workspace_id: str | None = None
destination_docker_image: str | None = None
source_docker_image: str | None = None
def update(self, **kwargs: Any) -> None:
180    def update(self, **kwargs: Any) -> None:
181        """Update fields from keyword arguments."""
182        for key, value in kwargs.items():
183            if hasattr(self, key):
184                setattr(self, key, value)
185            else:
186                raise AttributeError(
187                    f"{key} is not a valid field of {self.__class__.__name__}"
188                )

Update fields from keyword arguments.

def get_connection_dataplane_group(connection_id: str) -> str:
443def get_connection_dataplane_group(connection_id: str) -> str:
444    """Get the name of the connection's dataplane group.
445
446    The query does not filter out tombstoned connections, actors or workspaces,
447    so a deleted EU connection still reads as EU and fails closed.
448
449    Raises:
450        ConnectionNotFoundError: No connection has this ID.
451        ValueError: The connection ID is not a UUID, or the dataplane group
452            name is empty or NULL, so residency cannot be decided.
453    """
454    try:
455        uuid.UUID(connection_id)
456    except ValueError:
457        raise ValueError(
458            f"Connection ID '{connection_id}' is not a valid UUID."
459        ) from None
460
461    secret_manager_client = get_secret_manager_client()
462    connection_pool = get_pool(secret_manager_client)
463
464    with connection_pool.connect() as db_conn:
465        result = db_conn.execute(
466            SELECT_ON_CONNECTION_DATAPLANE_GROUP_NAME,
467            parameters={"connection_id": connection_id},
468        ).first()
469
470    if result is None:
471        raise ConnectionNotFoundError(f"Connection {connection_id} was not found.")
472    dataplane_group = result[0]
473    if not dataplane_group or not dataplane_group.strip():
474        raise ValueError(
475            f"Connection {connection_id} has no dataplane group name, "
476            "so its residency cannot be decided."
477        )
478    return dataplane_group

Get the name of the connection's dataplane group.

The query does not filter out tombstoned connections, actors or workspaces, so a deleted EU connection still reads as EU and fails closed.

Raises:
  • ConnectionNotFoundError: No connection has this ID.
  • ValueError: The connection ID is not a UUID, or the dataplane group name is empty or NULL, so residency cannot be decided.
def get_connection_residency(connection_id: str) -> Literal['eu', 'us']:
486def get_connection_residency(connection_id: str) -> Literal["eu", "us"]:
487    """Get a connection's data residency, `"eu"` or `"us"`.
488
489    The one place that decides residency for a connection ID. It raises the
490    errors of `get_connection_dataplane_group` rather than guessing a residency.
491    """
492    return residency_for_dataplane_group(get_connection_dataplane_group(connection_id))

Get a connection's data residency, "eu" or "us".

The one place that decides residency for a connection ID. It raises the errors of get_connection_dataplane_group rather than guessing a residency.

def residency_for_dataplane_group(name: str) -> Literal['eu', 'us']:
481def residency_for_dataplane_group(name: str) -> Literal["eu", "us"]:
482    """Map the EU dataplane group to EU residency and all others to US."""
483    return "eu" if name == "EU" else "us"

Map the EU dataplane group to EU residency and all others to US.

def retrieve_objects( connection_objects: list[ConnectionObject], retrieval_reason: str, connection_id: str) -> list[TestingCandidate]:
347def retrieve_objects(
348    connection_objects: list[ConnectionObject],
349    retrieval_reason: str,
350    connection_id: str,
351) -> list[TestingCandidate]:
352    """Retrieve connection objects for a given connection ID.
353
354    This is a simplified version that only supports retrieval by connection_id.
355    For testing candidate discovery by docker image, see issue #91.
356    """
357    connection_candidates = [TestingCandidate(connection_id=connection_id)]
358
359    secret_manager_client = get_secret_manager_client()
360    connection_pool = get_pool(secret_manager_client)
361
362    with connection_pool.connect() as db_conn:
363        for candidate in connection_candidates.copy():
364            is_eu_result = db_conn.execute(
365                SELECT_ON_CONNECTION_DATAPLANE_GROUP_IS_EU,
366                parameters={"connection_id": candidate.connection_id},
367            ).first()
368            if is_eu_result is None:
369                raise ConnectionNotFoundError(
370                    f"Credentials were not found for connection ID {candidate.connection_id}."
371                )
372            is_eu_connection = is_eu_result[0] is True
373            # Each runtime reads only connections in its own data region.
374            if is_eu_connection != is_running_in_eu_region():
375                connection_candidates.remove(candidate)
376                refusal = (
377                    EU_REGION_REQUIRED_MESSAGE
378                    if is_eu_connection
379                    else EU_REGION_EU_ONLY_MESSAGE
380                )
381                LOGGER.warning(
382                    f"{refusal}; connection ID "
383                    f"{candidate.connection_id} was removed from candidates."
384                )
385                continue
386
387            candidate.update(
388                **{
389                    connection_object.value.replace("-", "_"): retrieve_object(
390                        candidate.connection_id,
391                        connection_object,
392                        retrieval_reason,
393                        db_conn,
394                        secret_manager_client,
395                        is_eu_connection=is_eu_connection,
396                    )
397                    for connection_object in connection_objects
398                }
399            )
400
401    return connection_candidates

Retrieve connection objects for a given connection ID.

This is a simplified version that only supports retrieval by connection_id. For testing candidate discovery by docker image, see issue #91.