airbyte_ops_mcp.connection_config_retriever
Connection config retriever module.
This module provides functionality to retrieve unmasked connection configuration from Airbyte Cloud's internal database, including secret resolution from GCP Secret Manager and audit logging to GCP Cloud Logging.
Refactored from: live_tests/_connection_retriever Original source: airbyte-platform-internal/tools/connection-retriever
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Connection config retriever module. 3 4This module provides functionality to retrieve unmasked connection configuration 5from Airbyte Cloud's internal database, including secret resolution from GCP 6Secret Manager and audit logging to GCP Cloud Logging. 7 8Refactored from: live_tests/_connection_retriever 9Original source: airbyte-platform-internal/tools/connection-retriever 10""" 11 12from airbyte_ops_mcp.connection_config_retriever.retrieval import ( 13 ConnectionNotFoundError, 14 RetrievalMetadata, 15 TestingCandidate, 16 get_connection_dataplane_group, 17 get_connection_residency, 18 residency_for_dataplane_group, 19 retrieve_objects, 20) 21from airbyte_ops_mcp.constants import ConnectionObject 22 23__all__ = [ 24 "ConnectionNotFoundError", 25 "ConnectionObject", 26 "RetrievalMetadata", 27 "TestingCandidate", 28 "get_connection_dataplane_group", 29 "get_connection_residency", 30 "residency_for_dataplane_group", 31 "retrieve_objects", 32]
191class ConnectionNotFoundError(Exception): 192 """Raised when a connection cannot be found.""" 193 194 pass
Raised when a connection cannot be found.
227class ConnectionObject(Enum): 228 """Types of connection objects that can be retrieved.""" 229 230 CONNECTION = "connection" 231 SOURCE_ID = "source-id" 232 DESTINATION_ID = "destination-id" 233 DESTINATION_CONFIG = "destination-config" 234 SOURCE_CONFIG = "source-config" 235 CATALOG = "catalog" 236 CONFIGURED_CATALOG = "configured-catalog" 237 STATE = "state" 238 WORKSPACE_ID = "workspace-id" 239 DESTINATION_DOCKER_IMAGE = "destination-docker-image" 240 SOURCE_DOCKER_IMAGE = "source-docker-image"
Types of connection objects that can be retrieved.
145@dataclass 146class RetrievalMetadata: 147 """Metadata about a retrieval operation for audit logging.""" 148 149 connection_id: str 150 connection_object: ConnectionObject 151 retrieval_reason: str
Metadata about a retrieval operation for audit logging.
154@dataclass 155class TestingCandidate: 156 """A connection candidate for testing.""" 157 158 connection_id: str 159 connection_url: str | None = None 160 stream_count: int | None = None 161 last_attempt_duration_in_microseconds: int | None = None 162 is_internal: bool | None = None 163 streams_with_data: list[str] | None = None 164 165 # ConnectionObject fields 166 connection: str | None = None 167 source_id: str | None = None 168 destination_id: str | None = None 169 170 destination_config: Mapping | None = None 171 source_config: Mapping | None = None 172 catalog: Mapping | None = None 173 configured_catalog: Mapping | None = None 174 state: list[Mapping] | None = None 175 176 workspace_id: str | None = None 177 destination_docker_image: str | None = None 178 source_docker_image: str | None = None 179 180 def update(self, **kwargs: Any) -> None: 181 """Update fields from keyword arguments.""" 182 for key, value in kwargs.items(): 183 if hasattr(self, key): 184 setattr(self, key, value) 185 else: 186 raise AttributeError( 187 f"{key} is not a valid field of {self.__class__.__name__}" 188 )
A connection candidate for testing.
180 def update(self, **kwargs: Any) -> None: 181 """Update fields from keyword arguments.""" 182 for key, value in kwargs.items(): 183 if hasattr(self, key): 184 setattr(self, key, value) 185 else: 186 raise AttributeError( 187 f"{key} is not a valid field of {self.__class__.__name__}" 188 )
Update fields from keyword arguments.
443def get_connection_dataplane_group(connection_id: str) -> str: 444 """Get the name of the connection's dataplane group. 445 446 The query does not filter out tombstoned connections, actors or workspaces, 447 so a deleted EU connection still reads as EU and fails closed. 448 449 Raises: 450 ConnectionNotFoundError: No connection has this ID. 451 ValueError: The connection ID is not a UUID, or the dataplane group 452 name is empty or NULL, so residency cannot be decided. 453 """ 454 try: 455 uuid.UUID(connection_id) 456 except ValueError: 457 raise ValueError( 458 f"Connection ID '{connection_id}' is not a valid UUID." 459 ) from None 460 461 secret_manager_client = get_secret_manager_client() 462 connection_pool = get_pool(secret_manager_client) 463 464 with connection_pool.connect() as db_conn: 465 result = db_conn.execute( 466 SELECT_ON_CONNECTION_DATAPLANE_GROUP_NAME, 467 parameters={"connection_id": connection_id}, 468 ).first() 469 470 if result is None: 471 raise ConnectionNotFoundError(f"Connection {connection_id} was not found.") 472 dataplane_group = result[0] 473 if not dataplane_group or not dataplane_group.strip(): 474 raise ValueError( 475 f"Connection {connection_id} has no dataplane group name, " 476 "so its residency cannot be decided." 477 ) 478 return dataplane_group
Get the name of the connection's dataplane group.
The query does not filter out tombstoned connections, actors or workspaces, so a deleted EU connection still reads as EU and fails closed.
Raises:
- ConnectionNotFoundError: No connection has this ID.
- ValueError: The connection ID is not a UUID, or the dataplane group name is empty or NULL, so residency cannot be decided.
486def get_connection_residency(connection_id: str) -> Literal["eu", "us"]: 487 """Get a connection's data residency, `"eu"` or `"us"`. 488 489 The one place that decides residency for a connection ID. It raises the 490 errors of `get_connection_dataplane_group` rather than guessing a residency. 491 """ 492 return residency_for_dataplane_group(get_connection_dataplane_group(connection_id))
Get a connection's data residency, "eu" or "us".
The one place that decides residency for a connection ID. It raises the
errors of get_connection_dataplane_group rather than guessing a residency.
481def residency_for_dataplane_group(name: str) -> Literal["eu", "us"]: 482 """Map the EU dataplane group to EU residency and all others to US.""" 483 return "eu" if name == "EU" else "us"
Map the EU dataplane group to EU residency and all others to US.
347def retrieve_objects( 348 connection_objects: list[ConnectionObject], 349 retrieval_reason: str, 350 connection_id: str, 351) -> list[TestingCandidate]: 352 """Retrieve connection objects for a given connection ID. 353 354 This is a simplified version that only supports retrieval by connection_id. 355 For testing candidate discovery by docker image, see issue #91. 356 """ 357 connection_candidates = [TestingCandidate(connection_id=connection_id)] 358 359 secret_manager_client = get_secret_manager_client() 360 connection_pool = get_pool(secret_manager_client) 361 362 with connection_pool.connect() as db_conn: 363 for candidate in connection_candidates.copy(): 364 is_eu_result = db_conn.execute( 365 SELECT_ON_CONNECTION_DATAPLANE_GROUP_IS_EU, 366 parameters={"connection_id": candidate.connection_id}, 367 ).first() 368 if is_eu_result is None: 369 raise ConnectionNotFoundError( 370 f"Credentials were not found for connection ID {candidate.connection_id}." 371 ) 372 is_eu_connection = is_eu_result[0] is True 373 # Each runtime reads only connections in its own data region. 374 if is_eu_connection != is_running_in_eu_region(): 375 connection_candidates.remove(candidate) 376 refusal = ( 377 EU_REGION_REQUIRED_MESSAGE 378 if is_eu_connection 379 else EU_REGION_EU_ONLY_MESSAGE 380 ) 381 LOGGER.warning( 382 f"{refusal}; connection ID " 383 f"{candidate.connection_id} was removed from candidates." 384 ) 385 continue 386 387 candidate.update( 388 **{ 389 connection_object.value.replace("-", "_"): retrieve_object( 390 candidate.connection_id, 391 connection_object, 392 retrieval_reason, 393 db_conn, 394 secret_manager_client, 395 is_eu_connection=is_eu_connection, 396 ) 397 for connection_object in connection_objects 398 } 399 ) 400 401 return connection_candidates
Retrieve connection objects for a given connection ID.
This is a simplified version that only supports retrieval by connection_id. For testing candidate discovery by docker image, see issue #91.