airbyte_ops_mcp.regression_tests
Live tests module for running connector validation and regression tests.
This module provides tools for testing Airbyte connectors against live data without using Dagger. It uses Docker SDK directly for container orchestration.
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Live tests module for running connector validation and regression tests. 3 4This module provides tools for testing Airbyte connectors against live data 5without using Dagger. It uses Docker SDK directly for container orchestration. 6""" 7 8from airbyte_ops_mcp.regression_tests.connection_fetcher import ( 9 ConnectionData, 10 fetch_connection_data, 11) 12from airbyte_ops_mcp.regression_tests.connection_secret_retriever import ( 13 SecretRetrievalError, 14 enrich_config_with_secrets, 15 is_secret_retriever_enabled, 16 retrieve_unmasked_config, 17 should_use_secret_retriever, 18) 19from airbyte_ops_mcp.regression_tests.models import ( 20 Command, 21 ConnectorUnderTest, 22 ExecutionResult, 23 TargetOrControl, 24) 25 26__all__ = [ 27 "Command", 28 "ConnectionData", 29 "ConnectorUnderTest", 30 "ExecutionResult", 31 "SecretRetrievalError", 32 "TargetOrControl", 33 "enrich_config_with_secrets", 34 "fetch_connection_data", 35 "is_secret_retriever_enabled", 36 "retrieve_unmasked_config", 37 "should_use_secret_retriever", 38]
56class Command(Enum): 57 """Airbyte connector commands.""" 58 59 CHECK = "check" 60 DISCOVER = "discover" 61 READ = "read" 62 READ_WITH_STATE = "read-with-state" 63 SPEC = "spec" 64 65 def needs_config(self) -> bool: 66 return self in { 67 Command.CHECK, 68 Command.DISCOVER, 69 Command.READ, 70 Command.READ_WITH_STATE, 71 } 72 73 def needs_catalog(self) -> bool: 74 return self in {Command.READ, Command.READ_WITH_STATE} 75 76 def needs_state(self) -> bool: 77 return self in {Command.READ_WITH_STATE}
Airbyte connector commands.
29@dataclass 30class ConnectionData: 31 """Data fetched from an Airbyte Cloud connection.""" 32 33 connection_id: str 34 source_id: str 35 source_name: str 36 source_definition_id: str 37 config: dict[str, Any] 38 catalog: dict[str, Any] 39 stream_names: list[str] 40 workspace_id: str | None = None 41 docker_repository: str | None = None 42 docker_image_tag: str | None = None 43 state: list[dict[str, Any]] | None = field(default=None) 44 45 @property 46 def connector_image(self) -> str | None: 47 """Get the full connector image name with tag.""" 48 if self.docker_repository and self.docker_image_tag: 49 return f"{self.docker_repository}:{self.docker_image_tag}" 50 return None
Data fetched from an Airbyte Cloud connection.
45 @property 46 def connector_image(self) -> str | None: 47 """Get the full connector image name with tag.""" 48 if self.docker_repository and self.docker_image_tag: 49 return f"{self.docker_repository}:{self.docker_image_tag}" 50 return None
Get the full connector image name with tag.
94@dataclass 95class ConnectorUnderTest: 96 """Represents a connector being tested. 97 98 In validation tests, there would be one connector under test. 99 When running regression tests, there would be two connectors under test: 100 the target and the control versions of the same connector. 101 """ 102 103 image_name: str 104 target_or_control: TargetOrControl 105 106 @property 107 def name(self) -> str: 108 """Get connector name without registry prefix.""" 109 return self.image_name.replace("airbyte/", "").split(":")[0] 110 111 @property 112 def name_without_type_prefix(self) -> str: 113 """Get connector name without actor type prefix.""" 114 return self.name.replace(f"{self.actor_type.value}-", "") 115 116 @property 117 def version(self) -> str: 118 """Get connector version from image tag.""" 119 return self.image_name.replace("airbyte/", "").split(":")[1] 120 121 @property 122 def actor_type(self) -> ActorType: 123 """Infer actor type from image name.""" 124 if "airbyte/destination-" in self.image_name: 125 return ActorType.DESTINATION 126 elif "airbyte/source-" in self.image_name: 127 return ActorType.SOURCE 128 else: 129 raise ValueError( 130 f"Can't infer the actor type. Connector image name {self.image_name} " 131 "does not contain 'airbyte/source' or 'airbyte/destination'" 132 ) 133 134 @classmethod 135 def from_image_name( 136 cls, 137 image_name: str, 138 target_or_control: TargetOrControl, 139 ) -> ConnectorUnderTest: 140 """Create a ConnectorUnderTest from an image name.""" 141 return cls(image_name, target_or_control)
Represents a connector being tested.
In validation tests, there would be one connector under test. When running regression tests, there would be two connectors under test: the target and the control versions of the same connector.
106 @property 107 def name(self) -> str: 108 """Get connector name without registry prefix.""" 109 return self.image_name.replace("airbyte/", "").split(":")[0]
Get connector name without registry prefix.
111 @property 112 def name_without_type_prefix(self) -> str: 113 """Get connector name without actor type prefix.""" 114 return self.name.replace(f"{self.actor_type.value}-", "")
Get connector name without actor type prefix.
116 @property 117 def version(self) -> str: 118 """Get connector version from image tag.""" 119 return self.image_name.replace("airbyte/", "").split(":")[1]
Get connector version from image tag.
121 @property 122 def actor_type(self) -> ActorType: 123 """Infer actor type from image name.""" 124 if "airbyte/destination-" in self.image_name: 125 return ActorType.DESTINATION 126 elif "airbyte/source-" in self.image_name: 127 return ActorType.SOURCE 128 else: 129 raise ValueError( 130 f"Can't infer the actor type. Connector image name {self.image_name} " 131 "does not contain 'airbyte/source' or 'airbyte/destination'" 132 )
Infer actor type from image name.
134 @classmethod 135 def from_image_name( 136 cls, 137 image_name: str, 138 target_or_control: TargetOrControl, 139 ) -> ConnectorUnderTest: 140 """Create a ConnectorUnderTest from an image name.""" 141 return cls(image_name, target_or_control)
Create a ConnectorUnderTest from an image name.
331@dataclass 332class ExecutionResult: 333 """Result of executing a connector command. 334 335 The messages the run emitted stay on disk in `stdout_file_path`: for a 336 `read` they are the whole dataset, and holding them would make the 337 harness's memory grow with the connector's output. `airbyte_messages` 338 walks the file; the questions that have bounded answers -- counts, the 339 spec, the catalog, the final states -- are answered from one `MessageSummary` 340 collected on the first pass that needs it. 341 """ 342 343 connector_under_test: ConnectorUnderTest 344 command: Command 345 stdout_file_path: Path 346 stderr_file_path: Path 347 success: bool 348 exit_code: int 349 configured_catalog: ConfiguredAirbyteCatalog | None = None 350 config: dict[str, Any] | None = None 351 stopped_early: StoppedEarly | None = None 352 _summary: MessageSummary | None = field(default=None, init=False, repr=False) 353 354 @property 355 def logger(self) -> logging.Logger: 356 return logging.getLogger( 357 f"{self.connector_under_test.target_or_control.value}-{self.command.value}" 358 ) 359 360 @property 361 def airbyte_messages(self) -> Iterator[AirbyteMessage]: 362 """The run's messages, parsed from stdout one line at a time. 363 364 A fresh pass over the file on every access, and never a list: a 365 `read`'s output is the dataset, and the comparison keeps two results 366 alive at once. 367 """ 368 # `errors="replace"`: a byte that does not decode becomes U+FFFD and the 369 # line fails validation below like any other malformed one, rather than 370 # the iteration itself raising and taking the whole run's report with it. 371 with self.stdout_file_path.open(errors="replace") as stdout: 372 for line in stdout: 373 line = line.strip() 374 if not line: 375 continue 376 try: 377 message = AirbyteMessage.model_validate_json(line) 378 except ValidationError: 379 continue 380 yield message 381 382 def _scan( 383 self, on_message: Callable[[AirbyteMessage], None] | None = None 384 ) -> MessageSummary: 385 """One pass over the messages: collect the summary, hand each to `on_message`.""" 386 summary = MessageSummary() 387 for message in self.airbyte_messages: 388 summary.add(message) 389 if on_message is not None: 390 on_message(message) 391 self._summary = summary 392 return summary 393 394 @property 395 def summary(self) -> MessageSummary: 396 """The bounded facts about the run, parsed from stdout at most once.""" 397 if self._summary is None: 398 return self._scan() 399 return self._summary 400 401 @property 402 def configured_streams(self) -> list[str]: 403 """Get list of configured stream names.""" 404 if not self.configured_catalog: 405 return [] 406 return [stream.stream.name for stream in self.configured_catalog.streams] 407 408 def get_records(self) -> Iterator[AirbyteMessage]: 409 """Iterate over record messages.""" 410 for message in self.airbyte_messages: 411 if message.type is AirbyteMessageType.RECORD: 412 yield message 413 414 def get_records_per_stream(self, stream: str) -> Iterator[AirbyteMessage]: 415 """Get records for a specific stream.""" 416 for message in self.get_records(): 417 if message.record.stream == stream: # ty: ignore[unresolved-attribute] 418 yield message 419 420 def get_states(self) -> Iterator[AirbyteMessage]: 421 """Iterate over state messages.""" 422 for message in self.airbyte_messages: 423 if message.type is AirbyteMessageType.STATE: 424 yield message 425 426 def get_final_state_per_stream(self) -> dict[tuple[str | None, str], Any]: 427 """The state each stream would resume from, keyed by stream identity. 428 429 A connector emits state as it goes, so only the last message for a 430 stream describes where the next sync starts -- earlier ones are 431 checkpoints it has already moved past. Messages are walked in emission 432 order and later ones overwrite earlier ones, which is exactly how the 433 platform stores them. 434 435 Keyed by `(namespace, name)` rather than by the `namespace.name` label it 436 displays as, for the reason the catalog index is: a stream literally 437 called `public.users` must not share a key with `users` in the `public` 438 namespace and silently take the other's state with it. The comparison 439 turns these into the labels a reviewer reads, qualifying the collision 440 instead of collapsing it. 441 442 The three state types are all kept, because all three are what a 443 connector resumes from: 444 445 - `STREAM` -- one entry per stream, under its own identity. 446 - `GLOBAL` -- unpacked into one entry per stream, plus the shared state 447 under `SHARED_STATE_KEY`. 448 - `LEGACY` -- the protocol carries no stream identity here, so the whole 449 blob is one entry under `LEGACY_STATE_KEY`. Skipping it would let a 450 comparison report "state unchanged" over a state nothing looked at. 451 452 `sourceStats` and `destinationStats` are left out: they live beside the 453 state rather than in it, and `recordCount` legitimately differs between 454 two runs -- record counts are compared in their own table. 455 456 Returns: 457 Stream identity to state value, empty when the run emitted no state. 458 """ 459 return dict(self.summary.final_states) 460 461 def get_stream_statuses(self) -> dict[str, list[str]]: 462 """Get the bounded stream statuses emitted during the run.""" 463 return { 464 name: list(statuses) 465 for name, statuses in self.summary.stream_statuses.items() 466 } 467 468 def get_stream_errors(self) -> dict[str | None, list[StreamError]]: 469 """Get the bounded, public details of stream error traces.""" 470 return { 471 name: list(errors) for name, errors in self.summary.stream_errors.items() 472 } 473 474 def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]: 475 """Count messages by type.""" 476 return dict(self.summary.message_counts) 477 478 def get_record_count_per_stream(self) -> dict[str, int]: 479 """Count records by stream name. 480 481 Returns: 482 Dictionary mapping stream names to record counts. 483 """ 484 return dict(self.summary.record_counts_per_stream) 485 486 def get_catalog(self) -> AirbyteCatalog | None: 487 """Get discovered catalog from messages.""" 488 return self.summary.catalog 489 490 def get_spec(self) -> Any | None: 491 """Get connector spec from messages.""" 492 return self.summary.spec 493 494 def get_connection_status(self) -> Any | None: 495 """Get connection status from check command.""" 496 return self.summary.connection_status 497 498 def is_check_successful(self) -> bool: 499 """Check if the check command was successful.""" 500 status = self.get_connection_status() 501 if status is None: 502 return False 503 return status.status.value == "SUCCEEDED" 504 505 def save_artifacts(self, output_dir: Path) -> None: 506 """Save execution artifacts to the output directory. 507 508 One streaming pass writes every message to `airbyte_messages/<type>.jsonl` 509 and collects the run's summary, so the counts and declared objects the 510 caller asks for next cost no second parse. One handle per message type 511 stays open for the pass: the protocol has a handful of types, not a 512 stream's worth. 513 """ 514 output_dir.mkdir(parents=True, exist_ok=True) 515 516 airbyte_messages_dir = output_dir / "airbyte_messages" 517 airbyte_messages_dir.mkdir(parents=True, exist_ok=True) 518 519 handles: dict[str, IO[str]] = {} 520 521 def write(message: AirbyteMessage) -> None: 522 type_name = message.type.value.lower() 523 handle = handles.get(type_name) 524 if handle is None: 525 handle = (airbyte_messages_dir / f"{type_name}.jsonl").open("w") 526 handles[type_name] = handle 527 else: 528 handle.write("\n") 529 handle.write(message.model_dump_json()) 530 531 try: 532 self._scan(write) 533 finally: 534 for handle in handles.values(): 535 handle.close() 536 537 # Save configured catalog (input) if available 538 if self.configured_catalog is not None: 539 catalog_path = output_dir / "configured_catalog.json" 540 catalog_path.write_text( 541 self.configured_catalog.model_dump_json(indent=2, exclude_none=True) 542 ) 543 self.logger.info(f"Saved configured catalog to {catalog_path}") 544 545 self.logger.info(f"Artifacts saved to {output_dir}")
Result of executing a connector command.
The messages the run emitted stay on disk in stdout_file_path: for a
read they are the whole dataset, and holding them would make the
harness's memory grow with the connector's output. airbyte_messages
walks the file; the questions that have bounded answers -- counts, the
spec, the catalog, the final states -- are answered from one MessageSummary
collected on the first pass that needs it.
360 @property 361 def airbyte_messages(self) -> Iterator[AirbyteMessage]: 362 """The run's messages, parsed from stdout one line at a time. 363 364 A fresh pass over the file on every access, and never a list: a 365 `read`'s output is the dataset, and the comparison keeps two results 366 alive at once. 367 """ 368 # `errors="replace"`: a byte that does not decode becomes U+FFFD and the 369 # line fails validation below like any other malformed one, rather than 370 # the iteration itself raising and taking the whole run's report with it. 371 with self.stdout_file_path.open(errors="replace") as stdout: 372 for line in stdout: 373 line = line.strip() 374 if not line: 375 continue 376 try: 377 message = AirbyteMessage.model_validate_json(line) 378 except ValidationError: 379 continue 380 yield message
The run's messages, parsed from stdout one line at a time.
A fresh pass over the file on every access, and never a list: a
read's output is the dataset, and the comparison keeps two results
alive at once.
394 @property 395 def summary(self) -> MessageSummary: 396 """The bounded facts about the run, parsed from stdout at most once.""" 397 if self._summary is None: 398 return self._scan() 399 return self._summary
The bounded facts about the run, parsed from stdout at most once.
401 @property 402 def configured_streams(self) -> list[str]: 403 """Get list of configured stream names.""" 404 if not self.configured_catalog: 405 return [] 406 return [stream.stream.name for stream in self.configured_catalog.streams]
Get list of configured stream names.
408 def get_records(self) -> Iterator[AirbyteMessage]: 409 """Iterate over record messages.""" 410 for message in self.airbyte_messages: 411 if message.type is AirbyteMessageType.RECORD: 412 yield message
Iterate over record messages.
414 def get_records_per_stream(self, stream: str) -> Iterator[AirbyteMessage]: 415 """Get records for a specific stream.""" 416 for message in self.get_records(): 417 if message.record.stream == stream: # ty: ignore[unresolved-attribute] 418 yield message
Get records for a specific stream.
420 def get_states(self) -> Iterator[AirbyteMessage]: 421 """Iterate over state messages.""" 422 for message in self.airbyte_messages: 423 if message.type is AirbyteMessageType.STATE: 424 yield message
Iterate over state messages.
426 def get_final_state_per_stream(self) -> dict[tuple[str | None, str], Any]: 427 """The state each stream would resume from, keyed by stream identity. 428 429 A connector emits state as it goes, so only the last message for a 430 stream describes where the next sync starts -- earlier ones are 431 checkpoints it has already moved past. Messages are walked in emission 432 order and later ones overwrite earlier ones, which is exactly how the 433 platform stores them. 434 435 Keyed by `(namespace, name)` rather than by the `namespace.name` label it 436 displays as, for the reason the catalog index is: a stream literally 437 called `public.users` must not share a key with `users` in the `public` 438 namespace and silently take the other's state with it. The comparison 439 turns these into the labels a reviewer reads, qualifying the collision 440 instead of collapsing it. 441 442 The three state types are all kept, because all three are what a 443 connector resumes from: 444 445 - `STREAM` -- one entry per stream, under its own identity. 446 - `GLOBAL` -- unpacked into one entry per stream, plus the shared state 447 under `SHARED_STATE_KEY`. 448 - `LEGACY` -- the protocol carries no stream identity here, so the whole 449 blob is one entry under `LEGACY_STATE_KEY`. Skipping it would let a 450 comparison report "state unchanged" over a state nothing looked at. 451 452 `sourceStats` and `destinationStats` are left out: they live beside the 453 state rather than in it, and `recordCount` legitimately differs between 454 two runs -- record counts are compared in their own table. 455 456 Returns: 457 Stream identity to state value, empty when the run emitted no state. 458 """ 459 return dict(self.summary.final_states)
The state each stream would resume from, keyed by stream identity.
A connector emits state as it goes, so only the last message for a stream describes where the next sync starts -- earlier ones are checkpoints it has already moved past. Messages are walked in emission order and later ones overwrite earlier ones, which is exactly how the platform stores them.
Keyed by (namespace, name) rather than by the namespace.name label it
displays as, for the reason the catalog index is: a stream literally
called public.users must not share a key with users in the public
namespace and silently take the other's state with it. The comparison
turns these into the labels a reviewer reads, qualifying the collision
instead of collapsing it.
The three state types are all kept, because all three are what a connector resumes from:
STREAM-- one entry per stream, under its own identity.GLOBAL-- unpacked into one entry per stream, plus the shared state underSHARED_STATE_KEY.LEGACY-- the protocol carries no stream identity here, so the whole blob is one entry underLEGACY_STATE_KEY. Skipping it would let a comparison report "state unchanged" over a state nothing looked at.
sourceStats and destinationStats are left out: they live beside the
state rather than in it, and recordCount legitimately differs between
two runs -- record counts are compared in their own table.
Returns:
Stream identity to state value, empty when the run emitted no state.
461 def get_stream_statuses(self) -> dict[str, list[str]]: 462 """Get the bounded stream statuses emitted during the run.""" 463 return { 464 name: list(statuses) 465 for name, statuses in self.summary.stream_statuses.items() 466 }
Get the bounded stream statuses emitted during the run.
468 def get_stream_errors(self) -> dict[str | None, list[StreamError]]: 469 """Get the bounded, public details of stream error traces.""" 470 return { 471 name: list(errors) for name, errors in self.summary.stream_errors.items() 472 }
Get the bounded, public details of stream error traces.
474 def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]: 475 """Count messages by type.""" 476 return dict(self.summary.message_counts)
Count messages by type.
478 def get_record_count_per_stream(self) -> dict[str, int]: 479 """Count records by stream name. 480 481 Returns: 482 Dictionary mapping stream names to record counts. 483 """ 484 return dict(self.summary.record_counts_per_stream)
Count records by stream name.
Returns:
Dictionary mapping stream names to record counts.
486 def get_catalog(self) -> AirbyteCatalog | None: 487 """Get discovered catalog from messages.""" 488 return self.summary.catalog
Get discovered catalog from messages.
490 def get_spec(self) -> Any | None: 491 """Get connector spec from messages.""" 492 return self.summary.spec
Get connector spec from messages.
494 def get_connection_status(self) -> Any | None: 495 """Get connection status from check command.""" 496 return self.summary.connection_status
Get connection status from check command.
498 def is_check_successful(self) -> bool: 499 """Check if the check command was successful.""" 500 status = self.get_connection_status() 501 if status is None: 502 return False 503 return status.status.value == "SUCCEEDED"
Check if the check command was successful.
505 def save_artifacts(self, output_dir: Path) -> None: 506 """Save execution artifacts to the output directory. 507 508 One streaming pass writes every message to `airbyte_messages/<type>.jsonl` 509 and collects the run's summary, so the counts and declared objects the 510 caller asks for next cost no second parse. One handle per message type 511 stays open for the pass: the protocol has a handful of types, not a 512 stream's worth. 513 """ 514 output_dir.mkdir(parents=True, exist_ok=True) 515 516 airbyte_messages_dir = output_dir / "airbyte_messages" 517 airbyte_messages_dir.mkdir(parents=True, exist_ok=True) 518 519 handles: dict[str, IO[str]] = {} 520 521 def write(message: AirbyteMessage) -> None: 522 type_name = message.type.value.lower() 523 handle = handles.get(type_name) 524 if handle is None: 525 handle = (airbyte_messages_dir / f"{type_name}.jsonl").open("w") 526 handles[type_name] = handle 527 else: 528 handle.write("\n") 529 handle.write(message.model_dump_json()) 530 531 try: 532 self._scan(write) 533 finally: 534 for handle in handles.values(): 535 handle.close() 536 537 # Save configured catalog (input) if available 538 if self.configured_catalog is not None: 539 catalog_path = output_dir / "configured_catalog.json" 540 catalog_path.write_text( 541 self.configured_catalog.model_dump_json(indent=2, exclude_none=True) 542 ) 543 self.logger.info(f"Saved configured catalog to {catalog_path}") 544 545 self.logger.info(f"Artifacts saved to {output_dir}")
Save execution artifacts to the output directory.
One streaming pass writes every message to airbyte_messages/<type>.jsonl
and collects the run's summary, so the counts and declared objects the
caller asks for next cost no second parse. One handle per message type
stays open for the pass: the protocol has a handful of types, not a
stream's worth.
195class SecretRetrievalError(Exception): 196 """Raised when secret retrieval fails. 197 198 This exception is raised when USE_CONNECTION_SECRET_RETRIEVER is enabled 199 but secrets cannot be retrieved (e.g., EU data residency restrictions). 200 """
Raised when secret retrieval fails.
This exception is raised when USE_CONNECTION_SECRET_RETRIEVER is enabled but secrets cannot be retrieved (e.g., EU data residency restrictions).
80class TargetOrControl(Enum): 81 """Indicates whether a connector is the target (new) or control (baseline) version.""" 82 83 TARGET = "target" 84 CONTROL = "control"
Indicates whether a connector is the target (new) or control (baseline) version.
203def enrich_config_with_secrets( 204 connection_data: ConnectionData, 205 retrieval_reason: str = "MCP live tests", 206 raise_on_failure: bool = True, 207) -> ConnectionData: 208 """Enrich connection data with unmasked secrets from internal retriever. 209 210 This function takes a ConnectionData object (typically from the public 211 Cloud API with masked secrets) and replaces the config with unmasked 212 secrets from the internal connection-retriever. 213 214 Args: 215 connection_data: The connection data to enrich. 216 retrieval_reason: Reason for retrieval (for audit logging). 217 raise_on_failure: If True (default), raise SecretRetrievalError when 218 secrets cannot be retrieved. If False, return the original 219 connection_data with masked secrets (legacy behavior). 220 221 Returns: 222 A new ConnectionData with unmasked config. 223 224 Raises: 225 SecretRetrievalError: If raise_on_failure is True and secrets cannot 226 be retrieved (e.g., due to EU data residency restrictions). 227 """ 228 unmasked_config = retrieve_unmasked_config( 229 connection_id=connection_data.connection_id, 230 retrieval_reason=retrieval_reason, 231 ) 232 233 if unmasked_config is None: 234 error_msg = ( 235 "Could not retrieve unmasked secrets for connection " 236 f"{connection_data.connection_id}. This may be due to EU data " 237 "residency restrictions or database connectivity issues. " 238 "The connection's credentials cannot be used for regression testing." 239 ) 240 logger.warning(error_msg) 241 if raise_on_failure: 242 raise SecretRetrievalError(error_msg) 243 return connection_data 244 245 logger.info( 246 f"Successfully enriched config with unmasked secrets for " 247 f"{connection_data.connection_id}" 248 ) 249 250 # Return a new ConnectionData with the unmasked config 251 return replace(connection_data, config=unmasked_config)
Enrich connection data with unmasked secrets from internal retriever.
This function takes a ConnectionData object (typically from the public Cloud API with masked secrets) and replaces the config with unmasked secrets from the internal connection-retriever.
Arguments:
- connection_data: The connection data to enrich.
- retrieval_reason: Reason for retrieval (for audit logging).
- raise_on_failure: If True (default), raise SecretRetrievalError when secrets cannot be retrieved. If False, return the original connection_data with masked secrets (legacy behavior).
Returns:
A new ConnectionData with unmasked config.
Raises:
- SecretRetrievalError: If raise_on_failure is True and secrets cannot be retrieved (e.g., due to EU data residency restrictions).
77def fetch_connection_data( 78 connection_id: str, 79 client_id: str | None = None, 80 client_secret: str | None = None, 81) -> ConnectionData: 82 """Fetch connection configuration and catalog from Airbyte Cloud. 83 84 Args: 85 connection_id: The connection ID to fetch data for. 86 client_id: Airbyte Cloud client ID (defaults to env var). 87 client_secret: Airbyte Cloud client secret (defaults to env var). 88 89 Returns: 90 ConnectionData with config and catalog. 91 92 Raises: 93 PyAirbyteInputError: If the API request fails. 94 """ 95 client_id = client_id or os.getenv("AIRBYTE_CLOUD_CLIENT_ID") 96 client_secret = client_secret or os.getenv("AIRBYTE_CLOUD_CLIENT_SECRET") 97 98 if not client_id or not client_secret: 99 raise PyAirbyteInputError( 100 message="Missing Airbyte Cloud credentials", 101 context={ 102 "hint": "Set AIRBYTE_CLOUD_CLIENT_ID and AIRBYTE_CLOUD_CLIENT_SECRET env vars" 103 }, 104 ) 105 106 access_token = _get_access_token(client_id, client_secret) 107 public_api_root = constants.CLOUD_API_ROOT 108 headers = { 109 "Authorization": f"Bearer {access_token}", 110 "Content-Type": "application/json", 111 } 112 113 # Get connection details 114 conn_response = requests.get( 115 f"{public_api_root}/connections/{connection_id}", 116 headers=headers, 117 timeout=30, 118 ) 119 120 if conn_response.status_code != 200: 121 raise PyAirbyteInputError( 122 message=f"Failed to get connection: {conn_response.status_code}", 123 context={"connection_id": connection_id, "response": conn_response.text}, 124 ) 125 126 conn_data = conn_response.json() 127 source_id = conn_data["sourceId"] 128 129 # Get source details (includes config) 130 source_response = requests.get( 131 f"{public_api_root}/sources/{source_id}", 132 headers=headers, 133 timeout=30, 134 ) 135 136 if source_response.status_code != 200: 137 raise PyAirbyteInputError( 138 message=f"Failed to get source: {source_response.status_code}", 139 context={"source_id": source_id, "response": source_response.text}, 140 ) 141 142 source_data = source_response.json() 143 source_definition_id = source_data.get("definitionId", "") 144 145 # Try to get docker repository and image tag from source definition version 146 docker_repository = None 147 docker_image_tag = None 148 if source_definition_id: 149 try: 150 # Use the Config API to get version info for the source 151 config_api_root = constants.CLOUD_CONFIG_API_ROOT 152 version_response = requests.post( 153 f"{config_api_root}/actor_definition_versions/get_for_source", 154 json={"sourceId": source_id}, 155 headers=headers, 156 timeout=30, 157 ) 158 if version_response.status_code == 200: 159 version_data = version_response.json() 160 docker_repository = version_data.get("dockerRepository") 161 docker_image_tag = version_data.get("dockerImageTag") 162 except Exception: 163 # Non-fatal: we can still proceed without docker info 164 pass 165 166 # Build configured catalog from connection streams 167 streams_config = conn_data.get("configurations", {}).get("streams", []) 168 stream_names = [s["name"] for s in streams_config] 169 170 # Build Airbyte protocol catalog format 171 catalog = _build_configured_catalog( 172 streams_config, source_id, headers, public_api_root 173 ) 174 175 return ConnectionData( 176 connection_id=connection_id, 177 source_id=source_id, 178 source_name=source_data.get("name", ""), 179 source_definition_id=source_definition_id, 180 config=source_data.get("configuration", {}), 181 catalog=catalog, 182 stream_names=stream_names, 183 workspace_id=conn_data.get("workspaceId"), 184 docker_repository=docker_repository, 185 docker_image_tag=docker_image_tag, 186 )
Fetch connection configuration and catalog from Airbyte Cloud.
Arguments:
- connection_id: The connection ID to fetch data for.
- client_id: Airbyte Cloud client ID (defaults to env var).
- client_secret: Airbyte Cloud client secret (defaults to env var).
Returns:
ConnectionData with config and catalog.
Raises:
- PyAirbyteInputError: If the API request fails.
54def is_secret_retriever_enabled() -> bool: 55 """Check if secret retrieval is enabled via environment variable. 56 57 Returns: 58 True if USE_CONNECTION_SECRET_RETRIEVER is set to a truthy value. 59 """ 60 value = os.getenv(ENV_USE_SECRET_RETRIEVER, "").lower() 61 return value in ("true", "1", "yes")
Check if secret retrieval is enabled via environment variable.
Returns:
True if USE_CONNECTION_SECRET_RETRIEVER is set to a truthy value.
73def retrieve_unmasked_config( 74 connection_id: str, 75 retrieval_reason: str = "MCP live tests", 76) -> dict | None: 77 """Retrieve unmasked source config from vendored connection-retriever. 78 79 This function directly queries the internal Postgres database to get 80 the source configuration with unmasked secrets. 81 82 Args: 83 connection_id: The Airbyte Cloud connection ID. 84 retrieval_reason: Reason for retrieval (for audit logging). 85 86 Returns: 87 The unmasked source config dict, or None if retrieval fails. 88 """ 89 # Only request the source config - that's all we need for secrets 90 requested_objects = [ConnectionObject.SOURCE_CONFIG] 91 92 candidates = retrieve_objects( 93 connection_objects=requested_objects, 94 retrieval_reason=retrieval_reason, 95 connection_id=connection_id, 96 ) 97 98 if not candidates: 99 logger.warning( 100 f"No connection data found for connection ID {connection_id} " 101 "via connection-retriever" 102 ) 103 return None 104 105 candidate = candidates[0] 106 if candidate.source_config: 107 return dict(candidate.source_config) 108 109 return None
Retrieve unmasked source config from vendored connection-retriever.
This function directly queries the internal Postgres database to get the source configuration with unmasked secrets.
Arguments:
- connection_id: The Airbyte Cloud connection ID.
- retrieval_reason: Reason for retrieval (for audit logging).
Returns:
The unmasked source config dict, or None if retrieval fails.
64def should_use_secret_retriever() -> bool: 65 """Check if secret retrieval should be used. 66 67 Returns: 68 True if USE_CONNECTION_SECRET_RETRIEVER env var is set to a truthy value. 69 """ 70 return is_secret_retriever_enabled()
Check if secret retrieval should be used.
Returns:
True if USE_CONNECTION_SECRET_RETRIEVER env var is set to a truthy value.