airbyte_ops_mcp.regression_tests

Live tests module for running connector validation and regression tests.

This module provides tools for testing Airbyte connectors against live data without using Dagger. It uses Docker SDK directly for container orchestration.

 1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 2"""Live tests module for running connector validation and regression tests.
 3
 4This module provides tools for testing Airbyte connectors against live data
 5without using Dagger. It uses Docker SDK directly for container orchestration.
 6"""
 7
 8from airbyte_ops_mcp.regression_tests.connection_fetcher import (
 9    ConnectionData,
10    fetch_connection_data,
11)
12from airbyte_ops_mcp.regression_tests.connection_secret_retriever import (
13    SecretRetrievalError,
14    enrich_config_with_secrets,
15    is_secret_retriever_enabled,
16    retrieve_unmasked_config,
17    should_use_secret_retriever,
18)
19from airbyte_ops_mcp.regression_tests.models import (
20    Command,
21    ConnectorUnderTest,
22    ExecutionResult,
23    TargetOrControl,
24)
25
26__all__ = [
27    "Command",
28    "ConnectionData",
29    "ConnectorUnderTest",
30    "ExecutionResult",
31    "SecretRetrievalError",
32    "TargetOrControl",
33    "enrich_config_with_secrets",
34    "fetch_connection_data",
35    "is_secret_retriever_enabled",
36    "retrieve_unmasked_config",
37    "should_use_secret_retriever",
38]
class Command(enum.Enum):
56class Command(Enum):
57    """Airbyte connector commands."""
58
59    CHECK = "check"
60    DISCOVER = "discover"
61    READ = "read"
62    READ_WITH_STATE = "read-with-state"
63    SPEC = "spec"
64
65    def needs_config(self) -> bool:
66        return self in {
67            Command.CHECK,
68            Command.DISCOVER,
69            Command.READ,
70            Command.READ_WITH_STATE,
71        }
72
73    def needs_catalog(self) -> bool:
74        return self in {Command.READ, Command.READ_WITH_STATE}
75
76    def needs_state(self) -> bool:
77        return self in {Command.READ_WITH_STATE}

Airbyte connector commands.

CHECK = <Command.CHECK: 'check'>
DISCOVER = <Command.DISCOVER: 'discover'>
READ = <Command.READ: 'read'>
READ_WITH_STATE = <Command.READ_WITH_STATE: 'read-with-state'>
SPEC = <Command.SPEC: 'spec'>
def needs_config(self) -> bool:
65    def needs_config(self) -> bool:
66        return self in {
67            Command.CHECK,
68            Command.DISCOVER,
69            Command.READ,
70            Command.READ_WITH_STATE,
71        }
def needs_catalog(self) -> bool:
73    def needs_catalog(self) -> bool:
74        return self in {Command.READ, Command.READ_WITH_STATE}
def needs_state(self) -> bool:
76    def needs_state(self) -> bool:
77        return self in {Command.READ_WITH_STATE}
@dataclass
class ConnectionData:
29@dataclass
30class ConnectionData:
31    """Data fetched from an Airbyte Cloud connection."""
32
33    connection_id: str
34    source_id: str
35    source_name: str
36    source_definition_id: str
37    config: dict[str, Any]
38    catalog: dict[str, Any]
39    stream_names: list[str]
40    workspace_id: str | None = None
41    docker_repository: str | None = None
42    docker_image_tag: str | None = None
43    state: list[dict[str, Any]] | None = field(default=None)
44
45    @property
46    def connector_image(self) -> str | None:
47        """Get the full connector image name with tag."""
48        if self.docker_repository and self.docker_image_tag:
49            return f"{self.docker_repository}:{self.docker_image_tag}"
50        return None

Data fetched from an Airbyte Cloud connection.

ConnectionData( connection_id: str, source_id: str, source_name: str, source_definition_id: str, config: dict[str, typing.Any], catalog: dict[str, typing.Any], stream_names: list[str], workspace_id: str | None = None, docker_repository: str | None = None, docker_image_tag: str | None = None, state: list[dict[str, typing.Any]] | None = None)
connection_id: str
source_id: str
source_name: str
source_definition_id: str
config: dict[str, typing.Any]
catalog: dict[str, typing.Any]
stream_names: list[str]
workspace_id: str | None = None
docker_repository: str | None = None
docker_image_tag: str | None = None
state: list[dict[str, typing.Any]] | None = None
connector_image: str | None
45    @property
46    def connector_image(self) -> str | None:
47        """Get the full connector image name with tag."""
48        if self.docker_repository and self.docker_image_tag:
49            return f"{self.docker_repository}:{self.docker_image_tag}"
50        return None

Get the full connector image name with tag.

@dataclass
class ConnectorUnderTest:
 94@dataclass
 95class ConnectorUnderTest:
 96    """Represents a connector being tested.
 97
 98    In validation tests, there would be one connector under test.
 99    When running regression tests, there would be two connectors under test:
100    the target and the control versions of the same connector.
101    """
102
103    image_name: str
104    target_or_control: TargetOrControl
105
106    @property
107    def name(self) -> str:
108        """Get connector name without registry prefix."""
109        return self.image_name.replace("airbyte/", "").split(":")[0]
110
111    @property
112    def name_without_type_prefix(self) -> str:
113        """Get connector name without actor type prefix."""
114        return self.name.replace(f"{self.actor_type.value}-", "")
115
116    @property
117    def version(self) -> str:
118        """Get connector version from image tag."""
119        return self.image_name.replace("airbyte/", "").split(":")[1]
120
121    @property
122    def actor_type(self) -> ActorType:
123        """Infer actor type from image name."""
124        if "airbyte/destination-" in self.image_name:
125            return ActorType.DESTINATION
126        elif "airbyte/source-" in self.image_name:
127            return ActorType.SOURCE
128        else:
129            raise ValueError(
130                f"Can't infer the actor type. Connector image name {self.image_name} "
131                "does not contain 'airbyte/source' or 'airbyte/destination'"
132            )
133
134    @classmethod
135    def from_image_name(
136        cls,
137        image_name: str,
138        target_or_control: TargetOrControl,
139    ) -> ConnectorUnderTest:
140        """Create a ConnectorUnderTest from an image name."""
141        return cls(image_name, target_or_control)

Represents a connector being tested.

In validation tests, there would be one connector under test. When running regression tests, there would be two connectors under test: the target and the control versions of the same connector.

ConnectorUnderTest( image_name: str, target_or_control: TargetOrControl)
image_name: str
target_or_control: TargetOrControl
name: str
106    @property
107    def name(self) -> str:
108        """Get connector name without registry prefix."""
109        return self.image_name.replace("airbyte/", "").split(":")[0]

Get connector name without registry prefix.

name_without_type_prefix: str
111    @property
112    def name_without_type_prefix(self) -> str:
113        """Get connector name without actor type prefix."""
114        return self.name.replace(f"{self.actor_type.value}-", "")

Get connector name without actor type prefix.

version: str
116    @property
117    def version(self) -> str:
118        """Get connector version from image tag."""
119        return self.image_name.replace("airbyte/", "").split(":")[1]

Get connector version from image tag.

actor_type: airbyte_ops_mcp.regression_tests.models.ActorType
121    @property
122    def actor_type(self) -> ActorType:
123        """Infer actor type from image name."""
124        if "airbyte/destination-" in self.image_name:
125            return ActorType.DESTINATION
126        elif "airbyte/source-" in self.image_name:
127            return ActorType.SOURCE
128        else:
129            raise ValueError(
130                f"Can't infer the actor type. Connector image name {self.image_name} "
131                "does not contain 'airbyte/source' or 'airbyte/destination'"
132            )

Infer actor type from image name.

@classmethod
def from_image_name( cls, image_name: str, target_or_control: TargetOrControl) -> ConnectorUnderTest:
134    @classmethod
135    def from_image_name(
136        cls,
137        image_name: str,
138        target_or_control: TargetOrControl,
139    ) -> ConnectorUnderTest:
140        """Create a ConnectorUnderTest from an image name."""
141        return cls(image_name, target_or_control)

Create a ConnectorUnderTest from an image name.

@dataclass
class ExecutionResult:
331@dataclass
332class ExecutionResult:
333    """Result of executing a connector command.
334
335    The messages the run emitted stay on disk in `stdout_file_path`: for a
336    `read` they are the whole dataset, and holding them would make the
337    harness's memory grow with the connector's output. `airbyte_messages`
338    walks the file; the questions that have bounded answers -- counts, the
339    spec, the catalog, the final states -- are answered from one `MessageSummary`
340    collected on the first pass that needs it.
341    """
342
343    connector_under_test: ConnectorUnderTest
344    command: Command
345    stdout_file_path: Path
346    stderr_file_path: Path
347    success: bool
348    exit_code: int
349    configured_catalog: ConfiguredAirbyteCatalog | None = None
350    config: dict[str, Any] | None = None
351    stopped_early: StoppedEarly | None = None
352    _summary: MessageSummary | None = field(default=None, init=False, repr=False)
353
354    @property
355    def logger(self) -> logging.Logger:
356        return logging.getLogger(
357            f"{self.connector_under_test.target_or_control.value}-{self.command.value}"
358        )
359
360    @property
361    def airbyte_messages(self) -> Iterator[AirbyteMessage]:
362        """The run's messages, parsed from stdout one line at a time.
363
364        A fresh pass over the file on every access, and never a list: a
365        `read`'s output is the dataset, and the comparison keeps two results
366        alive at once.
367        """
368        # `errors="replace"`: a byte that does not decode becomes U+FFFD and the
369        # line fails validation below like any other malformed one, rather than
370        # the iteration itself raising and taking the whole run's report with it.
371        with self.stdout_file_path.open(errors="replace") as stdout:
372            for line in stdout:
373                line = line.strip()
374                if not line:
375                    continue
376                try:
377                    message = AirbyteMessage.model_validate_json(line)
378                except ValidationError:
379                    continue
380                yield message
381
382    def _scan(
383        self, on_message: Callable[[AirbyteMessage], None] | None = None
384    ) -> MessageSummary:
385        """One pass over the messages: collect the summary, hand each to `on_message`."""
386        summary = MessageSummary()
387        for message in self.airbyte_messages:
388            summary.add(message)
389            if on_message is not None:
390                on_message(message)
391        self._summary = summary
392        return summary
393
394    @property
395    def summary(self) -> MessageSummary:
396        """The bounded facts about the run, parsed from stdout at most once."""
397        if self._summary is None:
398            return self._scan()
399        return self._summary
400
401    @property
402    def configured_streams(self) -> list[str]:
403        """Get list of configured stream names."""
404        if not self.configured_catalog:
405            return []
406        return [stream.stream.name for stream in self.configured_catalog.streams]
407
408    def get_records(self) -> Iterator[AirbyteMessage]:
409        """Iterate over record messages."""
410        for message in self.airbyte_messages:
411            if message.type is AirbyteMessageType.RECORD:
412                yield message
413
414    def get_records_per_stream(self, stream: str) -> Iterator[AirbyteMessage]:
415        """Get records for a specific stream."""
416        for message in self.get_records():
417            if message.record.stream == stream:  # ty: ignore[unresolved-attribute]
418                yield message
419
420    def get_states(self) -> Iterator[AirbyteMessage]:
421        """Iterate over state messages."""
422        for message in self.airbyte_messages:
423            if message.type is AirbyteMessageType.STATE:
424                yield message
425
426    def get_final_state_per_stream(self) -> dict[tuple[str | None, str], Any]:
427        """The state each stream would resume from, keyed by stream identity.
428
429        A connector emits state as it goes, so only the last message for a
430        stream describes where the next sync starts -- earlier ones are
431        checkpoints it has already moved past. Messages are walked in emission
432        order and later ones overwrite earlier ones, which is exactly how the
433        platform stores them.
434
435        Keyed by `(namespace, name)` rather than by the `namespace.name` label it
436        displays as, for the reason the catalog index is: a stream literally
437        called `public.users` must not share a key with `users` in the `public`
438        namespace and silently take the other's state with it. The comparison
439        turns these into the labels a reviewer reads, qualifying the collision
440        instead of collapsing it.
441
442        The three state types are all kept, because all three are what a
443        connector resumes from:
444
445        - `STREAM` -- one entry per stream, under its own identity.
446        - `GLOBAL` -- unpacked into one entry per stream, plus the shared state
447          under `SHARED_STATE_KEY`.
448        - `LEGACY` -- the protocol carries no stream identity here, so the whole
449          blob is one entry under `LEGACY_STATE_KEY`. Skipping it would let a
450          comparison report "state unchanged" over a state nothing looked at.
451
452        `sourceStats` and `destinationStats` are left out: they live beside the
453        state rather than in it, and `recordCount` legitimately differs between
454        two runs -- record counts are compared in their own table.
455
456        Returns:
457            Stream identity to state value, empty when the run emitted no state.
458        """
459        return dict(self.summary.final_states)
460
461    def get_stream_statuses(self) -> dict[str, list[str]]:
462        """Get the bounded stream statuses emitted during the run."""
463        return {
464            name: list(statuses)
465            for name, statuses in self.summary.stream_statuses.items()
466        }
467
468    def get_stream_errors(self) -> dict[str | None, list[StreamError]]:
469        """Get the bounded, public details of stream error traces."""
470        return {
471            name: list(errors) for name, errors in self.summary.stream_errors.items()
472        }
473
474    def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]:
475        """Count messages by type."""
476        return dict(self.summary.message_counts)
477
478    def get_record_count_per_stream(self) -> dict[str, int]:
479        """Count records by stream name.
480
481        Returns:
482            Dictionary mapping stream names to record counts.
483        """
484        return dict(self.summary.record_counts_per_stream)
485
486    def get_catalog(self) -> AirbyteCatalog | None:
487        """Get discovered catalog from messages."""
488        return self.summary.catalog
489
490    def get_spec(self) -> Any | None:
491        """Get connector spec from messages."""
492        return self.summary.spec
493
494    def get_connection_status(self) -> Any | None:
495        """Get connection status from check command."""
496        return self.summary.connection_status
497
498    def is_check_successful(self) -> bool:
499        """Check if the check command was successful."""
500        status = self.get_connection_status()
501        if status is None:
502            return False
503        return status.status.value == "SUCCEEDED"
504
505    def save_artifacts(self, output_dir: Path) -> None:
506        """Save execution artifacts to the output directory.
507
508        One streaming pass writes every message to `airbyte_messages/<type>.jsonl`
509        and collects the run's summary, so the counts and declared objects the
510        caller asks for next cost no second parse. One handle per message type
511        stays open for the pass: the protocol has a handful of types, not a
512        stream's worth.
513        """
514        output_dir.mkdir(parents=True, exist_ok=True)
515
516        airbyte_messages_dir = output_dir / "airbyte_messages"
517        airbyte_messages_dir.mkdir(parents=True, exist_ok=True)
518
519        handles: dict[str, IO[str]] = {}
520
521        def write(message: AirbyteMessage) -> None:
522            type_name = message.type.value.lower()
523            handle = handles.get(type_name)
524            if handle is None:
525                handle = (airbyte_messages_dir / f"{type_name}.jsonl").open("w")
526                handles[type_name] = handle
527            else:
528                handle.write("\n")
529            handle.write(message.model_dump_json())
530
531        try:
532            self._scan(write)
533        finally:
534            for handle in handles.values():
535                handle.close()
536
537        # Save configured catalog (input) if available
538        if self.configured_catalog is not None:
539            catalog_path = output_dir / "configured_catalog.json"
540            catalog_path.write_text(
541                self.configured_catalog.model_dump_json(indent=2, exclude_none=True)
542            )
543            self.logger.info(f"Saved configured catalog to {catalog_path}")
544
545        self.logger.info(f"Artifacts saved to {output_dir}")

Result of executing a connector command.

The messages the run emitted stay on disk in stdout_file_path: for a read they are the whole dataset, and holding them would make the harness's memory grow with the connector's output. airbyte_messages walks the file; the questions that have bounded answers -- counts, the spec, the catalog, the final states -- are answered from one MessageSummary collected on the first pass that needs it.

ExecutionResult( connector_under_test: ConnectorUnderTest, command: Command, stdout_file_path: pathlib.Path, stderr_file_path: pathlib.Path, success: bool, exit_code: int, configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog | None = None, config: dict[str, typing.Any] | None = None, stopped_early: airbyte_ops_mcp.regression_tests.models.StoppedEarly | None = None)
connector_under_test: ConnectorUnderTest
command: Command
stdout_file_path: pathlib.Path
stderr_file_path: pathlib.Path
success: bool
exit_code: int
configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog | None = None
config: dict[str, typing.Any] | None = None
stopped_early: airbyte_ops_mcp.regression_tests.models.StoppedEarly | None = None
logger: logging.Logger
354    @property
355    def logger(self) -> logging.Logger:
356        return logging.getLogger(
357            f"{self.connector_under_test.target_or_control.value}-{self.command.value}"
358        )
airbyte_messages: Iterator[airbyte_protocol.models.airbyte_protocol.AirbyteMessage]
360    @property
361    def airbyte_messages(self) -> Iterator[AirbyteMessage]:
362        """The run's messages, parsed from stdout one line at a time.
363
364        A fresh pass over the file on every access, and never a list: a
365        `read`'s output is the dataset, and the comparison keeps two results
366        alive at once.
367        """
368        # `errors="replace"`: a byte that does not decode becomes U+FFFD and the
369        # line fails validation below like any other malformed one, rather than
370        # the iteration itself raising and taking the whole run's report with it.
371        with self.stdout_file_path.open(errors="replace") as stdout:
372            for line in stdout:
373                line = line.strip()
374                if not line:
375                    continue
376                try:
377                    message = AirbyteMessage.model_validate_json(line)
378                except ValidationError:
379                    continue
380                yield message

The run's messages, parsed from stdout one line at a time.

A fresh pass over the file on every access, and never a list: a read's output is the dataset, and the comparison keeps two results alive at once.

summary: airbyte_ops_mcp.regression_tests.models.MessageSummary
394    @property
395    def summary(self) -> MessageSummary:
396        """The bounded facts about the run, parsed from stdout at most once."""
397        if self._summary is None:
398            return self._scan()
399        return self._summary

The bounded facts about the run, parsed from stdout at most once.

configured_streams: list[str]
401    @property
402    def configured_streams(self) -> list[str]:
403        """Get list of configured stream names."""
404        if not self.configured_catalog:
405            return []
406        return [stream.stream.name for stream in self.configured_catalog.streams]

Get list of configured stream names.

def get_records( self) -> Iterator[airbyte_protocol.models.airbyte_protocol.AirbyteMessage]:
408    def get_records(self) -> Iterator[AirbyteMessage]:
409        """Iterate over record messages."""
410        for message in self.airbyte_messages:
411            if message.type is AirbyteMessageType.RECORD:
412                yield message

Iterate over record messages.

def get_records_per_stream( self, stream: str) -> Iterator[airbyte_protocol.models.airbyte_protocol.AirbyteMessage]:
414    def get_records_per_stream(self, stream: str) -> Iterator[AirbyteMessage]:
415        """Get records for a specific stream."""
416        for message in self.get_records():
417            if message.record.stream == stream:  # ty: ignore[unresolved-attribute]
418                yield message

Get records for a specific stream.

def get_states( self) -> Iterator[airbyte_protocol.models.airbyte_protocol.AirbyteMessage]:
420    def get_states(self) -> Iterator[AirbyteMessage]:
421        """Iterate over state messages."""
422        for message in self.airbyte_messages:
423            if message.type is AirbyteMessageType.STATE:
424                yield message

Iterate over state messages.

def get_final_state_per_stream(self) -> dict[tuple[str | None, str], typing.Any]:
426    def get_final_state_per_stream(self) -> dict[tuple[str | None, str], Any]:
427        """The state each stream would resume from, keyed by stream identity.
428
429        A connector emits state as it goes, so only the last message for a
430        stream describes where the next sync starts -- earlier ones are
431        checkpoints it has already moved past. Messages are walked in emission
432        order and later ones overwrite earlier ones, which is exactly how the
433        platform stores them.
434
435        Keyed by `(namespace, name)` rather than by the `namespace.name` label it
436        displays as, for the reason the catalog index is: a stream literally
437        called `public.users` must not share a key with `users` in the `public`
438        namespace and silently take the other's state with it. The comparison
439        turns these into the labels a reviewer reads, qualifying the collision
440        instead of collapsing it.
441
442        The three state types are all kept, because all three are what a
443        connector resumes from:
444
445        - `STREAM` -- one entry per stream, under its own identity.
446        - `GLOBAL` -- unpacked into one entry per stream, plus the shared state
447          under `SHARED_STATE_KEY`.
448        - `LEGACY` -- the protocol carries no stream identity here, so the whole
449          blob is one entry under `LEGACY_STATE_KEY`. Skipping it would let a
450          comparison report "state unchanged" over a state nothing looked at.
451
452        `sourceStats` and `destinationStats` are left out: they live beside the
453        state rather than in it, and `recordCount` legitimately differs between
454        two runs -- record counts are compared in their own table.
455
456        Returns:
457            Stream identity to state value, empty when the run emitted no state.
458        """
459        return dict(self.summary.final_states)

The state each stream would resume from, keyed by stream identity.

A connector emits state as it goes, so only the last message for a stream describes where the next sync starts -- earlier ones are checkpoints it has already moved past. Messages are walked in emission order and later ones overwrite earlier ones, which is exactly how the platform stores them.

Keyed by (namespace, name) rather than by the namespace.name label it displays as, for the reason the catalog index is: a stream literally called public.users must not share a key with users in the public namespace and silently take the other's state with it. The comparison turns these into the labels a reviewer reads, qualifying the collision instead of collapsing it.

The three state types are all kept, because all three are what a connector resumes from:

  • STREAM -- one entry per stream, under its own identity.
  • GLOBAL -- unpacked into one entry per stream, plus the shared state under SHARED_STATE_KEY.
  • LEGACY -- the protocol carries no stream identity here, so the whole blob is one entry under LEGACY_STATE_KEY. Skipping it would let a comparison report "state unchanged" over a state nothing looked at.

sourceStats and destinationStats are left out: they live beside the state rather than in it, and recordCount legitimately differs between two runs -- record counts are compared in their own table.

Returns:

Stream identity to state value, empty when the run emitted no state.

def get_stream_statuses(self) -> dict[str, list[str]]:
461    def get_stream_statuses(self) -> dict[str, list[str]]:
462        """Get the bounded stream statuses emitted during the run."""
463        return {
464            name: list(statuses)
465            for name, statuses in self.summary.stream_statuses.items()
466        }

Get the bounded stream statuses emitted during the run.

def get_stream_errors( self) -> dict[str | None, list[airbyte_ops_mcp.regression_tests.models.StreamError]]:
468    def get_stream_errors(self) -> dict[str | None, list[StreamError]]:
469        """Get the bounded, public details of stream error traces."""
470        return {
471            name: list(errors) for name, errors in self.summary.stream_errors.items()
472        }

Get the bounded, public details of stream error traces.

def get_message_count_per_type(self) -> dict[airbyte_protocol.models.airbyte_protocol.Type, int]:
474    def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]:
475        """Count messages by type."""
476        return dict(self.summary.message_counts)

Count messages by type.

def get_record_count_per_stream(self) -> dict[str, int]:
478    def get_record_count_per_stream(self) -> dict[str, int]:
479        """Count records by stream name.
480
481        Returns:
482            Dictionary mapping stream names to record counts.
483        """
484        return dict(self.summary.record_counts_per_stream)

Count records by stream name.

Returns:

Dictionary mapping stream names to record counts.

def get_catalog(self) -> airbyte_protocol.models.airbyte_protocol.AirbyteCatalog | None:
486    def get_catalog(self) -> AirbyteCatalog | None:
487        """Get discovered catalog from messages."""
488        return self.summary.catalog

Get discovered catalog from messages.

def get_spec(self) -> typing.Any | None:
490    def get_spec(self) -> Any | None:
491        """Get connector spec from messages."""
492        return self.summary.spec

Get connector spec from messages.

def get_connection_status(self) -> typing.Any | None:
494    def get_connection_status(self) -> Any | None:
495        """Get connection status from check command."""
496        return self.summary.connection_status

Get connection status from check command.

def is_check_successful(self) -> bool:
498    def is_check_successful(self) -> bool:
499        """Check if the check command was successful."""
500        status = self.get_connection_status()
501        if status is None:
502            return False
503        return status.status.value == "SUCCEEDED"

Check if the check command was successful.

def save_artifacts(self, output_dir: pathlib.Path) -> None:
505    def save_artifacts(self, output_dir: Path) -> None:
506        """Save execution artifacts to the output directory.
507
508        One streaming pass writes every message to `airbyte_messages/<type>.jsonl`
509        and collects the run's summary, so the counts and declared objects the
510        caller asks for next cost no second parse. One handle per message type
511        stays open for the pass: the protocol has a handful of types, not a
512        stream's worth.
513        """
514        output_dir.mkdir(parents=True, exist_ok=True)
515
516        airbyte_messages_dir = output_dir / "airbyte_messages"
517        airbyte_messages_dir.mkdir(parents=True, exist_ok=True)
518
519        handles: dict[str, IO[str]] = {}
520
521        def write(message: AirbyteMessage) -> None:
522            type_name = message.type.value.lower()
523            handle = handles.get(type_name)
524            if handle is None:
525                handle = (airbyte_messages_dir / f"{type_name}.jsonl").open("w")
526                handles[type_name] = handle
527            else:
528                handle.write("\n")
529            handle.write(message.model_dump_json())
530
531        try:
532            self._scan(write)
533        finally:
534            for handle in handles.values():
535                handle.close()
536
537        # Save configured catalog (input) if available
538        if self.configured_catalog is not None:
539            catalog_path = output_dir / "configured_catalog.json"
540            catalog_path.write_text(
541                self.configured_catalog.model_dump_json(indent=2, exclude_none=True)
542            )
543            self.logger.info(f"Saved configured catalog to {catalog_path}")
544
545        self.logger.info(f"Artifacts saved to {output_dir}")

Save execution artifacts to the output directory.

One streaming pass writes every message to airbyte_messages/<type>.jsonl and collects the run's summary, so the counts and declared objects the caller asks for next cost no second parse. One handle per message type stays open for the pass: the protocol has a handful of types, not a stream's worth.

class SecretRetrievalError(builtins.Exception):
195class SecretRetrievalError(Exception):
196    """Raised when secret retrieval fails.
197
198    This exception is raised when USE_CONNECTION_SECRET_RETRIEVER is enabled
199    but secrets cannot be retrieved (e.g., EU data residency restrictions).
200    """

Raised when secret retrieval fails.

This exception is raised when USE_CONNECTION_SECRET_RETRIEVER is enabled but secrets cannot be retrieved (e.g., EU data residency restrictions).

class TargetOrControl(enum.Enum):
80class TargetOrControl(Enum):
81    """Indicates whether a connector is the target (new) or control (baseline) version."""
82
83    TARGET = "target"
84    CONTROL = "control"

Indicates whether a connector is the target (new) or control (baseline) version.

TARGET = <TargetOrControl.TARGET: 'target'>
CONTROL = <TargetOrControl.CONTROL: 'control'>
def enrich_config_with_secrets( connection_data: ConnectionData, retrieval_reason: str = 'MCP live tests', raise_on_failure: bool = True) -> ConnectionData:
203def enrich_config_with_secrets(
204    connection_data: ConnectionData,
205    retrieval_reason: str = "MCP live tests",
206    raise_on_failure: bool = True,
207) -> ConnectionData:
208    """Enrich connection data with unmasked secrets from internal retriever.
209
210    This function takes a ConnectionData object (typically from the public
211    Cloud API with masked secrets) and replaces the config with unmasked
212    secrets from the internal connection-retriever.
213
214    Args:
215        connection_data: The connection data to enrich.
216        retrieval_reason: Reason for retrieval (for audit logging).
217        raise_on_failure: If True (default), raise SecretRetrievalError when
218            secrets cannot be retrieved. If False, return the original
219            connection_data with masked secrets (legacy behavior).
220
221    Returns:
222        A new ConnectionData with unmasked config.
223
224    Raises:
225        SecretRetrievalError: If raise_on_failure is True and secrets cannot
226            be retrieved (e.g., due to EU data residency restrictions).
227    """
228    unmasked_config = retrieve_unmasked_config(
229        connection_id=connection_data.connection_id,
230        retrieval_reason=retrieval_reason,
231    )
232
233    if unmasked_config is None:
234        error_msg = (
235            "Could not retrieve unmasked secrets for connection "
236            f"{connection_data.connection_id}. This may be due to EU data "
237            "residency restrictions or database connectivity issues. "
238            "The connection's credentials cannot be used for regression testing."
239        )
240        logger.warning(error_msg)
241        if raise_on_failure:
242            raise SecretRetrievalError(error_msg)
243        return connection_data
244
245    logger.info(
246        f"Successfully enriched config with unmasked secrets for "
247        f"{connection_data.connection_id}"
248    )
249
250    # Return a new ConnectionData with the unmasked config
251    return replace(connection_data, config=unmasked_config)

Enrich connection data with unmasked secrets from internal retriever.

This function takes a ConnectionData object (typically from the public Cloud API with masked secrets) and replaces the config with unmasked secrets from the internal connection-retriever.

Arguments:
  • connection_data: The connection data to enrich.
  • retrieval_reason: Reason for retrieval (for audit logging).
  • raise_on_failure: If True (default), raise SecretRetrievalError when secrets cannot be retrieved. If False, return the original connection_data with masked secrets (legacy behavior).
Returns:

A new ConnectionData with unmasked config.

Raises:
  • SecretRetrievalError: If raise_on_failure is True and secrets cannot be retrieved (e.g., due to EU data residency restrictions).
def fetch_connection_data( connection_id: str, client_id: str | None = None, client_secret: str | None = None) -> ConnectionData:
 77def fetch_connection_data(
 78    connection_id: str,
 79    client_id: str | None = None,
 80    client_secret: str | None = None,
 81) -> ConnectionData:
 82    """Fetch connection configuration and catalog from Airbyte Cloud.
 83
 84    Args:
 85        connection_id: The connection ID to fetch data for.
 86        client_id: Airbyte Cloud client ID (defaults to env var).
 87        client_secret: Airbyte Cloud client secret (defaults to env var).
 88
 89    Returns:
 90        ConnectionData with config and catalog.
 91
 92    Raises:
 93        PyAirbyteInputError: If the API request fails.
 94    """
 95    client_id = client_id or os.getenv("AIRBYTE_CLOUD_CLIENT_ID")
 96    client_secret = client_secret or os.getenv("AIRBYTE_CLOUD_CLIENT_SECRET")
 97
 98    if not client_id or not client_secret:
 99        raise PyAirbyteInputError(
100            message="Missing Airbyte Cloud credentials",
101            context={
102                "hint": "Set AIRBYTE_CLOUD_CLIENT_ID and AIRBYTE_CLOUD_CLIENT_SECRET env vars"
103            },
104        )
105
106    access_token = _get_access_token(client_id, client_secret)
107    public_api_root = constants.CLOUD_API_ROOT
108    headers = {
109        "Authorization": f"Bearer {access_token}",
110        "Content-Type": "application/json",
111    }
112
113    # Get connection details
114    conn_response = requests.get(
115        f"{public_api_root}/connections/{connection_id}",
116        headers=headers,
117        timeout=30,
118    )
119
120    if conn_response.status_code != 200:
121        raise PyAirbyteInputError(
122            message=f"Failed to get connection: {conn_response.status_code}",
123            context={"connection_id": connection_id, "response": conn_response.text},
124        )
125
126    conn_data = conn_response.json()
127    source_id = conn_data["sourceId"]
128
129    # Get source details (includes config)
130    source_response = requests.get(
131        f"{public_api_root}/sources/{source_id}",
132        headers=headers,
133        timeout=30,
134    )
135
136    if source_response.status_code != 200:
137        raise PyAirbyteInputError(
138            message=f"Failed to get source: {source_response.status_code}",
139            context={"source_id": source_id, "response": source_response.text},
140        )
141
142    source_data = source_response.json()
143    source_definition_id = source_data.get("definitionId", "")
144
145    # Try to get docker repository and image tag from source definition version
146    docker_repository = None
147    docker_image_tag = None
148    if source_definition_id:
149        try:
150            # Use the Config API to get version info for the source
151            config_api_root = constants.CLOUD_CONFIG_API_ROOT
152            version_response = requests.post(
153                f"{config_api_root}/actor_definition_versions/get_for_source",
154                json={"sourceId": source_id},
155                headers=headers,
156                timeout=30,
157            )
158            if version_response.status_code == 200:
159                version_data = version_response.json()
160                docker_repository = version_data.get("dockerRepository")
161                docker_image_tag = version_data.get("dockerImageTag")
162        except Exception:
163            # Non-fatal: we can still proceed without docker info
164            pass
165
166    # Build configured catalog from connection streams
167    streams_config = conn_data.get("configurations", {}).get("streams", [])
168    stream_names = [s["name"] for s in streams_config]
169
170    # Build Airbyte protocol catalog format
171    catalog = _build_configured_catalog(
172        streams_config, source_id, headers, public_api_root
173    )
174
175    return ConnectionData(
176        connection_id=connection_id,
177        source_id=source_id,
178        source_name=source_data.get("name", ""),
179        source_definition_id=source_definition_id,
180        config=source_data.get("configuration", {}),
181        catalog=catalog,
182        stream_names=stream_names,
183        workspace_id=conn_data.get("workspaceId"),
184        docker_repository=docker_repository,
185        docker_image_tag=docker_image_tag,
186    )

Fetch connection configuration and catalog from Airbyte Cloud.

Arguments:
  • connection_id: The connection ID to fetch data for.
  • client_id: Airbyte Cloud client ID (defaults to env var).
  • client_secret: Airbyte Cloud client secret (defaults to env var).
Returns:

ConnectionData with config and catalog.

Raises:
  • PyAirbyteInputError: If the API request fails.
def is_secret_retriever_enabled() -> bool:
54def is_secret_retriever_enabled() -> bool:
55    """Check if secret retrieval is enabled via environment variable.
56
57    Returns:
58        True if USE_CONNECTION_SECRET_RETRIEVER is set to a truthy value.
59    """
60    value = os.getenv(ENV_USE_SECRET_RETRIEVER, "").lower()
61    return value in ("true", "1", "yes")

Check if secret retrieval is enabled via environment variable.

Returns:

True if USE_CONNECTION_SECRET_RETRIEVER is set to a truthy value.

def retrieve_unmasked_config( connection_id: str, retrieval_reason: str = 'MCP live tests') -> dict | None:
 73def retrieve_unmasked_config(
 74    connection_id: str,
 75    retrieval_reason: str = "MCP live tests",
 76) -> dict | None:
 77    """Retrieve unmasked source config from vendored connection-retriever.
 78
 79    This function directly queries the internal Postgres database to get
 80    the source configuration with unmasked secrets.
 81
 82    Args:
 83        connection_id: The Airbyte Cloud connection ID.
 84        retrieval_reason: Reason for retrieval (for audit logging).
 85
 86    Returns:
 87        The unmasked source config dict, or None if retrieval fails.
 88    """
 89    # Only request the source config - that's all we need for secrets
 90    requested_objects = [ConnectionObject.SOURCE_CONFIG]
 91
 92    candidates = retrieve_objects(
 93        connection_objects=requested_objects,
 94        retrieval_reason=retrieval_reason,
 95        connection_id=connection_id,
 96    )
 97
 98    if not candidates:
 99        logger.warning(
100            f"No connection data found for connection ID {connection_id} "
101            "via connection-retriever"
102        )
103        return None
104
105    candidate = candidates[0]
106    if candidate.source_config:
107        return dict(candidate.source_config)
108
109    return None

Retrieve unmasked source config from vendored connection-retriever.

This function directly queries the internal Postgres database to get the source configuration with unmasked secrets.

Arguments:
  • connection_id: The Airbyte Cloud connection ID.
  • retrieval_reason: Reason for retrieval (for audit logging).
Returns:

The unmasked source config dict, or None if retrieval fails.

def should_use_secret_retriever() -> bool:
64def should_use_secret_retriever() -> bool:
65    """Check if secret retrieval should be used.
66
67    Returns:
68        True if USE_CONNECTION_SECRET_RETRIEVER env var is set to a truthy value.
69    """
70    return is_secret_retriever_enabled()

Check if secret retrieval should be used.

Returns:

True if USE_CONNECTION_SECRET_RETRIEVER env var is set to a truthy value.