airbyte_ops_mcp.registry

Registry operations for Airbyte connectors.

This package provides functionality for reading, listing, publishing, compiling, and validating connector registry artifacts.

  1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
  2"""Registry operations for Airbyte connectors.
  3
  4This package provides functionality for reading, listing, publishing, compiling,
  5and validating connector registry artifacts.
  6"""
  7
  8from __future__ import annotations
  9
 10from airbyte_ops_mcp.registry._constants import (
 11    DEFAULT_METADATA_SERVICE_BUCKET_NAME,
 12    DEV_METADATA_SERVICE_BUCKET_NAME,
 13    LATEST_GCS_FOLDER_NAME,
 14    METADATA_FILE_NAME,
 15    METADATA_FOLDER,
 16    PROD_METADATA_SERVICE_BUCKET_NAME,
 17    RELEASE_CANDIDATE_GCS_FOLDER_NAME,
 18    SONAR_DEV_BUCKET_NAME,
 19    SONAR_PROD_BUCKET_NAME,
 20)
 21from airbyte_ops_mcp.registry._enums import (
 22    ConnectorLanguage,
 23    ConnectorType,
 24    SupportLevel,
 25)
 26from airbyte_ops_mcp.registry.audit import (
 27    AuditResult,
 28    UnpublishedConnector,
 29    find_unpublished_connectors,
 30)
 31from airbyte_ops_mcp.registry.compile import (
 32    CompileResult,
 33    PurgeLatestResult,
 34    compile_registry,
 35    purge_latest_dirs,
 36)
 37from airbyte_ops_mcp.registry.generate import (
 38    GenerateResult,
 39    generate_version_artifacts,
 40)
 41from airbyte_ops_mcp.registry.models import (
 42    ConnectorListResult,
 43    ConnectorMetadata,
 44    MetadataPublishResult,
 45    RegistryEntryResult,
 46    VersionListResult,
 47)
 48from airbyte_ops_mcp.registry.operations import (
 49    get_registry_entry,
 50    get_registry_spec,
 51    list_connector_versions,
 52    list_registry_connectors,
 53    list_registry_connectors_filtered,
 54)
 55from airbyte_ops_mcp.registry.publish import (
 56    CONNECTOR_PATH_PREFIX,
 57    get_connector_metadata,
 58    get_gcs_publish_path,
 59    publish_connector_metadata,
 60)
 61from airbyte_ops_mcp.registry.publish_artifacts import (
 62    PublishArtifactsResult,
 63    publish_version_artifacts,
 64)
 65from airbyte_ops_mcp.registry.rebuild import (
 66    OutputMode,
 67    RebuildResult,
 68    rebuild_registry,
 69)
 70from airbyte_ops_mcp.registry.registry_store_base import (
 71    Registry,
 72    get_registry,
 73)
 74from airbyte_ops_mcp.registry.store import (
 75    REGISTRY_STORE_ENV_VAR,
 76    RegistryStore,
 77    StoreType,
 78    resolve_registry_store,
 79)
 80from airbyte_ops_mcp.registry.validate import (
 81    ValidateOptions,
 82    ValidationResult,
 83    validate_metadata,
 84)
 85from airbyte_ops_mcp.registry.yank import (
 86    YANK_FILE_NAME,
 87    YankResult,
 88    unyank_connector_version,
 89    yank_connector_version,
 90)
 91
 92__all__ = [
 93    "CONNECTOR_PATH_PREFIX",
 94    "DEFAULT_METADATA_SERVICE_BUCKET_NAME",
 95    "DEV_METADATA_SERVICE_BUCKET_NAME",
 96    "LATEST_GCS_FOLDER_NAME",
 97    "METADATA_FILE_NAME",
 98    "METADATA_FOLDER",
 99    "PROD_METADATA_SERVICE_BUCKET_NAME",
100    "REGISTRY_STORE_ENV_VAR",
101    "RELEASE_CANDIDATE_GCS_FOLDER_NAME",
102    "SONAR_DEV_BUCKET_NAME",
103    "SONAR_PROD_BUCKET_NAME",
104    "YANK_FILE_NAME",
105    "AuditResult",
106    "CompileResult",
107    "ConnectorLanguage",
108    "ConnectorListResult",
109    "ConnectorMetadata",
110    "ConnectorType",
111    "GenerateResult",
112    "MetadataPublishResult",
113    "OutputMode",
114    "PublishArtifactsResult",
115    "PurgeLatestResult",
116    "RebuildResult",
117    "Registry",
118    "RegistryEntryResult",
119    "RegistryStore",
120    "StoreType",
121    "SupportLevel",
122    "UnpublishedConnector",
123    "ValidateOptions",
124    "ValidationResult",
125    "VersionListResult",
126    "YankResult",
127    "compile_registry",
128    "find_unpublished_connectors",
129    "generate_version_artifacts",
130    "get_connector_metadata",
131    "get_gcs_publish_path",
132    "get_registry",
133    "get_registry_entry",
134    "get_registry_spec",
135    "list_connector_versions",
136    "list_registry_connectors",
137    "list_registry_connectors_filtered",
138    "publish_connector_metadata",
139    "publish_version_artifacts",
140    "purge_latest_dirs",
141    "rebuild_registry",
142    "resolve_registry_store",
143    "unyank_connector_version",
144    "validate_metadata",
145    "yank_connector_version",
146]
CONNECTOR_PATH_PREFIX = 'airbyte-integrations/connectors'
DEFAULT_METADATA_SERVICE_BUCKET_NAME = 'dev-airbyte-cloud-connector-metadata-service-2'
DEV_METADATA_SERVICE_BUCKET_NAME = 'dev-airbyte-cloud-connector-metadata-service-2'
LATEST_GCS_FOLDER_NAME = 'latest'
METADATA_FILE_NAME = 'metadata.yaml'
METADATA_FOLDER = 'metadata'
PROD_METADATA_SERVICE_BUCKET_NAME = 'prod-airbyte-cloud-connector-metadata-service'
REGISTRY_STORE_ENV_VAR = 'AIRBYTE_REGISTRY_STORE'
RELEASE_CANDIDATE_GCS_FOLDER_NAME = 'release_candidate'
SONAR_DEV_BUCKET_NAME = 'airbyte-connector-registry-dev'
SONAR_PROD_BUCKET_NAME = 'airbyte-connector-registry'
YANK_FILE_NAME = 'version-yank.yml'
@dataclass
class AuditResult:
37@dataclass
38class AuditResult:
39    """Result of auditing which connectors have unpublished versions."""
40
41    unpublished: list[UnpublishedConnector] = field(default_factory=list)
42    checked_count: int = 0
43    skipped_archived: list[str] = field(default_factory=list)
44    skipped_rc: list[str] = field(default_factory=list)
45    skipped_disabled: list[str] = field(default_factory=list)
46    errors: list[str] = field(default_factory=list)

Result of auditing which connectors have unpublished versions.

AuditResult( unpublished: list[UnpublishedConnector] = <factory>, checked_count: int = 0, skipped_archived: list[str] = <factory>, skipped_rc: list[str] = <factory>, skipped_disabled: list[str] = <factory>, errors: list[str] = <factory>)
unpublished: list[UnpublishedConnector]
checked_count: int = 0
skipped_archived: list[str]
skipped_rc: list[str]
skipped_disabled: list[str]
errors: list[str]
@dataclass
class CompileResult:
110@dataclass
111class CompileResult:
112    """Result of a registry compile operation."""
113
114    target: str
115    connectors_scanned: int = 0
116    versions_found: int = 0
117    yanked_versions: int = 0
118    latest_updated: int = 0
119    latest_already_current: int = 0
120    cloud_registry_entries: int = 0
121    oss_registry_entries: int = 0
122    composite_registry_entries: int = 0
123    metrics_connector_count: int = 0
124    metrics_registry_entries: int = 0
125    metrics_source: str | None = None
126    metrics_error: str | None = None
127    version_indexes_written: int = 0
128    specs_secrets_mask_properties: int = 0
129    errors: list[str] = field(default_factory=list)
130    dry_run: bool = False
131
132    @property
133    def status(self) -> str:
134        if self.dry_run:
135            return "dry-run"
136        if self.errors:
137            return "completed-with-errors"
138        return "success"
139
140    def summary(self) -> str:
141        return (
142            f"[{self.status}] Scanned {self.connectors_scanned} connectors, "
143            f"{self.versions_found} versions ({self.yanked_versions} yanked). "
144            f"Latest updated: {self.latest_updated}, "
145            f"already current: {self.latest_already_current}. "
146            f"Registry entries: cloud={self.cloud_registry_entries}, "
147            f"oss={self.oss_registry_entries}, "
148            f"composite={self.composite_registry_entries}. "
149            f"Metrics loaded for {self.metrics_connector_count} connectors, "
150            f"injected into {self.metrics_registry_entries} registry entries. "
151            f"Version indexes: {self.version_indexes_written}. "
152            f"Specs secrets mask: {self.specs_secrets_mask_properties} properties. "
153            f"Errors: {len(self.errors)}."
154        )

Result of a registry compile operation.

CompileResult( target: str, connectors_scanned: int = 0, versions_found: int = 0, yanked_versions: int = 0, latest_updated: int = 0, latest_already_current: int = 0, cloud_registry_entries: int = 0, oss_registry_entries: int = 0, composite_registry_entries: int = 0, metrics_connector_count: int = 0, metrics_registry_entries: int = 0, metrics_source: str | None = None, metrics_error: str | None = None, version_indexes_written: int = 0, specs_secrets_mask_properties: int = 0, errors: list[str] = <factory>, dry_run: bool = False)
target: str
connectors_scanned: int = 0
versions_found: int = 0
yanked_versions: int = 0
latest_updated: int = 0
latest_already_current: int = 0
cloud_registry_entries: int = 0
oss_registry_entries: int = 0
composite_registry_entries: int = 0
metrics_connector_count: int = 0
metrics_registry_entries: int = 0
metrics_source: str | None = None
metrics_error: str | None = None
version_indexes_written: int = 0
specs_secrets_mask_properties: int = 0
errors: list[str]
dry_run: bool = False
status: str
132    @property
133    def status(self) -> str:
134        if self.dry_run:
135            return "dry-run"
136        if self.errors:
137            return "completed-with-errors"
138        return "success"
def summary(self) -> str:
140    def summary(self) -> str:
141        return (
142            f"[{self.status}] Scanned {self.connectors_scanned} connectors, "
143            f"{self.versions_found} versions ({self.yanked_versions} yanked). "
144            f"Latest updated: {self.latest_updated}, "
145            f"already current: {self.latest_already_current}. "
146            f"Registry entries: cloud={self.cloud_registry_entries}, "
147            f"oss={self.oss_registry_entries}, "
148            f"composite={self.composite_registry_entries}. "
149            f"Metrics loaded for {self.metrics_connector_count} connectors, "
150            f"injected into {self.metrics_registry_entries} registry entries. "
151            f"Version indexes: {self.version_indexes_written}. "
152            f"Specs secrets mask: {self.specs_secrets_mask_properties} properties. "
153            f"Errors: {len(self.errors)}."
154        )
class ConnectorLanguage(enum.StrEnum):
 88class ConnectorLanguage(StrEnum):
 89    """Connector implementation languages."""
 90
 91    PYTHON = "python"
 92    JAVA = "java"
 93    LOW_CODE = "low-code"
 94    MANIFEST_ONLY = "manifest-only"
 95
 96    @classmethod
 97    def parse(cls, value: str) -> ConnectorLanguage:
 98        """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch."""
 99        try:
100            return cls(value)
101        except ValueError:
102            valid = ", ".join(f"`{m.value}`" for m in cls)
103            raise ValueError(
104                f"Unrecognized language: {value!r}. Expected one of: {valid}."
105            ) from None

Connector implementation languages.

PYTHON = <ConnectorLanguage.PYTHON: 'python'>
JAVA = <ConnectorLanguage.JAVA: 'java'>
LOW_CODE = <ConnectorLanguage.LOW_CODE: 'low-code'>
MANIFEST_ONLY = <ConnectorLanguage.MANIFEST_ONLY: 'manifest-only'>
@classmethod
def parse(cls, value: str) -> ConnectorLanguage:
 96    @classmethod
 97    def parse(cls, value: str) -> ConnectorLanguage:
 98        """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch."""
 99        try:
100            return cls(value)
101        except ValueError:
102            valid = ", ".join(f"`{m.value}`" for m in cls)
103            raise ValueError(
104                f"Unrecognized language: {value!r}. Expected one of: {valid}."
105            ) from None

Parse a string into a ConnectorLanguage, raising ValueError on mismatch.

class ConnectorListResult(pydantic.main.BaseModel):
73class ConnectorListResult(BaseModel):
74    """Result of listing connectors in the registry."""
75
76    bucket_name: str = Field(description="The GCS bucket name")
77    connector_count: int = Field(description="Number of connectors found")
78    connectors: list[str] = Field(description="List of connector names")

Result of listing connectors in the registry.

bucket_name: str = PydanticUndefined

The GCS bucket name

connector_count: int = PydanticUndefined

Number of connectors found

connectors: list[str] = PydanticUndefined

List of connector names

class ConnectorMetadata(pydantic.main.BaseModel):
12class ConnectorMetadata(BaseModel):
13    """Connector metadata from metadata.yaml.
14
15    This model represents the essential metadata about a connector
16    read from its metadata.yaml file in the Airbyte monorepo.
17    """
18
19    name: str = Field(description="The connector technical name")
20    docker_repository: str = Field(description="The Docker repository")
21    docker_image_tag: str = Field(description="The Docker image tag/version")
22    support_level: str | None = Field(
23        default=None, description="The support level (certified, community, etc.)"
24    )
25    definition_id: str | None = Field(
26        default=None, description="The connector definition ID"
27    )

Connector metadata from metadata.yaml.

This model represents the essential metadata about a connector read from its metadata.yaml file in the Airbyte monorepo.

name: str = PydanticUndefined

The connector technical name

docker_repository: str = PydanticUndefined

The Docker repository

docker_image_tag: str = PydanticUndefined

The Docker image tag/version

support_level: str | None = None

The support level (certified, community, etc.)

definition_id: str | None = None

The connector definition ID

class ConnectorType(enum.StrEnum):
70class ConnectorType(StrEnum):
71    """Connector type: source or destination."""
72
73    SOURCE = "source"
74    DESTINATION = "destination"
75
76    @classmethod
77    def parse(cls, value: str) -> ConnectorType:
78        """Parse a string into a `ConnectorType`, raising `ValueError` on mismatch."""
79        try:
80            return cls(value)
81        except ValueError:
82            valid = ", ".join(f"`{m.value}`" for m in cls)
83            raise ValueError(
84                f"Unrecognized connector type: {value!r}. Expected one of: {valid}."
85            ) from None

Connector type: source or destination.

SOURCE = <ConnectorType.SOURCE: 'source'>
DESTINATION = <ConnectorType.DESTINATION: 'destination'>
@classmethod
def parse(cls, value: str) -> ConnectorType:
76    @classmethod
77    def parse(cls, value: str) -> ConnectorType:
78        """Parse a string into a `ConnectorType`, raising `ValueError` on mismatch."""
79        try:
80            return cls(value)
81        except ValueError:
82            valid = ", ".join(f"`{m.value}`" for m in cls)
83            raise ValueError(
84                f"Unrecognized connector type: {value!r}. Expected one of: {valid}."
85            ) from None

Parse a string into a ConnectorType, raising ValueError on mismatch.

@dataclass
class GenerateResult:
127@dataclass
128class GenerateResult:
129    """Result of a local artifact generation run."""
130
131    connector_name: str
132    version: str
133    docker_image: str
134    output_dir: str
135    artifacts_written: list[str] = field(default_factory=list)
136    errors: list[str] = field(default_factory=list)
137    validation_errors: list[str] = field(default_factory=list)
138    dry_run: bool = False
139
140    @property
141    def success(self) -> bool:
142        return len(self.errors) == 0 and len(self.validation_errors) == 0
143
144    def to_dict(self) -> dict[str, Any]:
145        return {
146            "connector_name": self.connector_name,
147            "version": self.version,
148            "docker_image": self.docker_image,
149            "output_dir": self.output_dir,
150            "artifacts_written": self.artifacts_written,
151            "errors": self.errors,
152            "validation_errors": self.validation_errors,
153            "dry_run": self.dry_run,
154            "success": self.success,
155        }

Result of a local artifact generation run.

GenerateResult( connector_name: str, version: str, docker_image: str, output_dir: str, artifacts_written: list[str] = <factory>, errors: list[str] = <factory>, validation_errors: list[str] = <factory>, dry_run: bool = False)
connector_name: str
version: str
docker_image: str
output_dir: str
artifacts_written: list[str]
errors: list[str]
validation_errors: list[str]
dry_run: bool = False
success: bool
140    @property
141    def success(self) -> bool:
142        return len(self.errors) == 0 and len(self.validation_errors) == 0
def to_dict(self) -> dict[str, typing.Any]:
144    def to_dict(self) -> dict[str, Any]:
145        return {
146            "connector_name": self.connector_name,
147            "version": self.version,
148            "docker_image": self.docker_image,
149            "output_dir": self.output_dir,
150            "artifacts_written": self.artifacts_written,
151            "errors": self.errors,
152            "validation_errors": self.validation_errors,
153            "dry_run": self.dry_run,
154            "success": self.success,
155        }
class MetadataPublishResult(pydantic.main.BaseModel):
30class MetadataPublishResult(BaseModel):
31    """Result of a metadata publish operation to GCS.
32
33    This model provides detailed information about the outcome of
34    publishing connector metadata to the registry.
35    """
36
37    connector_name: str = Field(description="The connector technical name")
38    version: str = Field(description="The version that was published")
39    bucket_name: str = Field(description="The GCS bucket name")
40    versioned_path: str = Field(description="The versioned GCS path")
41    latest_path: str | None = Field(
42        default=None, description="The latest GCS path if updated"
43    )
44    versioned_uploaded: bool = Field(
45        default=False, description="Whether the versioned metadata was uploaded"
46    )
47    latest_uploaded: bool = Field(
48        default=False, description="Whether the latest metadata was uploaded"
49    )
50    status: Literal["success", "dry-run", "already-up-to-date"] = Field(
51        description="The status of the operation"
52    )
53    message: str = Field(description="Status message describing the outcome")
54
55    def __str__(self) -> str:
56        """Return a string representation of the publish result."""
57        return f"[{self.status}] {self.connector_name}:{self.version} -> {self.versioned_path}"

Result of a metadata publish operation to GCS.

This model provides detailed information about the outcome of publishing connector metadata to the registry.

connector_name: str = PydanticUndefined

The connector technical name

version: str = PydanticUndefined

The version that was published

bucket_name: str = PydanticUndefined

The GCS bucket name

versioned_path: str = PydanticUndefined

The versioned GCS path

latest_path: str | None = None

The latest GCS path if updated

versioned_uploaded: bool = False

Whether the versioned metadata was uploaded

latest_uploaded: bool = False

Whether the latest metadata was uploaded

status: Literal['success', 'dry-run', 'already-up-to-date'] = PydanticUndefined

The status of the operation

message: str = PydanticUndefined

Status message describing the outcome

OutputMode = typing.Literal['local', 'gcs', 's3']
@dataclass
class PublishArtifactsResult:
55@dataclass
56class PublishArtifactsResult:
57    """Result of a version-artifacts publish operation."""
58
59    connector_name: str
60    version: str
61    target: str
62    gcs_destination: str
63    files_uploaded: list[str] = field(default_factory=list)
64    errors: list[str] = field(default_factory=list)
65    validation_errors: list[str] = field(default_factory=list)
66    progressive_rollout_overridden_by_breaking_change: bool = False
67    progressive_rollout_overridden_by_published_ga: bool = False
68    dry_run: bool = False
69
70    @property
71    def success(self) -> bool:
72        return len(self.errors) == 0 and len(self.validation_errors) == 0
73
74    @property
75    def status(self) -> str:
76        if self.dry_run:
77            return "dry-run"
78        if self.errors or self.validation_errors:
79            return "completed-with-errors"
80        return "success"

Result of a version-artifacts publish operation.

PublishArtifactsResult( connector_name: str, version: str, target: str, gcs_destination: str, files_uploaded: list[str] = <factory>, errors: list[str] = <factory>, validation_errors: list[str] = <factory>, progressive_rollout_overridden_by_breaking_change: bool = False, progressive_rollout_overridden_by_published_ga: bool = False, dry_run: bool = False)
connector_name: str
version: str
target: str
gcs_destination: str
files_uploaded: list[str]
errors: list[str]
validation_errors: list[str]
progressive_rollout_overridden_by_breaking_change: bool = False
progressive_rollout_overridden_by_published_ga: bool = False
dry_run: bool = False
success: bool
70    @property
71    def success(self) -> bool:
72        return len(self.errors) == 0 and len(self.validation_errors) == 0
status: str
74    @property
75    def status(self) -> str:
76        if self.dry_run:
77            return "dry-run"
78        if self.errors or self.validation_errors:
79            return "completed-with-errors"
80        return "success"
@dataclass
class PurgeLatestResult:
157@dataclass
158class PurgeLatestResult:
159    """Result of a purge-latest operation."""
160
161    target: str
162    connectors_found: int = 0
163    latest_dirs_deleted: int = 0
164    errors: list[str] = field(default_factory=list)
165    dry_run: bool = False
166
167    @property
168    def status(self) -> str:
169        if self.dry_run:
170            return "dry-run"
171        if self.errors:
172            return "completed-with-errors"
173        return "success"
174
175    def summary(self) -> str:
176        return (
177            f"[{self.status}] Found {self.connectors_found} connectors, "
178            f"deleted {self.latest_dirs_deleted} latest/ directories. "
179            f"Errors: {len(self.errors)}."
180        )

Result of a purge-latest operation.

PurgeLatestResult( target: str, connectors_found: int = 0, latest_dirs_deleted: int = 0, errors: list[str] = <factory>, dry_run: bool = False)
target: str
connectors_found: int = 0
latest_dirs_deleted: int = 0
errors: list[str]
dry_run: bool = False
status: str
167    @property
168    def status(self) -> str:
169        if self.dry_run:
170            return "dry-run"
171        if self.errors:
172            return "completed-with-errors"
173        return "success"
def summary(self) -> str:
175    def summary(self) -> str:
176        return (
177            f"[{self.status}] Found {self.connectors_found} connectors, "
178            f"deleted {self.latest_dirs_deleted} latest/ directories. "
179            f"Errors: {len(self.errors)}."
180        )
@dataclass
class RebuildResult:
43@dataclass
44class RebuildResult:
45    """Result of a registry rebuild operation."""
46
47    source_bucket: str
48    output_mode: OutputMode
49    output_root: str
50    connectors_processed: int = 0
51    blobs_copied: int = 0
52    blobs_skipped: int = 0
53    errors: list[str] = field(default_factory=list)
54    dry_run: bool = False
55
56    @property
57    def status(self) -> str:
58        """Return the status of the rebuild operation."""
59        if self.dry_run:
60            return "dry-run"
61        if self.errors:
62            return "completed-with-errors"
63        return "success"
64
65    def summary(self) -> str:
66        """Return a human-readable summary."""
67        return (
68            f"[{self.status}] Rebuilt {self.connectors_processed} connectors, "
69            f"{self.blobs_copied} blobs copied, {self.blobs_skipped} skipped, "
70            f"{len(self.errors)} errors. Output: {self.output_root}"
71        )

Result of a registry rebuild operation.

RebuildResult( source_bucket: str, output_mode: Literal['local', 'gcs', 's3'], output_root: str, connectors_processed: int = 0, blobs_copied: int = 0, blobs_skipped: int = 0, errors: list[str] = <factory>, dry_run: bool = False)
source_bucket: str
output_mode: Literal['local', 'gcs', 's3']
output_root: str
connectors_processed: int = 0
blobs_copied: int = 0
blobs_skipped: int = 0
errors: list[str]
dry_run: bool = False
status: str
56    @property
57    def status(self) -> str:
58        """Return the status of the rebuild operation."""
59        if self.dry_run:
60            return "dry-run"
61        if self.errors:
62            return "completed-with-errors"
63        return "success"

Return the status of the rebuild operation.

def summary(self) -> str:
65    def summary(self) -> str:
66        """Return a human-readable summary."""
67        return (
68            f"[{self.status}] Rebuilt {self.connectors_processed} connectors, "
69            f"{self.blobs_copied} blobs copied, {self.blobs_skipped} skipped, "
70            f"{len(self.errors)} errors. Output: {self.output_root}"
71        )

Return a human-readable summary.

class Registry(abc.ABC):
 44class Registry(ABC):
 45    """A configured connector registry store.
 46
 47    A Registry is bound to a specific `airbyte_ops_mcp.registry.store.RegistryStore`
 48    (store type + env + optional prefix), and provides methods used by the CLI to
 49    read/write registry contents.
 50    """
 51
 52    def __init__(self, store: RegistryStore) -> None:
 53        self.store = store
 54
 55    @property
 56    def store_type(self) -> StoreType:
 57        return self.store.store_type
 58
 59    @property
 60    def bucket_name(self) -> str:
 61        return self.store.bucket
 62
 63    @property
 64    def prefix(self) -> str:
 65        return self.store.prefix
 66
 67    def _require_no_prefix(self, op_name: str) -> None:
 68        """Raise if this op doesn't support prefixed targets."""
 69
 70        if self.prefix:
 71            raise NotImplementedError(
 72                f"Operation '{op_name}' does not yet support store prefixes (got prefix='{self.prefix}')."
 73            )
 74
 75    # ---------------------------------------------------------------------
 76    # Read operations
 77    # ---------------------------------------------------------------------
 78
 79    @abstractmethod
 80    def list_connectors(
 81        self,
 82        *,
 83        support_level: SupportLevel | None = None,
 84        min_support_level: SupportLevel | None = None,
 85        connector_type: ConnectorType | None = None,
 86        language: ConnectorLanguage | None = None,
 87    ) -> list[str]:
 88        raise NotImplementedError(
 89            _op_not_implemented_message(self.store_type, "list_connectors")
 90        )
 91
 92    def list_connector_versions(self, connector_name: str) -> list[str]:
 93        raise NotImplementedError(
 94            _op_not_implemented_message(self.store_type, "list_connector_versions")
 95        )
 96
 97    def get_connector_metadata(
 98        self, connector_name: str, version: str = "latest"
 99    ) -> dict[str, Any]:
100        raise NotImplementedError(
101            _op_not_implemented_message(self.store_type, "get_connector_metadata")
102        )
103
104    def list_yanked_versions(
105        self,
106        *,
107        with_details: bool = True,
108    ) -> list[YankedVersion]:
109        raise NotImplementedError(
110            _op_not_implemented_message(self.store_type, "list_yanked_versions")
111        )
112
113    def get_yank_marker(
114        self,
115        connector_name: str,
116        version: str,
117    ) -> YankMarkerDetail | None:
118        raise NotImplementedError(
119            _op_not_implemented_message(self.store_type, "get_yank_marker")
120        )
121
122    # ---------------------------------------------------------------------
123    # Write / mutate operations
124    # ---------------------------------------------------------------------
125
126    def yank(
127        self,
128        connector_name: str,
129        version: str,
130        reason: str = "",
131        approval_url: str = "",
132        dry_run: bool = False,
133    ) -> YankResult:
134        raise NotImplementedError(_op_not_implemented_message(self.store_type, "yank"))
135
136    def unyank(
137        self,
138        connector_name: str,
139        version: str,
140        dry_run: bool = False,
141    ) -> YankResult:
142        raise NotImplementedError(
143            _op_not_implemented_message(self.store_type, "unyank")
144        )
145
146    def finalize_progressive_rollout_marker(
147        self,
148        connector_name: str,
149        outcome: Literal["promoted", "aborted"],
150        version: str | None = None,
151        dry_run: bool = False,
152    ) -> ProgressiveRolloutMarkerResult:
153        raise NotImplementedError(
154            _op_not_implemented_message(
155                self.store_type,
156                "finalize_progressive_rollout_marker",
157            )
158        )
159
160    def publish_version_artifacts(
161        self,
162        connector_name: str,
163        version: str,
164        artifacts_dir: Path,
165        dry_run: bool = False,
166        with_validate: bool = True,
167    ) -> PublishArtifactsResult:
168        raise NotImplementedError(
169            _op_not_implemented_message(self.store_type, "publish_version_artifacts")
170        )
171
172    def delete_dev_latest(
173        self,
174        connector_name: list[str] | None = None,
175        dry_run: bool = False,
176    ) -> PurgeLatestResult:
177        raise NotImplementedError(
178            _op_not_implemented_message(self.store_type, "delete_dev_latest")
179        )
180
181    def compile(
182        self,
183        connector_name: list[str] | None = None,
184        dry_run: bool = False,
185        with_secrets_mask: bool = False,
186        with_legacy_migration: str | None = None,
187        with_metrics: bool = True,
188        force: bool = False,
189    ) -> CompileResult:
190        raise NotImplementedError(
191            _op_not_implemented_message(self.store_type, "compile")
192        )
193
194    def marketing_stubs_check(self, repo_root: Path) -> dict[str, Any]:
195        raise NotImplementedError(
196            _op_not_implemented_message(self.store_type, "marketing_stubs_check")
197        )
198
199    def marketing_stubs_sync(
200        self,
201        repo_root: Path,
202        dry_run: bool = False,
203    ) -> dict[str, Any]:
204        raise NotImplementedError(
205            _op_not_implemented_message(self.store_type, "marketing_stubs_sync")
206        )
207
208    def mirror(
209        self,
210        output_mode: OutputMode,
211        output_path_root: str | None = None,
212        gcs_bucket: str | None = None,
213        s3_bucket: str | None = None,
214        dry_run: bool = False,
215        connector_name: list[str] | None = None,
216    ) -> RebuildResult:
217        raise NotImplementedError(
218            _op_not_implemented_message(self.store_type, "mirror")
219        )

A configured connector registry store.

A Registry is bound to a specific airbyte_ops_mcp.registry.store.RegistryStore (store type + env + optional prefix), and provides methods used by the CLI to read/write registry contents.

store
store_type: StoreType
55    @property
56    def store_type(self) -> StoreType:
57        return self.store.store_type
bucket_name: str
59    @property
60    def bucket_name(self) -> str:
61        return self.store.bucket
prefix: str
63    @property
64    def prefix(self) -> str:
65        return self.store.prefix
@abstractmethod
def list_connectors( self, *, support_level: SupportLevel | None = None, min_support_level: SupportLevel | None = None, connector_type: ConnectorType | None = None, language: ConnectorLanguage | None = None) -> list[str]:
79    @abstractmethod
80    def list_connectors(
81        self,
82        *,
83        support_level: SupportLevel | None = None,
84        min_support_level: SupportLevel | None = None,
85        connector_type: ConnectorType | None = None,
86        language: ConnectorLanguage | None = None,
87    ) -> list[str]:
88        raise NotImplementedError(
89            _op_not_implemented_message(self.store_type, "list_connectors")
90        )
def list_connector_versions(self, connector_name: str) -> list[str]:
92    def list_connector_versions(self, connector_name: str) -> list[str]:
93        raise NotImplementedError(
94            _op_not_implemented_message(self.store_type, "list_connector_versions")
95        )
def get_connector_metadata( self, connector_name: str, version: str = 'latest') -> dict[str, typing.Any]:
 97    def get_connector_metadata(
 98        self, connector_name: str, version: str = "latest"
 99    ) -> dict[str, Any]:
100        raise NotImplementedError(
101            _op_not_implemented_message(self.store_type, "get_connector_metadata")
102        )
def list_yanked_versions( self, *, with_details: bool = True) -> list[airbyte_ops_mcp.registry.yank.YankedVersion]:
104    def list_yanked_versions(
105        self,
106        *,
107        with_details: bool = True,
108    ) -> list[YankedVersion]:
109        raise NotImplementedError(
110            _op_not_implemented_message(self.store_type, "list_yanked_versions")
111        )
def get_yank_marker( self, connector_name: str, version: str) -> airbyte_ops_mcp.registry.yank.YankMarkerDetail | None:
113    def get_yank_marker(
114        self,
115        connector_name: str,
116        version: str,
117    ) -> YankMarkerDetail | None:
118        raise NotImplementedError(
119            _op_not_implemented_message(self.store_type, "get_yank_marker")
120        )
def yank( self, connector_name: str, version: str, reason: str = '', approval_url: str = '', dry_run: bool = False) -> YankResult:
126    def yank(
127        self,
128        connector_name: str,
129        version: str,
130        reason: str = "",
131        approval_url: str = "",
132        dry_run: bool = False,
133    ) -> YankResult:
134        raise NotImplementedError(_op_not_implemented_message(self.store_type, "yank"))
def unyank( self, connector_name: str, version: str, dry_run: bool = False) -> YankResult:
136    def unyank(
137        self,
138        connector_name: str,
139        version: str,
140        dry_run: bool = False,
141    ) -> YankResult:
142        raise NotImplementedError(
143            _op_not_implemented_message(self.store_type, "unyank")
144        )
def finalize_progressive_rollout_marker( self, connector_name: str, outcome: Literal['promoted', 'aborted'], version: str | None = None, dry_run: bool = False) -> airbyte_ops_mcp.registry.progressive_rollout_marker.ProgressiveRolloutMarkerResult:
146    def finalize_progressive_rollout_marker(
147        self,
148        connector_name: str,
149        outcome: Literal["promoted", "aborted"],
150        version: str | None = None,
151        dry_run: bool = False,
152    ) -> ProgressiveRolloutMarkerResult:
153        raise NotImplementedError(
154            _op_not_implemented_message(
155                self.store_type,
156                "finalize_progressive_rollout_marker",
157            )
158        )
def publish_version_artifacts( self, connector_name: str, version: str, artifacts_dir: pathlib.Path, dry_run: bool = False, with_validate: bool = True) -> PublishArtifactsResult:
160    def publish_version_artifacts(
161        self,
162        connector_name: str,
163        version: str,
164        artifacts_dir: Path,
165        dry_run: bool = False,
166        with_validate: bool = True,
167    ) -> PublishArtifactsResult:
168        raise NotImplementedError(
169            _op_not_implemented_message(self.store_type, "publish_version_artifacts")
170        )
def delete_dev_latest( self, connector_name: list[str] | None = None, dry_run: bool = False) -> PurgeLatestResult:
172    def delete_dev_latest(
173        self,
174        connector_name: list[str] | None = None,
175        dry_run: bool = False,
176    ) -> PurgeLatestResult:
177        raise NotImplementedError(
178            _op_not_implemented_message(self.store_type, "delete_dev_latest")
179        )
def compile( self, connector_name: list[str] | None = None, dry_run: bool = False, with_secrets_mask: bool = False, with_legacy_migration: str | None = None, with_metrics: bool = True, force: bool = False) -> CompileResult:
181    def compile(
182        self,
183        connector_name: list[str] | None = None,
184        dry_run: bool = False,
185        with_secrets_mask: bool = False,
186        with_legacy_migration: str | None = None,
187        with_metrics: bool = True,
188        force: bool = False,
189    ) -> CompileResult:
190        raise NotImplementedError(
191            _op_not_implemented_message(self.store_type, "compile")
192        )
def marketing_stubs_check(self, repo_root: pathlib.Path) -> dict[str, typing.Any]:
194    def marketing_stubs_check(self, repo_root: Path) -> dict[str, Any]:
195        raise NotImplementedError(
196            _op_not_implemented_message(self.store_type, "marketing_stubs_check")
197        )
def marketing_stubs_sync( self, repo_root: pathlib.Path, dry_run: bool = False) -> dict[str, typing.Any]:
199    def marketing_stubs_sync(
200        self,
201        repo_root: Path,
202        dry_run: bool = False,
203    ) -> dict[str, Any]:
204        raise NotImplementedError(
205            _op_not_implemented_message(self.store_type, "marketing_stubs_sync")
206        )
def mirror( self, output_mode: Literal['local', 'gcs', 's3'], output_path_root: str | None = None, gcs_bucket: str | None = None, s3_bucket: str | None = None, dry_run: bool = False, connector_name: list[str] | None = None) -> RebuildResult:
208    def mirror(
209        self,
210        output_mode: OutputMode,
211        output_path_root: str | None = None,
212        gcs_bucket: str | None = None,
213        s3_bucket: str | None = None,
214        dry_run: bool = False,
215        connector_name: list[str] | None = None,
216    ) -> RebuildResult:
217        raise NotImplementedError(
218            _op_not_implemented_message(self.store_type, "mirror")
219        )
class RegistryEntryResult(pydantic.main.BaseModel):
60class RegistryEntryResult(BaseModel):
61    """Result of reading a registry entry from GCS.
62
63    This model wraps the raw metadata dictionary with additional context.
64    """
65
66    connector_name: str = Field(description="The connector technical name")
67    version: str = Field(description="The version that was read")
68    bucket_name: str = Field(description="The GCS bucket name")
69    gcs_path: str = Field(description="The GCS path that was read")
70    metadata: dict = Field(description="The raw metadata dictionary")

Result of reading a registry entry from GCS.

This model wraps the raw metadata dictionary with additional context.

connector_name: str = PydanticUndefined

The connector technical name

version: str = PydanticUndefined

The version that was read

bucket_name: str = PydanticUndefined

The GCS bucket name

gcs_path: str = PydanticUndefined

The GCS path that was read

metadata: dict = PydanticUndefined

The raw metadata dictionary

@dataclass(frozen=True, kw_only=True)
class RegistryStore:
132@dataclass(frozen=True, kw_only=True)
133class RegistryStore:
134    """Parsed store target (type + environment + optional prefix).
135
136    Examples::
137
138        RegistryStore.parse("coral:dev")
139        # -> RegistryStore(store_type=StoreType.CORAL, env="dev", prefix="")
140        RegistryStore.parse("coral:dev/aj-test")
141        # -> RegistryStore(store_type=StoreType.CORAL, env="dev", prefix="aj-test")
142        RegistryStore.parse("sonar:prod")
143        # -> RegistryStore(store_type=StoreType.SONAR, env="prod", prefix="")
144    """
145
146    store_type: StoreType
147    env: str
148    prefix: str = ""
149
150    # -- Derived helpers -----------------------------------------------------
151
152    @property
153    def bucket(self) -> str:
154        """Resolve the concrete bucket name for this target."""
155        env_map = BUCKET_MAP.get(self.store_type)
156        if env_map is None:
157            raise ValueError(f"Unknown store type: {self.store_type!r}")
158        bucket_name = env_map.get(self.env)
159        if bucket_name is None:
160            raise ValueError(
161                f"Unknown environment '{self.env}' for store type '{self.store_type.value}'. "
162                f"Expected one of: {', '.join(sorted(env_map))}."
163            )
164        return bucket_name
165
166    @property
167    def bucket_root(self) -> str:
168        """Bucket name with optional prefix appended (`bucket/prefix`)."""
169        if self.prefix:
170            return f"{self.bucket}/{self.prefix}"
171        return self.bucket
172
173    # -- Parsing -------------------------------------------------------------
174
175    @classmethod
176    def parse(cls, target: str) -> RegistryStore:
177        """Parse a store target string.
178
179        Accepted formats:
180
181            "coral:dev"
182
183            "coral:prod"
184            "coral:dev/aj-test100"
185            "sonar:prod"
186
187        Raises:
188            ValueError: If the string cannot be parsed or references an
189                unknown store type / environment.
190        """
191        if ":" not in target:
192            raise ValueError(
193                f"Invalid store target '{target}'. "
194                "Expected format: '<store_type>:<env>[/<prefix>]' "
195                "(e.g. 'coral:dev', 'sonar:prod', 'coral:dev/my-test')."
196            )
197
198        store_part, env_part = target.split(":", 1)
199
200        # Validate store type
201        store_part_lower = store_part.lower()
202        valid_types = {t.value: t for t in StoreType}
203        if store_part_lower not in valid_types:
204            raise ValueError(
205                f"Unknown store type '{store_part}'. "
206                f"Expected one of: {', '.join(sorted(valid_types))}."
207            )
208        store_type = valid_types[store_part_lower]
209
210        # Split env and prefix
211        env_key, _, prefix = env_part.partition("/")
212        prefix = prefix.strip("/")
213
214        # Validate env
215        env_map = BUCKET_MAP.get(store_type, {})
216        if env_key not in env_map:
217            raise ValueError(
218                f"Unknown environment '{env_key}' for store type '{store_type.value}'. "
219                f"Expected one of: {', '.join(sorted(env_map))}."
220            )
221
222        return cls(store_type=store_type, env=env_key, prefix=prefix)

Parsed store target (type + environment + optional prefix).

Examples::

RegistryStore.parse("coral:dev")
# -> RegistryStore(store_type=StoreType.CORAL, env="dev", prefix="")
RegistryStore.parse("coral:dev/aj-test")
# -> RegistryStore(store_type=StoreType.CORAL, env="dev", prefix="aj-test")
RegistryStore.parse("sonar:prod")
# -> RegistryStore(store_type=StoreType.SONAR, env="prod", prefix="")
RegistryStore( *, store_type: StoreType, env: str, prefix: str = '')
store_type: StoreType
env: str
prefix: str = ''
bucket: str
152    @property
153    def bucket(self) -> str:
154        """Resolve the concrete bucket name for this target."""
155        env_map = BUCKET_MAP.get(self.store_type)
156        if env_map is None:
157            raise ValueError(f"Unknown store type: {self.store_type!r}")
158        bucket_name = env_map.get(self.env)
159        if bucket_name is None:
160            raise ValueError(
161                f"Unknown environment '{self.env}' for store type '{self.store_type.value}'. "
162                f"Expected one of: {', '.join(sorted(env_map))}."
163            )
164        return bucket_name

Resolve the concrete bucket name for this target.

bucket_root: str
166    @property
167    def bucket_root(self) -> str:
168        """Bucket name with optional prefix appended (`bucket/prefix`)."""
169        if self.prefix:
170            return f"{self.bucket}/{self.prefix}"
171        return self.bucket

Bucket name with optional prefix appended (bucket/prefix).

@classmethod
def parse(cls, target: str) -> RegistryStore:
175    @classmethod
176    def parse(cls, target: str) -> RegistryStore:
177        """Parse a store target string.
178
179        Accepted formats:
180
181            "coral:dev"
182
183            "coral:prod"
184            "coral:dev/aj-test100"
185            "sonar:prod"
186
187        Raises:
188            ValueError: If the string cannot be parsed or references an
189                unknown store type / environment.
190        """
191        if ":" not in target:
192            raise ValueError(
193                f"Invalid store target '{target}'. "
194                "Expected format: '<store_type>:<env>[/<prefix>]' "
195                "(e.g. 'coral:dev', 'sonar:prod', 'coral:dev/my-test')."
196            )
197
198        store_part, env_part = target.split(":", 1)
199
200        # Validate store type
201        store_part_lower = store_part.lower()
202        valid_types = {t.value: t for t in StoreType}
203        if store_part_lower not in valid_types:
204            raise ValueError(
205                f"Unknown store type '{store_part}'. "
206                f"Expected one of: {', '.join(sorted(valid_types))}."
207            )
208        store_type = valid_types[store_part_lower]
209
210        # Split env and prefix
211        env_key, _, prefix = env_part.partition("/")
212        prefix = prefix.strip("/")
213
214        # Validate env
215        env_map = BUCKET_MAP.get(store_type, {})
216        if env_key not in env_map:
217            raise ValueError(
218                f"Unknown environment '{env_key}' for store type '{store_type.value}'. "
219                f"Expected one of: {', '.join(sorted(env_map))}."
220            )
221
222        return cls(store_type=store_type, env=env_key, prefix=prefix)

Parse a store target string.

Accepted formats:

"coral:dev"

"coral:prod" "coral:dev/aj-test100" "sonar:prod"

Raises:
  • ValueError: If the string cannot be parsed or references an unknown store type / environment.
class StoreType(builtins.str, enum.Enum):
 59class StoreType(str, Enum):
 60    """Registry store type identifier."""
 61
 62    SONAR = "sonar"
 63    CORAL = "coral"
 64
 65    # -- Auto-detection class methods ----------------------------------------
 66
 67    @classmethod
 68    def get_from_connector_name(cls, name: str) -> StoreType:
 69        """Infer the store type from a connector's technical name.
 70
 71        Connectors whose name starts with `source-` or `destination-` belong
 72        to the **coral** registry.  All other names belong to **sonar**.
 73
 74        Args:
 75            name: Connector technical name (e.g. `"source-github"` or `"stripe"`).
 76
 77        Returns:
 78            The inferred `StoreType`.
 79        """
 80        if name.startswith("source-") or name.startswith("destination-"):
 81            return cls.CORAL
 82        return cls.SONAR
 83
 84    @classmethod
 85    def detect_from_repo_dir(cls, path: Path | None = None) -> StoreType | None:
 86        """Infer the store type from a repository working directory.
 87
 88        Checks for well-known directory markers:
 89
 90        * **sonar** -- `integrations/` alongside `connector-sdk/`
 91        * **coral** -- `airbyte-integrations/connectors/`
 92
 93        Args:
 94            path: Directory to inspect.  Defaults to `Path.cwd`.
 95
 96        Returns:
 97            The inferred `StoreType`, or `None` if the directory does
 98            not match any known registry repository layout.
 99        """
100        if path is None:
101            path = Path.cwd()
102
103        # Sonar markers
104        if (path / "integrations").is_dir() and (path / "connector-sdk").is_dir():
105            return cls.SONAR
106
107        # Coral markers
108        if (path / "airbyte-integrations" / "connectors").is_dir():
109            return cls.CORAL
110
111        return None

Registry store type identifier.

SONAR = <StoreType.SONAR: 'sonar'>
CORAL = <StoreType.CORAL: 'coral'>
@classmethod
def get_from_connector_name(cls, name: str) -> StoreType:
67    @classmethod
68    def get_from_connector_name(cls, name: str) -> StoreType:
69        """Infer the store type from a connector's technical name.
70
71        Connectors whose name starts with `source-` or `destination-` belong
72        to the **coral** registry.  All other names belong to **sonar**.
73
74        Args:
75            name: Connector technical name (e.g. `"source-github"` or `"stripe"`).
76
77        Returns:
78            The inferred `StoreType`.
79        """
80        if name.startswith("source-") or name.startswith("destination-"):
81            return cls.CORAL
82        return cls.SONAR

Infer the store type from a connector's technical name.

Connectors whose name starts with source- or destination- belong to the coral registry. All other names belong to sonar.

Arguments:
  • name: Connector technical name (e.g. "source-github" or "stripe").
Returns:

The inferred StoreType.

@classmethod
def detect_from_repo_dir( cls, path: pathlib.Path | None = None) -> StoreType | None:
 84    @classmethod
 85    def detect_from_repo_dir(cls, path: Path | None = None) -> StoreType | None:
 86        """Infer the store type from a repository working directory.
 87
 88        Checks for well-known directory markers:
 89
 90        * **sonar** -- `integrations/` alongside `connector-sdk/`
 91        * **coral** -- `airbyte-integrations/connectors/`
 92
 93        Args:
 94            path: Directory to inspect.  Defaults to `Path.cwd`.
 95
 96        Returns:
 97            The inferred `StoreType`, or `None` if the directory does
 98            not match any known registry repository layout.
 99        """
100        if path is None:
101            path = Path.cwd()
102
103        # Sonar markers
104        if (path / "integrations").is_dir() and (path / "connector-sdk").is_dir():
105            return cls.SONAR
106
107        # Coral markers
108        if (path / "airbyte-integrations" / "connectors").is_dir():
109            return cls.CORAL
110
111        return None

Infer the store type from a repository working directory.

Checks for well-known directory markers:

  • sonar -- integrations/ alongside connector-sdk/
  • coral -- airbyte-integrations/connectors/
Arguments:
  • path: Directory to inspect. Defaults to Path.cwd.
Returns:

The inferred StoreType, or None if the directory does not match any known registry repository layout.

class SupportLevel(enum.StrEnum):
14class SupportLevel(StrEnum):
15    """Connector support levels ordered by precedence."""
16
17    ARCHIVED = "archived"
18    COMMUNITY = "community"
19    CERTIFIED = "certified"
20
21    @property
22    def precedence(self) -> int:
23        """Numeric precedence for ordering comparisons.
24
25        Higher values indicate higher support commitment.
26        """
27        return _SUPPORT_LEVEL_PRECEDENCE[self]
28
29    @classmethod
30    def from_precedence(cls, precedence: int) -> SupportLevel:
31        """Look up a `SupportLevel` by its numeric precedence value.
32
33        Raises `ValueError` when the precedence is not recognised.
34        """
35        for member in cls:
36            if _SUPPORT_LEVEL_PRECEDENCE[member] == precedence:
37                return member
38        valid = ", ".join(f"`{_SUPPORT_LEVEL_PRECEDENCE[m]}`" for m in cls)
39        raise ValueError(
40            f"Unrecognized support-level precedence: {precedence!r}. "
41            f"Expected one of: {valid}."
42        ) from None
43
44    @classmethod
45    def parse(cls, value: str) -> SupportLevel:
46        """Parse a string into a `SupportLevel`.
47
48        Accepts a keyword (`archived`, `community`, `certified`)
49        or a legacy integer string (`100`, `200`, `300`).
50
51        Raises `ValueError` when the value is not recognised.
52        """
53        try:
54            return cls(value)
55        except ValueError:
56            pass
57        # Fallback: try interpreting as an integer precedence value.
58        try:
59            return cls.from_precedence(int(value))
60        except (ValueError, KeyError):
61            pass
62        valid_kw = ", ".join(f"`{m.value}`" for m in cls)
63        valid_int = ", ".join(f"`{_SUPPORT_LEVEL_PRECEDENCE[m]}`" for m in cls)
64        raise ValueError(
65            f"Unrecognized support level: {value!r}. "
66            f"Expected keyword ({valid_kw}) or integer ({valid_int})."
67        ) from None

Connector support levels ordered by precedence.

ARCHIVED = <SupportLevel.ARCHIVED: 'archived'>
COMMUNITY = <SupportLevel.COMMUNITY: 'community'>
CERTIFIED = <SupportLevel.CERTIFIED: 'certified'>
precedence: int
21    @property
22    def precedence(self) -> int:
23        """Numeric precedence for ordering comparisons.
24
25        Higher values indicate higher support commitment.
26        """
27        return _SUPPORT_LEVEL_PRECEDENCE[self]

Numeric precedence for ordering comparisons.

Higher values indicate higher support commitment.

@classmethod
def from_precedence(cls, precedence: int) -> SupportLevel:
29    @classmethod
30    def from_precedence(cls, precedence: int) -> SupportLevel:
31        """Look up a `SupportLevel` by its numeric precedence value.
32
33        Raises `ValueError` when the precedence is not recognised.
34        """
35        for member in cls:
36            if _SUPPORT_LEVEL_PRECEDENCE[member] == precedence:
37                return member
38        valid = ", ".join(f"`{_SUPPORT_LEVEL_PRECEDENCE[m]}`" for m in cls)
39        raise ValueError(
40            f"Unrecognized support-level precedence: {precedence!r}. "
41            f"Expected one of: {valid}."
42        ) from None

Look up a SupportLevel by its numeric precedence value.

Raises ValueError when the precedence is not recognised.

@classmethod
def parse(cls, value: str) -> SupportLevel:
44    @classmethod
45    def parse(cls, value: str) -> SupportLevel:
46        """Parse a string into a `SupportLevel`.
47
48        Accepts a keyword (`archived`, `community`, `certified`)
49        or a legacy integer string (`100`, `200`, `300`).
50
51        Raises `ValueError` when the value is not recognised.
52        """
53        try:
54            return cls(value)
55        except ValueError:
56            pass
57        # Fallback: try interpreting as an integer precedence value.
58        try:
59            return cls.from_precedence(int(value))
60        except (ValueError, KeyError):
61            pass
62        valid_kw = ", ".join(f"`{m.value}`" for m in cls)
63        valid_int = ", ".join(f"`{_SUPPORT_LEVEL_PRECEDENCE[m]}`" for m in cls)
64        raise ValueError(
65            f"Unrecognized support level: {value!r}. "
66            f"Expected keyword ({valid_kw}) or integer ({valid_int})."
67        ) from None

Parse a string into a SupportLevel.

Accepts a keyword (archived, community, certified) or a legacy integer string (100, 200, 300).

Raises ValueError when the value is not recognised.

@dataclass
class UnpublishedConnector:
29@dataclass
30class UnpublishedConnector:
31    """A connector whose current local version is not published on GCS."""
32
33    connector_name: str
34    local_version: str

A connector whose current local version is not published on GCS.

UnpublishedConnector(connector_name: str, local_version: str)
connector_name: str
local_version: str
@dataclass(frozen=True)
class ValidateOptions:
37@dataclass(frozen=True)
38class ValidateOptions:
39    """Options that influence which validators run and how."""
40
41    docs_path: str | None = None
42    """Path to the connector's documentation file (for `validate_docs_path_exists`)."""
43
44    is_prerelease: bool = False
45    """Whether this is a pre-release build (skips version-decrement checks)."""

Options that influence which validators run and how.

ValidateOptions(docs_path: str | None = None, is_prerelease: bool = False)
docs_path: str | None = None

Path to the connector's documentation file (for validate_docs_path_exists).

is_prerelease: bool = False

Whether this is a pre-release build (skips version-decrement checks).

@dataclass
class ValidationResult:
48@dataclass
49class ValidationResult:
50    """Aggregate result from running all validators."""
51
52    passed: bool = True
53    errors: list[str] = field(default_factory=list)
54    validators_run: int = 0
55
56    def add_error(self, message: str) -> None:
57        self.passed = False
58        self.errors.append(message)

Aggregate result from running all validators.

ValidationResult( passed: bool = True, errors: list[str] = <factory>, validators_run: int = 0)
passed: bool = True
errors: list[str]
validators_run: int = 0
def add_error(self, message: str) -> None:
56    def add_error(self, message: str) -> None:
57        self.passed = False
58        self.errors.append(message)
class VersionListResult(pydantic.main.BaseModel):
81class VersionListResult(BaseModel):
82    """Result of listing versions for a connector."""
83
84    connector_name: str = Field(description="The connector technical name")
85    bucket_name: str = Field(description="The GCS bucket name")
86    version_count: int = Field(description="Number of versions found")
87    versions: list[str] = Field(description="List of version strings")

Result of listing versions for a connector.

connector_name: str = PydanticUndefined

The connector technical name

bucket_name: str = PydanticUndefined

The GCS bucket name

version_count: int = PydanticUndefined

Number of versions found

versions: list[str] = PydanticUndefined

List of version strings

@dataclass
class YankResult:
34@dataclass
35class YankResult:
36    """Result of a yank or unyank operation."""
37
38    connector_name: str
39    version: str
40    bucket_name: str
41    action: str  # "yank" or "unyank"
42    success: bool
43    message: str
44    dry_run: bool = False
45
46    def to_dict(self) -> dict[str, Any]:
47        """Convert to a dictionary for JSON serialization."""
48        return {
49            "connector_name": self.connector_name,
50            "version": self.version,
51            "bucket_name": self.bucket_name,
52            "action": self.action,
53            "success": self.success,
54            "message": self.message,
55            "dry_run": self.dry_run,
56        }

Result of a yank or unyank operation.

YankResult( connector_name: str, version: str, bucket_name: str, action: str, success: bool, message: str, dry_run: bool = False)
connector_name: str
version: str
bucket_name: str
action: str
success: bool
message: str
dry_run: bool = False
def to_dict(self) -> dict[str, typing.Any]:
46    def to_dict(self) -> dict[str, Any]:
47        """Convert to a dictionary for JSON serialization."""
48        return {
49            "connector_name": self.connector_name,
50            "version": self.version,
51            "bucket_name": self.bucket_name,
52            "action": self.action,
53            "success": self.success,
54            "message": self.message,
55            "dry_run": self.dry_run,
56        }

Convert to a dictionary for JSON serialization.

def compile_registry( *, store: RegistryStore, connector_name: list[str] | None = None, dry_run: bool = False, with_secrets_mask: bool = False, with_legacy_migration: str | None = None, with_metrics: bool = True, force: bool = False) -> CompileResult:
1624def compile_registry(
1625    *,
1626    store: RegistryStore,
1627    connector_name: list[str] | None = None,
1628    dry_run: bool = False,
1629    with_secrets_mask: bool = False,
1630    with_legacy_migration: str | None = None,
1631    with_metrics: bool = True,
1632    force: bool = False,
1633) -> CompileResult:
1634    """Compile the registry: sync latest/ dirs and write index files.
1635
1636    Steps:
1637        1. Glob for all `metadata.yaml` to discover (connector, version) pairs.
1638        2. Glob for active marker files.
1639        3. Compute the latest GA semver per connector.
1640        4. Compute active release candidates from versioned markers.
1641        5. Glob for `version=*` markers in `latest/` dirs for a fast check.
1642        6. Delete stale `latest/` dirs and recursively copy the versioned dir.
1643        7. Synthesize missing registry entries from pinned latest overrides.
1644        8. (Optional) Legacy migration: delete disabled registry entries.
1645        9. (Optional) Read latest connector metrics.
1646        10. Write global registry JSONs.
1647        11. Write composite registry JSON.
1648        12. Write per-connector `versions.json`.
1649        13. (Optional) Regenerate `specs_secrets_mask.yaml`.
1650
1651    Args:
1652        store: Registry store (bucket + optional prefix).
1653        connector_name: If provided, only resync `latest/` directories for
1654            these connectors (steps 5-6).  Index rebuilds (steps 9-12)
1655            always operate on the full set of connectors so that global
1656            registry files remain complete.
1657        dry_run: If True, report what would be done without writing.
1658        with_secrets_mask: If True, regenerate `specs_secrets_mask.yaml`.
1659        with_legacy_migration: If set, run the named migration step.
1660            Currently supported: `"v1"` — delete `{registry_type}.json`
1661            files for connectors whose `registryOverrides.{registry}.enabled`
1662            is `false`.
1663        with_metrics: If True, inject latest connector metrics from the
1664            analytics JSONL export into `generated.metrics`.
1665        force: If True, resync all connectors' latest/ directories even if the
1666            existing version marker matches the computed latest version. This
1667            is useful when metadata content changes without a version bump.
1668
1669    Returns:
1670        A `CompileResult` describing what was done.
1671    """
1672    if with_legacy_migration and with_legacy_migration not in LEGACY_MIGRATION_VERSIONS:
1673        raise ValueError(
1674            f"Unknown legacy migration version: {with_legacy_migration!r}. "
1675            f"Supported: {', '.join(LEGACY_MIGRATION_VERSIONS)}"
1676        )
1677
1678    result = CompileResult(target=store.bucket_root, dry_run=dry_run)
1679
1680    token = get_gcs_credentials_token()
1681    fs = gcsfs.GCSFileSystem(token=token)
1682
1683    # --- Steps 1 and 2: Scan versions and active markers ---
1684    # Always scan ALL connectors so that index rebuilds are complete.
1685    _log_progress("Step 1-2: Scanning versions and active markers...")
1686    connector_versions, yanked, progressive_rollouts = _scan_versions_and_markers(
1687        fs,
1688        store=store,
1689        connector_name=None,
1690    )
1691    result.connectors_scanned = len(connector_versions)
1692    result.versions_found = sum(len(v) for v in connector_versions.values())
1693    result.yanked_versions = len(yanked)
1694    _log_progress(
1695        "  Found %d connectors, %d versions, %d yanked",
1696        result.connectors_scanned,
1697        result.versions_found,
1698        result.yanked_versions,
1699    )
1700    _log_progress("  Found %d progressive rollout markers", len(progressive_rollouts))
1701
1702    # --- Step 3: Compute latest ---
1703    _log_progress("Step 3: Computing latest GA version per connector...")
1704    latest_versions = _compute_latest_versions(
1705        connector_versions=connector_versions,
1706        yanked=yanked,
1707        progressive_rollouts=progressive_rollouts,
1708    )
1709    _log_progress("  Computed latest for %d connectors", len(latest_versions))
1710
1711    # --- Step 4: Compute release candidates ---
1712    _log_progress("Step 4: Computing active release candidates...")
1713    rc_versions = _compute_release_candidates(
1714        connector_versions=connector_versions,
1715        yanked=yanked,
1716        progressive_rollouts=progressive_rollouts,
1717    )
1718    _log_progress("  Computed %d active release candidates", len(rc_versions))
1719
1720    # --- Step 5: Check existing latest markers ---
1721    # When --connector-name is set, only check/sync those connectors (steps 5-6).
1722    # Index rebuilds always use the full unfiltered data.
1723    if connector_name:
1724        connector_name_set = set(connector_name)
1725        sync_scope = {
1726            c: v for c, v in latest_versions.items() if c in connector_name_set
1727        }
1728        _log_progress(
1729            "  --connector-name filter: syncing %d of %d connectors",
1730            len(sync_scope),
1731            len(latest_versions),
1732        )
1733    else:
1734        sync_scope = latest_versions
1735
1736    _log_progress("Step 5: Checking existing latest/ markers...")
1737    sync_scope_names = list(sync_scope) if connector_name else None
1738    existing_markers = _scan_latest_markers(
1739        fs,
1740        store=store,
1741        connector_name=sync_scope_names,
1742    )
1743    _log_progress("  Found %d existing markers", len(existing_markers))
1744
1745    stale_connectors: list[str] = []
1746    pinned_override_synthesis_connectors: list[str] = []
1747    for connector, expected_version in sync_scope.items():
1748        current_marker = existing_markers.get(connector)
1749        if force or current_marker != expected_version:
1750            stale_connectors.append(connector)
1751            continue
1752
1753        if _requires_pinned_override_synthesis(
1754            fs,
1755            store=store,
1756            connector=connector,
1757            version=expected_version,
1758        ):
1759            pinned_override_synthesis_connectors.append(connector)
1760            continue
1761
1762        result.latest_already_current += 1
1763
1764    _log_progress(
1765        "  %d connectors need latest/ update, %d already current",
1766        len(stale_connectors),
1767        result.latest_already_current,
1768    )
1769
1770    # --- Step 6: Resync stale latest/ dirs (parallel) ---
1771    if stale_connectors:
1772        _log_progress(
1773            "Step 6: Syncing %d stale latest/ directories (max_workers=%d)...",
1774            len(stale_connectors),
1775            _COMPILE_SYNC_MAX_WORKERS,
1776        )
1777
1778        def _sync_one_connector(connector: str) -> None:
1779            """Sync a single connector's latest/ dir."""
1780            version = latest_versions[connector]
1781            _sync_latest_dir(
1782                fs,
1783                store=store,
1784                connector=connector,
1785                version=version,
1786                dry_run=dry_run,
1787            )
1788            if not dry_run:
1789                _apply_overrides_to_latest_entry(
1790                    fs,
1791                    store=store,
1792                    connector=connector,
1793                    version=version,
1794                )
1795
1796        sorted_stale = sorted(stale_connectors)
1797        with ThreadPoolExecutor(max_workers=_COMPILE_SYNC_MAX_WORKERS) as pool:
1798            futures = {pool.submit(_sync_one_connector, c): c for c in sorted_stale}
1799            for i, future in enumerate(as_completed(futures), 1):
1800                connector = futures[future]
1801                try:
1802                    future.result()
1803                    result.latest_updated += 1
1804                except Exception as exc:
1805                    error_msg = f"Failed to sync latest/ for {connector}: {exc}"
1806                    logger.error(error_msg)
1807                    result.errors.append(error_msg)
1808                    # Delete the (possibly partial) latest/ dir so the next
1809                    # compile retries this connector from scratch.
1810                    try:
1811                        _delete_latest_dir(
1812                            fs,
1813                            store=store,
1814                            connector=connector,
1815                        )
1816                        logger.info(
1817                            "Cleaned up partial latest/ for %s after failure",
1818                            connector,
1819                        )
1820                    except Exception as cleanup_exc:
1821                        logger.warning(
1822                            "Could not clean up latest/ for %s: %s",
1823                            connector,
1824                            cleanup_exc,
1825                        )
1826                if i % 100 == 0:
1827                    _log_progress("  Synced %d / %d...", i, len(sorted_stale))
1828    else:
1829        _log_progress("Step 6: All latest/ directories are current, nothing to sync.")
1830
1831    # --- Step 7: Synthesize missing latest entries from pinned overrides ---
1832    if pinned_override_synthesis_connectors:
1833        _log_progress(
1834            "Step 7: Synthesizing %d latest/ registry entries from pinned overrides...",
1835            len(pinned_override_synthesis_connectors),
1836        )
1837        for connector in sorted(pinned_override_synthesis_connectors):
1838            if dry_run:
1839                _log_progress(
1840                    "  [DRY RUN] Would synthesize pinned latest entries for %s",
1841                    connector,
1842                )
1843                result.latest_updated += 1
1844                continue
1845            try:
1846                _apply_overrides_to_latest_entry(
1847                    fs,
1848                    store=store,
1849                    connector=connector,
1850                    version=sync_scope[connector],
1851                )
1852                result.latest_updated += 1
1853            except Exception as exc:
1854                error_msg = (
1855                    f"Failed to synthesize pinned latest entries for {connector}: {exc}"
1856                )
1857                logger.error(error_msg)
1858                result.errors.append(error_msg)
1859
1860    # --- Step 8: Legacy migration (optional) ---
1861    if with_legacy_migration == "v1":
1862        _log_progress(
1863            "Step 8: Legacy migration v1 — deleting disabled registry entries..."
1864        )
1865        migration_deleted = _cleanup_disabled_registry_entries(
1866            fs,
1867            store=store,
1868            connector_versions=connector_versions,
1869            dry_run=dry_run,
1870        )
1871        total_deleted = sum(len(v) for v in migration_deleted.values())
1872        if migration_deleted:
1873            for conn, paths in sorted(migration_deleted.items()):
1874                _log_progress(
1875                    "  %s: %s %d files",
1876                    conn,
1877                    "would delete" if dry_run else "deleted",
1878                    len(paths),
1879                )
1880        _log_progress(
1881            "  Migration v1: %s %d files across %d connectors",
1882            "would delete" if dry_run else "deleted",
1883            total_deleted,
1884            len(migration_deleted),
1885        )
1886
1887    # --- Step 9: Read latest connector metrics (optional) ---
1888    metrics_bundle = None
1889    if with_metrics and store.store_type == StoreType.CORAL:
1890        _log_progress("Step 9: Reading latest connector metrics JSONL...")
1891        try:
1892            metrics_bundle = read_latest_connector_metrics()
1893            result.metrics_source = metrics_bundle.blob_path
1894            result.metrics_connector_count = metrics_bundle.connector_count
1895            if metrics_bundle.blob_path:
1896                _log_progress(
1897                    "  Loaded metrics for %d connectors from gs://%s",
1898                    metrics_bundle.connector_count,
1899                    metrics_bundle.blob_path,
1900                )
1901            else:
1902                _log_progress("  No connector metrics JSONL file found.")
1903        except Exception as exc:
1904            error_msg = f"Failed to read connector metrics JSONL: {exc}"
1905            logger.warning(error_msg)
1906            result.metrics_error = error_msg
1907            _log_progress("  %s", error_msg)
1908    elif with_metrics:
1909        _log_progress("Step 9: Skipping connector metrics for non-coral registry.")
1910    else:
1911        _log_progress("Step 9: Connector metrics injection disabled.")
1912
1913    # --- Step 10: Compile global registry JSONs ---
1914    _log_progress("Step 10: Compiling global registry JSON files...")
1915    all_registry_entries: list[dict[str, Any]] = []  # collected for Step 13
1916    entries_by_registry_type: dict[str, list[dict[str, Any]]] = {}
1917    for registry_type in VALID_REGISTRIES:
1918        entries = _compile_global_registry(
1919            fs,
1920            store=store,
1921            latest_versions=latest_versions,
1922            registry_type=registry_type,
1923        )
1924
1925        # Inject release candidate info into entries that have active RCs.
1926        if rc_versions:
1927            rc_entries: dict[str, list[dict[str, Any]]] = {}
1928            for connector, rc_ver_list in rc_versions.items():
1929                for rc_ver in rc_ver_list:
1930                    rc_entry = _read_rc_registry_entry(
1931                        fs,
1932                        store=store,
1933                        connector=connector,
1934                        rc_version=rc_ver,
1935                        registry_type=registry_type,
1936                    )
1937                    if rc_entry:
1938                        docker_repo = rc_entry.get(
1939                            "dockerRepository",
1940                            f"airbyte/{connector}",
1941                        )
1942                        rc_entries.setdefault(docker_repo, []).append(
1943                            {
1944                                "version": rc_ver,
1945                                "entry": rc_entry,
1946                            }
1947                        )
1948            if rc_entries:
1949                entries = _apply_release_candidates_to_entries(entries, rc_entries)
1950                total_rcs = sum(len(v) for v in rc_entries.values())
1951                _log_progress(
1952                    "  Injected %d release candidate(s) for %d connector(s) into %s registry",
1953                    total_rcs,
1954                    len(rc_entries),
1955                    registry_type,
1956                )
1957
1958        if metrics_bundle is not None:
1959            injected = apply_metrics_to_registry_entries(entries, metrics_bundle)
1960            result.metrics_registry_entries += injected
1961            _log_progress(
1962                "  Injected metrics into %d %s registry entries",
1963                injected,
1964                registry_type,
1965            )
1966
1967        all_registry_entries.extend(entries)
1968        entries_by_registry_type[registry_type] = entries
1969        registry_json = _build_global_registry_json(entries)
1970        entry_count = len(registry_json["sources"]) + len(registry_json["destinations"])
1971
1972        if registry_type == "cloud":
1973            result.cloud_registry_entries = entry_count
1974        else:
1975            result.oss_registry_entries = entry_count
1976
1977        if dry_run:
1978            _log_progress(
1979                "  [DRY RUN] Would write %s_registry.json (%d entries)",
1980                registry_type,
1981                entry_count,
1982            )
1983        else:
1984            content = json.dumps(registry_json, indent=2, sort_keys=True) + "\n"
1985            path_prefix = f"{store.prefix}/" if store.prefix else ""
1986            _write_gcs_blob_with_custom_ttl(
1987                bucket_name=store.bucket,
1988                blob_path=f"{path_prefix}{_REGISTRIES_PREFIX}/{registry_type}_registry.json",
1989                content=content,
1990                cache_control=_REGISTRY_INDEX_CACHE_CONTROL,
1991            )
1992            _log_progress(
1993                "  Wrote %s_registry.json (%d entries)",
1994                registry_type,
1995                entry_count,
1996            )
1997
1998    # --- Step 11: Compile composite registry JSON (superset) ---
1999    _log_progress("Step 11: Compiling composite_registry.json (superset)...")
2000    composite_json = _build_composite_registry_json(
2001        cloud_entries=entries_by_registry_type.get("cloud", []),
2002        oss_entries=entries_by_registry_type.get("oss", []),
2003    )
2004    composite_entry_count = len(composite_json["sources"]) + len(
2005        composite_json["destinations"]
2006    )
2007    result.composite_registry_entries = composite_entry_count
2008    if dry_run:
2009        _log_progress(
2010            "  [DRY RUN] Would write composite_registry.json (%d entries)",
2011            composite_entry_count,
2012        )
2013    else:
2014        composite_content = json.dumps(composite_json, indent=2, sort_keys=True) + "\n"
2015        path_prefix = f"{store.prefix}/" if store.prefix else ""
2016        _write_gcs_blob_with_custom_ttl(
2017            bucket_name=store.bucket,
2018            blob_path=f"{path_prefix}{_REGISTRIES_PREFIX}/composite_registry.json",
2019            content=composite_content,
2020            cache_control=_REGISTRY_INDEX_CACHE_CONTROL,
2021        )
2022        _log_progress(
2023            "  Wrote composite_registry.json (%d entries)",
2024            composite_entry_count,
2025        )
2026
2027    # --- Step 12: Per-connector version indexes (parallel) ---
2028    _log_progress(
2029        "Step 12: Writing per-connector version indexes (max_workers=%d)...",
2030        _COMPILE_WRITE_MAX_WORKERS,
2031    )
2032    base = f"{store.bucket_root}/{METADATA_FOLDER}/airbyte"
2033    sorted_connectors = sorted(connector_versions)
2034
2035    def _write_one_version_index(connector: str) -> None:
2036        """Build and write a single connector's versions.json."""
2037        versions = connector_versions[connector]
2038        latest_v = latest_versions.get(connector)
2039        rc_v_list = rc_versions.get(connector)
2040        index = _build_version_index(
2041            fs,
2042            store=store,
2043            connector=connector,
2044            versions=versions,
2045            yanked=yanked,
2046            latest_version=latest_v,
2047            rc_version=rc_v_list[0] if rc_v_list else None,
2048            rc_versions_all=rc_v_list,
2049        )
2050        index_path = f"{base}/{connector}/versions.json"
2051        if dry_run:
2052            _log_progress(
2053                "  [DRY RUN] Would write %s/versions.json (%d versions)",
2054                connector,
2055                len(versions),
2056            )
2057        else:
2058            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
2059            with fs.open(index_path, "w") as f:
2060                f.write(content)
2061
2062    with ThreadPoolExecutor(max_workers=_COMPILE_WRITE_MAX_WORKERS) as pool:
2063        futures = {
2064            pool.submit(_write_one_version_index, c): c for c in sorted_connectors
2065        }
2066        for i, future in enumerate(as_completed(futures), 1):
2067            connector = futures[future]
2068            try:
2069                future.result()
2070                result.version_indexes_written += 1
2071            except Exception as exc:
2072                error_msg = f"Failed to write versions.json for {connector}: {exc}"
2073                logger.error(error_msg)
2074                result.errors.append(error_msg)
2075            if i % 100 == 0:
2076                _log_progress(
2077                    "  Wrote %d / %d version indexes...", i, len(sorted_connectors)
2078                )
2079
2080    # --- Step 13: Specs secrets mask (optional) ---
2081    if with_secrets_mask:
2082        _log_progress("Step 13: Generating specs secrets mask...")
2083        # Reuse entries collected during Step 10 to avoid redundant GCS reads.
2084        secret_names = _extract_secret_property_names(all_registry_entries)
2085        sorted_names = sorted(secret_names)
2086        result.specs_secrets_mask_properties = len(sorted_names)
2087        mask_content = yaml.dump({"properties": sorted_names}, default_flow_style=False)
2088        mask_path = (
2089            f"{store.bucket_root}/{_REGISTRIES_PREFIX}/{_SPECS_SECRETS_MASK_FILENAME}"
2090        )
2091
2092        _log_progress(
2093            "  Found %d secret properties: %s",
2094            len(sorted_names),
2095            ", ".join(sorted_names),
2096        )
2097
2098        if dry_run:
2099            _log_progress(
2100                "  [DRY RUN] Would write %s",
2101                _SPECS_SECRETS_MASK_FILENAME,
2102            )
2103        else:
2104            with fs.open(mask_path, "w") as f:
2105                f.write(mask_content)
2106            _log_progress(
2107                "  Wrote %s",
2108                _SPECS_SECRETS_MASK_FILENAME,
2109            )
2110
2111    _log_progress(result.summary())
2112    return result

Compile the registry: sync latest/ dirs and write index files.

Steps:
  1. Glob for all metadata.yaml to discover (connector, version) pairs.
  2. Glob for active marker files.
  3. Compute the latest GA semver per connector.
  4. Compute active release candidates from versioned markers.
  5. Glob for version=* markers in latest/ dirs for a fast check.
  6. Delete stale latest/ dirs and recursively copy the versioned dir.
  7. Synthesize missing registry entries from pinned latest overrides.
  8. (Optional) Legacy migration: delete disabled registry entries.
  9. (Optional) Read latest connector metrics.
  10. Write global registry JSONs.
  11. Write composite registry JSON.
  12. Write per-connector versions.json.
  13. (Optional) Regenerate specs_secrets_mask.yaml.
Arguments:
  • store: Registry store (bucket + optional prefix).
  • connector_name: If provided, only resync latest/ directories for these connectors (steps 5-6). Index rebuilds (steps 9-12) always operate on the full set of connectors so that global registry files remain complete.
  • dry_run: If True, report what would be done without writing.
  • with_secrets_mask: If True, regenerate specs_secrets_mask.yaml.
  • with_legacy_migration: If set, run the named migration step. Currently supported: "v1" — delete {registry_type}.json files for connectors whose registryOverrides.{registry}.enabled is false.
  • with_metrics: If True, inject latest connector metrics from the analytics JSONL export into generated.metrics.
  • force: If True, resync all connectors' latest/ directories even if the existing version marker matches the computed latest version. This is useful when metadata content changes without a version bump.
Returns:

A CompileResult describing what was done.

def find_unpublished_connectors( repo_path: str | pathlib.Path, bucket_name: str, connector_names: list[str] | None = None) -> AuditResult:
 90def find_unpublished_connectors(
 91    repo_path: str | Path,
 92    bucket_name: str,
 93    connector_names: list[str] | None = None,
 94) -> AuditResult:
 95    """Find connectors whose local version is not published on GCS.
 96
 97    For each connector in the local checkout, reads `dockerImageTag` from
 98    `metadata.yaml` and checks whether `metadata/<docker-repo>/<version>/metadata.yaml`
 99    exists in the GCS bucket.  Connectors that are archived, disabled on all
100    registries, or have RC versions are skipped.
101
102    Args:
103        repo_path: Path to the Airbyte monorepo checkout.
104        bucket_name: GCS bucket name to check against.
105        connector_names: Optional list of connector names to check.
106            If `None`, discovers all connectors in the repo.
107
108    Returns:
109        An `AuditResult` containing unpublished connectors and metadata.
110    """
111    repo_path = Path(repo_path)
112    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
113
114    if not connectors_dir.exists():
115        raise ValueError(f"Connectors directory not found: {connectors_dir}")
116
117    # Discover connector names if not provided
118    if connector_names is None:
119        connector_names = sorted(
120            d.name
121            for d in connectors_dir.iterdir()
122            if d.is_dir() and (d / METADATA_FILE_NAME).exists()
123        )
124
125    result = AuditResult()
126
127    # Collect connectors and their versions first, then batch-check GCS
128    to_check: list[tuple[str, str]] = []  # (connector_name, version)
129
130    for name in connector_names:
131        metadata_path = connectors_dir / name / METADATA_FILE_NAME
132        metadata = _read_local_metadata(metadata_path)
133        if metadata is None:
134            result.errors.append(f"{name}: metadata.yaml not found or unreadable")
135            continue
136
137        if _is_archived(metadata):
138            result.skipped_archived.append(name)
139            continue
140
141        if _is_disabled_on_all_registries(metadata):
142            result.skipped_disabled.append(name)
143            continue
144
145        data = metadata.get("data", {})
146        version = data.get("dockerImageTag")
147        if not version:
148            result.errors.append(f"{name}: no dockerImageTag in metadata")
149            continue
150
151        if _is_rc_version(version):
152            result.skipped_rc.append(name)
153            continue
154
155        to_check.append((name, version))
156
157    if not to_check:
158        result.checked_count = 0
159        return result
160
161    # Check GCS for each connector version
162    storage_client = get_gcs_storage_client()
163    bucket = storage_client.bucket(bucket_name)
164
165    for name, version in to_check:
166        result.checked_count += 1
167        blob_path = f"{METADATA_FOLDER}/airbyte/{name}/{version}/{METADATA_FILE_NAME}"
168        blob = bucket.blob(blob_path)
169
170        try:
171            exists = blob.exists()
172        except Exception as e:
173            result.errors.append(f"{name}: GCS check failed: {e}")
174            continue
175
176        if not exists:
177            logger.info(
178                "Unpublished: %s version %s (checked %s)",
179                name,
180                version,
181                blob_path,
182            )
183            result.unpublished.append(
184                UnpublishedConnector(connector_name=name, local_version=version)
185            )
186
187    logger.info(
188        "Audit complete: %d checked, %d unpublished, %d archived-skipped, %d disabled-skipped, %d rc-skipped",
189        result.checked_count,
190        len(result.unpublished),
191        len(result.skipped_archived),
192        len(result.skipped_disabled),
193        len(result.skipped_rc),
194    )
195
196    return result

Find connectors whose local version is not published on GCS.

For each connector in the local checkout, reads dockerImageTag from metadata.yaml and checks whether metadata/<docker-repo>/<version>/metadata.yaml exists in the GCS bucket. Connectors that are archived, disabled on all registries, or have RC versions are skipped.

Arguments:
  • repo_path: Path to the Airbyte monorepo checkout.
  • bucket_name: GCS bucket name to check against.
  • connector_names: Optional list of connector names to check. If None, discovers all connectors in the repo.
Returns:

An AuditResult containing unpublished connectors and metadata.

def generate_version_artifacts( metadata_file: pathlib.Path, docker_image: str, output_dir: pathlib.Path | None = None, repo_root: pathlib.Path | None = None, dry_run: bool = False, with_validate: bool = True, with_dependency_dump: bool = True, with_sbom: bool = True) -> GenerateResult:
646def generate_version_artifacts(
647    metadata_file: Path,
648    docker_image: str,
649    output_dir: Path | None = None,
650    repo_root: Path | None = None,
651    dry_run: bool = False,
652    with_validate: bool = True,
653    with_dependency_dump: bool = True,
654    with_sbom: bool = True,
655) -> GenerateResult:
656    """Generate all version artifacts for a connector release.
657
658    Artifacts are enriched with git commit info, SBOM URL, and (when applicable)
659    components SHA before writing.  Validation is run after generation by default.
660
661    Args:
662        metadata_file: Path to the connector's `metadata.yaml`.
663        docker_image: Docker image to run spec against (e.g. `airbyte/source-faker:6.2.38`).
664        output_dir: Directory to write artifacts to.  If `None`, a temp directory is created.
665        repo_root: Root of the Airbyte repo checkout (for resolving `doc.md`).
666            If `None`, inferred by walking up from `metadata_file`.
667        dry_run: If `True`, report what would be generated without writing or running docker.
668        with_validate: If `True` (default), run metadata validators after generation.
669            Pass `False` (`--no-validate`) to skip.
670        with_dependency_dump: If `True` (default), generate `dependencies.json`
671            for Python connectors.  Pass `False` (`--no-dependency-dump`) to skip.
672        with_sbom: If `True` (default), generate `spdx.json` (SBOM) for
673            connectors.  Pass `False` (`--no-sbom`) to skip.
674
675    Returns:
676        A `GenerateResult` describing what was produced.
677    """
678    # --- Load metadata ---
679    if not metadata_file.exists():
680        raise FileNotFoundError(f"Metadata file not found: {metadata_file}")
681
682    raw_metadata: dict[str, Any] = yaml.safe_load(metadata_file.read_text())
683    metadata_data: dict[str, Any] = raw_metadata.get("data", {})
684
685    connector_name = metadata_data.get("dockerRepository", "unknown").replace(
686        "airbyte/", ""
687    )
688    version = metadata_data.get("dockerImageTag", "unknown")
689
690    # --- Resolve output directory ---
691    if output_dir is None:
692        output_dir = Path(
693            tempfile.mkdtemp(prefix=f"connector-artifacts-{connector_name}-{version}-")
694        )
695    output_dir.mkdir(parents=True, exist_ok=True)
696
697    result = GenerateResult(
698        connector_name=connector_name,
699        version=version,
700        docker_image=docker_image,
701        output_dir=str(output_dir),
702        dry_run=dry_run,
703    )
704
705    if dry_run:
706        logger.info("[DRY RUN] Would generate artifacts to %s", output_dir)
707        result.artifacts_written = [
708            "metadata.yaml",
709            "icon.svg",
710            "doc.md",
711            "cloud.json",
712            "oss.json",
713            "manifest.yaml (if present)",
714            "components.zip (if components.py present)",
715            "components.zip.sha256 (if components.py present)",
716            f"version={version}",
717        ]
718        if with_sbom:
719            result.artifacts_written.append(SBOM_FILE_NAME)
720        if with_dependency_dump:
721            result.artifacts_written.append("dependencies.json (if Python connector)")
722        return result
723
724    # --- Prepare metadata output ---
725    metadata_out = output_dir / "metadata.yaml"
726    result.artifacts_written.append("metadata.yaml")
727
728    # --- Enrich metadata with git info *before* building registry entries so
729    #     that `generated.git` propagates into `cloud.json` / `oss.json`. ---
730    raw_metadata = _enrich_metadata_git_info(raw_metadata, metadata_file)
731
732    # --- Generate SBOM from the connector Docker image ---
733    sbom_generated = False
734    if not with_sbom:
735        logger.info("SBOM generation disabled via --no-sbom.")
736    else:
737        try:
738            sbom_path = generate_sbom(docker_image, output_dir)
739        except RuntimeError as exc:
740            logger.warning("SBOM generation failed (non-fatal): %s", exc)
741        except (FileNotFoundError, subprocess.TimeoutExpired):
742            logger.warning("Docker not available or SBOM generation timed out.")
743        else:
744            result.artifacts_written.append(SBOM_FILE_NAME)
745            sbom_generated = True
746            logger.info("Generated SBOM: %s", sbom_path)
747
748    # --- Enrich metadata with SBOM URL ---
749    raw_metadata = _enrich_metadata_sbom_url(
750        raw_metadata, sbom_generated=sbom_generated
751    )
752
753    # --- Run docker spec for cloud and oss ---
754    specs: dict[str, dict[str, Any]] = {}
755    for mode in VALID_REGISTRIES:
756        try:
757            specs[mode] = _run_docker_spec(docker_image, mode)
758            logger.info("Got %s spec from docker image %s", mode, docker_image)
759        except RuntimeError as exc:
760            error_msg = f"Failed to get {mode} spec: {exc}"
761            logger.error(error_msg)
762            result.errors.append(error_msg)
763
764    # --- Generate dependencies.json for Python connectors ---
765    # This must happen *before* building registry entries so that the
766    # local dependencies data can be used for packageInfo without a GCS
767    # round-trip.
768    local_dependencies: dict[str, Any] | None = None
769    if not with_dependency_dump:
770        logger.info("Dependency generation disabled via --no-dependency-dump.")
771    elif _is_python_connector(metadata_data):
772        logger.info("Python connector detected — generating dependencies.json")
773        local_dependencies = generate_python_dependencies_file(
774            metadata_data=metadata_data,
775            docker_image=docker_image,
776            output_dir=output_dir,
777        )
778        if local_dependencies is not None:
779            result.artifacts_written.append(CONNECTOR_DEPENDENCY_FILE_NAME)
780    else:
781        logger.info(
782            "Non-Python connector (%s) — skipping dependencies.json generation.",
783            connector_name,
784        )
785
786    # --- Generate registry entries (cloud.json, oss.json) ---
787    for registry_type in VALID_REGISTRIES:
788        if not is_registry_enabled(metadata_data, registry_type):
789            logger.info(
790                "Registry type %s is not enabled for %s, skipping %s.json generation.",
791                registry_type,
792                connector_name,
793                registry_type,
794            )
795            continue
796
797        spec = specs.get(registry_type)
798        if spec is None:
799            error_msg = (
800                f"Cannot generate {registry_type}.json: no spec available "
801                f"(docker spec for {registry_type} failed or was not run)."
802            )
803            result.errors.append(error_msg)
804            continue
805
806        registry_entry = _build_registry_entry(
807            metadata_data,
808            registry_type,
809            spec,
810            local_dependencies=local_dependencies,
811        )
812
813        out_path = output_dir / f"{registry_type}.json"
814        out_path.write_text(
815            json.dumps(registry_entry, indent=2, sort_keys=True, default=_json_serial)
816            + "\n"
817        )
818        result.artifacts_written.append(f"{registry_type}.json")
819        logger.info("Wrote %s", out_path)
820
821    # --- Copy icon.svg (sibling of metadata.yaml in the connector directory) ---
822    icon_source = metadata_file.parent / "icon.svg"
823    if icon_source.is_file():
824        icon_out = output_dir / "icon.svg"
825        shutil.copy2(icon_source, icon_out)
826        result.artifacts_written.append("icon.svg")
827        logger.info("Wrote %s", icon_out)
828    else:
829        logger.warning("No icon.svg found at %s.", icon_source)
830        result.errors.append("Icon file is missing.")
831
832    # --- Copy doc.md (derived from documentationUrl in metadata) ---
833    if repo_root is None:
834        # Infer repo root by walking up from metadata_file looking for .git
835        # Note: .git can be a directory (normal clone) or a file (git worktree)
836        # Resolve to absolute path first so the walk-up works with relative paths.
837        candidate = metadata_file.resolve().parent
838        while candidate != candidate.parent:
839            git_indicator = candidate / ".git"
840            if git_indicator.is_dir() or git_indicator.is_file():
841                repo_root = candidate
842                break
843            candidate = candidate.parent
844
845    if repo_root is not None:
846        doc_source = _resolve_doc_path(metadata_data, repo_root)
847        if doc_source is not None and doc_source.is_file():
848            doc_out = output_dir / DOC_FILE_NAME
849            shutil.copy2(doc_source, doc_out)
850            result.artifacts_written.append(DOC_FILE_NAME)
851            logger.info("Wrote %s (from %s)", doc_out, doc_source)
852        else:
853            error_msg = (
854                f"Documentation file not found: {doc_source}. "
855                f"Derived from documentationUrl in metadata."
856            )
857            logger.error(error_msg)
858            result.errors.append(error_msg)
859    else:
860        error_msg = "Cannot resolve doc.md: repo root not found."
861        logger.error(error_msg)
862        result.errors.append(error_msg)
863
864    # --- Copy manifest.yaml (from connector root, if present) ---
865    connector_dir = metadata_file.parent
866    manifest_source = connector_dir / MANIFEST_FILE_NAME
867    components_sha256: str | None = None
868    if manifest_source.is_file():
869        manifest_out = output_dir / MANIFEST_FILE_NAME
870        shutil.copy2(manifest_source, manifest_out)
871        result.artifacts_written.append(MANIFEST_FILE_NAME)
872        logger.info("Wrote %s", manifest_out)
873
874        # --- Generate components.zip if components.py exists ---
875        components_source = connector_dir / COMPONENTS_PY_FILE_NAME
876        if components_source.is_file():
877            zip_path, sha256_path = _create_components_zip(
878                manifest_path=manifest_source,
879                components_path=components_source,
880                output_dir=output_dir,
881            )
882            result.artifacts_written.append(COMPONENTS_ZIP_FILE_NAME)
883            result.artifacts_written.append(COMPONENTS_ZIP_SHA256_FILE_NAME)
884            logger.info("Wrote %s and %s", zip_path, sha256_path)
885            # Read back the SHA256 for metadata enrichment
886            components_sha256 = sha256_path.read_text().strip()
887    else:
888        logger.info(
889            "No manifest.yaml at %s — skipping manifest artifacts.", manifest_source
890        )
891
892    # --- Enrich metadata with components SHA (after zip creation) ---
893    raw_metadata = _enrich_metadata_components_sha(raw_metadata, components_sha256)
894
895    # --- Write final enriched metadata.yaml ---
896    # Use sort_keys=True to match the legacy pipeline's alphabetical key ordering.
897    # After Registry 2.0 launches we are free to change the key ordering.
898    metadata_out.write_text(
899        yaml.dump(raw_metadata, default_flow_style=False, sort_keys=True)
900    )
901    logger.info("Wrote enriched %s", metadata_out)
902
903    # --- Write version marker file (version=<semver>) ---
904    # This zero-byte file is used by the compile step as a fast-check marker.
905    # Including it in the generated artifacts means `latest/` gets the marker
906    # for free via a recursive copy, removing the need for a separate write.
907    marker_file = output_dir / f"version={version}"
908    marker_file.write_bytes(b"")
909    result.artifacts_written.append(f"version={version}")
910    logger.info("Wrote version marker %s", marker_file)
911
912    # --- Validate metadata (after generation) ---
913    if with_validate:
914        logger.info("Running post-generation validation...")
915        doc_path: str | None = None
916        if repo_root is not None:
917            resolved = _resolve_doc_path(metadata_data, repo_root)
918            doc_path = str(resolved) if resolved else None
919        validation = validate_metadata(
920            metadata_data=metadata_data,
921            opts=ValidateOptions(docs_path=doc_path),
922        )
923        if not validation.passed:
924            for err in validation.errors:
925                logger.error("Validation error: %s", err)
926            result.validation_errors = validation.errors
927        else:
928            logger.info("Validation passed (%d validators).", validation.validators_run)
929
930    return result

Generate all version artifacts for a connector release.

Artifacts are enriched with git commit info, SBOM URL, and (when applicable) components SHA before writing. Validation is run after generation by default.

Arguments:
  • metadata_file: Path to the connector's metadata.yaml.
  • docker_image: Docker image to run spec against (e.g. airbyte/source-faker:6.2.38).
  • output_dir: Directory to write artifacts to. If None, a temp directory is created.
  • repo_root: Root of the Airbyte repo checkout (for resolving doc.md). If None, inferred by walking up from metadata_file.
  • dry_run: If True, report what would be generated without writing or running docker.
  • with_validate: If True (default), run metadata validators after generation. Pass False (--no-validate) to skip.
  • with_dependency_dump: If True (default), generate dependencies.json for Python connectors. Pass False (--no-dependency-dump) to skip.
  • with_sbom: If True (default), generate spdx.json (SBOM) for connectors. Pass False (--no-sbom) to skip.
Returns:

A GenerateResult describing what was produced.

def get_connector_metadata( repo_path: pathlib.Path, connector_name: str) -> ConnectorMetadata:
37def get_connector_metadata(repo_path: Path, connector_name: str) -> ConnectorMetadata:
38    """Read connector metadata from metadata.yaml.
39
40    Args:
41        repo_path: Path to the Airbyte monorepo.
42        connector_name: The connector technical name (e.g., 'source-github').
43
44    Returns:
45        ConnectorMetadata object with the connector's metadata.
46
47    Raises:
48        FileNotFoundError: If the connector directory or metadata file doesn't exist.
49    """
50    connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name
51    if not connector_dir.exists():
52        raise FileNotFoundError(f"Connector directory not found: {connector_dir}")
53
54    metadata_file = connector_dir / METADATA_FILE_NAME
55    if not metadata_file.exists():
56        raise FileNotFoundError(f"Metadata file not found: {metadata_file}")
57
58    with open(metadata_file) as f:
59        metadata = yaml.safe_load(f)
60
61    data = metadata.get("data", {})
62    return ConnectorMetadata(
63        name=connector_name,
64        docker_repository=data.get("dockerRepository", f"airbyte/{connector_name}"),
65        docker_image_tag=data.get("dockerImageTag", "unknown"),
66        support_level=data.get("supportLevel"),
67        definition_id=data.get("definitionId"),
68    )

Read connector metadata from metadata.yaml.

Arguments:
  • repo_path: Path to the Airbyte monorepo.
  • connector_name: The connector technical name (e.g., 'source-github').
Returns:

ConnectorMetadata object with the connector's metadata.

Raises:
  • FileNotFoundError: If the connector directory or metadata file doesn't exist.
def get_gcs_publish_path(connector_name: str, artifact_type: str, version: str = 'latest') -> str:
71def get_gcs_publish_path(
72    connector_name: str,
73    artifact_type: str,
74    version: str = LATEST_GCS_FOLDER_NAME,
75) -> str:
76    """Compute the GCS path for a connector artifact for publishing.
77
78    All connectors use the airbyte/{connector_name} convention.
79    """
80    artifact_files = {
81        "metadata": METADATA_FILE_NAME,
82        "spec": "spec.json",
83        "icon": "icon.svg",
84        "doc": "doc.md",
85    }
86
87    if artifact_type not in artifact_files:
88        raise ValueError(
89            f"Unknown artifact type: {artifact_type}. "
90            f"Valid types are: {', '.join(artifact_files.keys())}"
91        )
92
93    file_name = artifact_files[artifact_type]
94    return f"{METADATA_FOLDER}/airbyte/{connector_name}/{version}/{file_name}"

Compute the GCS path for a connector artifact for publishing.

All connectors use the airbyte/{connector_name} convention.

def get_registry( store: RegistryStore) -> Registry:
222def get_registry(store: RegistryStore) -> Registry:
223    """Factory for obtaining the right store implementation."""
224
225    if store.store_type == StoreType.CORAL:
226        from airbyte_ops_mcp.registry.coral_registry_store import CoralRegistry
227
228        return CoralRegistry(store)
229
230    if store.store_type == StoreType.SONAR:
231        from airbyte_ops_mcp.registry.sonar_registry_store import SonarRegistry
232
233        return SonarRegistry(store)
234
235    # defensive: StoreType is an Enum, but keep this for readability
236    raise ValueError(f"Unknown store type: {store.store_type}")

Factory for obtaining the right store implementation.

def get_registry_entry( connector_name: str, bucket_name: str, version: str = 'latest', prefix: str = '') -> dict[str, typing.Any]:
 43def get_registry_entry(
 44    connector_name: str,
 45    bucket_name: str,
 46    version: str = LATEST_GCS_FOLDER_NAME,
 47    prefix: str = "",
 48) -> dict[str, Any]:
 49    """Get a connector's registry entry from GCS.
 50
 51    Reads metadata for a connector from the registry stored in GCS.
 52
 53    Args:
 54        connector_name: The connector name (e.g., "source-faker", "destination-postgres")
 55        bucket_name: Name of the GCS bucket containing the registry
 56        version: Version folder name (e.g., "latest", "1.2.3")
 57        prefix: Optional path prefix within the bucket; leading and trailing
 58            slashes are ignored.
 59
 60    Returns:
 61        dict: The connector's metadata as a dictionary
 62
 63    Raises:
 64        ValueError: If GCS credentials are not configured, or if the metadata has an invalid structure
 65        FileNotFoundError: If the connector metadata is not found in the registry
 66        yaml.YAMLError: If the metadata file contains invalid YAML syntax
 67    """
 68    storage_client = get_gcs_storage_client()
 69    bucket = storage_client.bucket(bucket_name)
 70
 71    # Construct the path to the metadata file
 72    # Pattern: metadata/airbyte/{connector_name}/{version}/metadata.yaml
 73    normalized_prefix = prefix.strip("/")
 74    prefix_part = f"{normalized_prefix}/" if normalized_prefix else ""
 75    blob_path = (
 76        f"{prefix_part}{METADATA_FOLDER}/airbyte/"
 77        f"{connector_name}/{version}/{METADATA_FILE_NAME}"
 78    )
 79    blob = bucket.blob(blob_path)
 80
 81    logger.info(f"Reading registry entry for {connector_name} from {blob_path}")
 82
 83    # Read the file
 84    content = safe_read_gcs_file(blob)
 85    if content is None:
 86        raise FileNotFoundError(
 87            f"Connector metadata not found in registry: {connector_name}. "
 88            f"Checked path: {blob_path}"
 89        )
 90
 91    # Parse YAML
 92    try:
 93        metadata = yaml.safe_load(content)
 94        if metadata is None or not isinstance(metadata, dict):
 95            raise ValueError(f"Metadata file {blob_path} has an invalid structure")
 96        return metadata
 97    except yaml.YAMLError as e:
 98        logger.error(
 99            "Failed to parse metadata for %s from %s: %s",
100            connector_name,
101            blob_path,
102            e,
103        )
104        raise

Get a connector's registry entry from GCS.

Reads metadata for a connector from the registry stored in GCS.

Arguments:
  • connector_name: The connector name (e.g., "source-faker", "destination-postgres")
  • bucket_name: Name of the GCS bucket containing the registry
  • version: Version folder name (e.g., "latest", "1.2.3")
  • prefix: Optional path prefix within the bucket; leading and trailing slashes are ignored.
Returns:

dict: The connector's metadata as a dictionary

Raises:
  • ValueError: If GCS credentials are not configured, or if the metadata has an invalid structure
  • FileNotFoundError: If the connector metadata is not found in the registry
  • yaml.YAMLError: If the metadata file contains invalid YAML syntax
def get_registry_spec( connector_name: str, bucket_name: str, version: str = 'latest') -> dict[str, typing.Any]:
107def get_registry_spec(
108    connector_name: str,
109    bucket_name: str,
110    version: str = LATEST_GCS_FOLDER_NAME,
111) -> dict[str, Any]:
112    """Get a connector's spec from GCS.
113
114    Reads the connector specification from the registry stored in GCS.
115
116    Args:
117        connector_name: The connector name (e.g., "source-faker", "destination-postgres")
118        bucket_name: Name of the GCS bucket containing the registry
119        version: Version folder name (e.g., "latest", "1.2.3")
120
121    Returns:
122        dict: The connector's spec as a dictionary
123
124    Raises:
125        ValueError: If GCS credentials are not configured, or if the spec is not a JSON object
126        FileNotFoundError: If the connector spec is not found in the registry
127        json.JSONDecodeError: If the spec file contains invalid JSON syntax
128    """
129    storage_client = get_gcs_storage_client()
130    bucket = storage_client.bucket(bucket_name)
131
132    # Construct the path to the spec file
133    # Pattern: metadata/airbyte/{connector_name}/{version}/spec.json
134    blob_path = f"{METADATA_FOLDER}/airbyte/{connector_name}/{version}/{SPEC_FILE_NAME}"
135    blob = bucket.blob(blob_path)
136
137    logger.info(f"Reading spec for {connector_name} from {blob_path}")
138
139    # Read the file
140    content = safe_read_gcs_file(blob)
141    if content is None:
142        raise FileNotFoundError(
143            f"Connector spec not found in registry: {connector_name}. "
144            f"Checked path: {blob_path}"
145        )
146
147    # Parse JSON
148    try:
149        spec = json.loads(content)
150        if spec is None or not isinstance(spec, dict):
151            raise ValueError(
152                f"Spec file for {connector_name} at {blob_path} is not a JSON object"
153            )
154        return spec
155    except json.JSONDecodeError as e:
156        logger.error(
157            "Failed to parse spec for %s from %s: %s",
158            connector_name,
159            blob_path,
160            e,
161        )
162        raise

Get a connector's spec from GCS.

Reads the connector specification from the registry stored in GCS.

Arguments:
  • connector_name: The connector name (e.g., "source-faker", "destination-postgres")
  • bucket_name: Name of the GCS bucket containing the registry
  • version: Version folder name (e.g., "latest", "1.2.3")
Returns:

dict: The connector's spec as a dictionary

Raises:
  • ValueError: If GCS credentials are not configured, or if the spec is not a JSON object
  • FileNotFoundError: If the connector spec is not found in the registry
  • json.JSONDecodeError: If the spec file contains invalid JSON syntax
def list_connector_versions(connector_name: str, bucket_name: str) -> list[str]:
331def list_connector_versions(connector_name: str, bucket_name: str) -> list[str]:
332    """List all versions of a connector in the registry.
333
334    Scans the GCS bucket to find all versions of a specific connector.
335
336    Args:
337        connector_name: The connector name (e.g., "source-faker")
338        bucket_name: Name of the GCS bucket containing the registry
339
340    Returns:
341        list[str]: Sorted list of version strings (excluding 'latest' and 'release_candidate')
342
343    Raises:
344        ValueError: If GCS credentials are not configured
345    """
346    storage_client = get_gcs_storage_client()
347    bucket = storage_client.bucket(bucket_name)
348
349    # List all blobs matching the pattern: metadata/airbyte/{connector_name}/*/metadata.yaml
350    glob_pattern = f"{METADATA_FOLDER}/airbyte/{connector_name}/*/{METADATA_FILE_NAME}"
351    logger.info(f"Listing versions for {connector_name} with pattern: {glob_pattern}")
352
353    try:
354        blobs = bucket.list_blobs(match_glob=glob_pattern)
355    except Exception as e:
356        logger.error(f"Error listing blobs in bucket {bucket_name}: {e}")
357        raise
358
359    # Extract versions from blob paths
360    # Path format: metadata/airbyte/{connector-name}/{version}/metadata.yaml
361    versions: set[str] = set()
362    for blob in blobs:
363        path_parts = blob.name.split("/")
364        # Path should be: metadata / airbyte / connector-name / version / metadata.yaml
365        if len(path_parts) >= 5:
366            version = path_parts[3]
367            # Exclude special folders
368            if version not in ("latest", "release_candidate"):
369                versions.add(version)
370
371    return sorted(versions)

List all versions of a connector in the registry.

Scans the GCS bucket to find all versions of a specific connector.

Arguments:
  • connector_name: The connector name (e.g., "source-faker")
  • bucket_name: Name of the GCS bucket containing the registry
Returns:

list[str]: Sorted list of version strings (excluding 'latest' and 'release_candidate')

Raises:
  • ValueError: If GCS credentials are not configured
def list_registry_connectors(bucket_name: str) -> list[str]:
165def list_registry_connectors(bucket_name: str) -> list[str]:
166    """List all connectors in the registry.
167
168    Scans the GCS bucket to find all connectors that have metadata files.
169
170    Args:
171        bucket_name: Name of the GCS bucket containing the registry
172
173    Returns:
174        list[str]: Sorted list of connector names
175
176    Raises:
177        ValueError: If GCS credentials are not configured
178    """
179    storage_client = get_gcs_storage_client()
180    bucket = storage_client.bucket(bucket_name)
181
182    # List all blobs matching the pattern: metadata/airbyte/*/latest/metadata.yaml
183    glob_pattern = (
184        f"{METADATA_FOLDER}/airbyte/*/{LATEST_GCS_FOLDER_NAME}/{METADATA_FILE_NAME}"
185    )
186    logger.info(f"Listing connectors with pattern: {glob_pattern}")
187
188    try:
189        blobs = bucket.list_blobs(match_glob=glob_pattern)
190    except Exception as e:
191        logger.error(f"Error listing blobs in bucket {bucket_name}: {e}")
192        raise
193
194    # Extract connector names from blob paths
195    # Path format: metadata/airbyte/{connector-name}/latest/metadata.yaml
196    connector_names: set[str] = set()
197    for blob in blobs:
198        path_parts = blob.name.split("/")
199        # Path should be: metadata / airbyte / connector-name / latest / metadata.yaml
200        if len(path_parts) >= 5:
201            connector_name = path_parts[2]
202            connector_names.add(connector_name)
203
204    return sorted(connector_names)

List all connectors in the registry.

Scans the GCS bucket to find all connectors that have metadata files.

Arguments:
  • bucket_name: Name of the GCS bucket containing the registry
Returns:

list[str]: Sorted list of connector names

Raises:
  • ValueError: If GCS credentials are not configured
def list_registry_connectors_filtered( bucket_name: str, *, support_level: SupportLevel | None = None, min_support_level: SupportLevel | None = None, connector_type: ConnectorType | None = None, language: ConnectorLanguage | None = None, prefix: str = '') -> list[str]:
207def list_registry_connectors_filtered(
208    bucket_name: str,
209    *,
210    support_level: SupportLevel | None = None,
211    min_support_level: SupportLevel | None = None,
212    connector_type: ConnectorType | None = None,
213    language: ConnectorLanguage | None = None,
214    prefix: str = "",
215) -> list[str]:
216    """List connectors from the compiled cloud registry index with filtering.
217
218    When any filter is applied, reads the compiled `cloud_registry.json` index
219    instead of globbing individual metadata blobs. This is significantly faster
220    because the index is a single JSON file containing all connector entries.
221
222    When no filters are applied, falls back to the existing glob-based search
223    which captures all connectors (including OSS-only connectors not in the
224    Cloud index).
225
226    Args:
227        bucket_name: Name of the GCS bucket containing the registry.
228        support_level: Exact support level to match (e.g., `SupportLevel.CERTIFIED`).
229        min_support_level: Minimum support level threshold. Returns connectors
230            at or above this level.
231        connector_type: Filter by connector type (`ConnectorType.SOURCE` or
232            `ConnectorType.DESTINATION`).
233        language: Filter by implementation language (e.g., `ConnectorLanguage.PYTHON`).
234        prefix: Optional bucket prefix (e.g., `"aj-test100"`).
235
236    Returns:
237        Sorted list of connector technical names (e.g., `"source-github"`).
238
239    Raises:
240        ValueError: If `support_level` and `min_support_level` are both provided.
241    """
242    has_filters = any([support_level, min_support_level, connector_type, language])
243
244    if not has_filters:
245        return list_registry_connectors(bucket_name=bucket_name)
246
247    if support_level and min_support_level:
248        raise ValueError(
249            "Cannot specify both `support_level` and `min_support_level`. "
250            "Use `support_level` for an exact match or `min_support_level` for a threshold."
251        )
252
253    entries = _read_cloud_registry_index(bucket_name=bucket_name, prefix=prefix)
254
255    # Apply support_level exact match
256    if support_level:
257        entries = [e for e in entries if e.get("supportLevel") == support_level]
258
259    # Apply min_support_level threshold
260    if min_support_level:
261        threshold = min_support_level.precedence
262        known_levels = {m.value for m in SupportLevel}
263        entries = [
264            e
265            for e in entries
266            if e.get("supportLevel")
267            and e["supportLevel"] in known_levels
268            and SupportLevel(e["supportLevel"]).precedence >= threshold
269        ]
270
271    # Apply connector_type filter
272    if connector_type == ConnectorType.SOURCE:
273        entries = [e for e in entries if "sourceDefinitionId" in e]
274    elif connector_type == ConnectorType.DESTINATION:
275        entries = [e for e in entries if "destinationDefinitionId" in e]
276
277    # Apply language filter
278    if language:
279        entries = [e for e in entries if e.get("language") == language]
280
281    # Extract connector names from dockerRepository (e.g., "airbyte/source-github" -> "source-github")
282    names: set[str] = set()
283    for entry in entries:
284        docker_repo = entry.get("dockerRepository", "")
285        if "/" in docker_repo:
286            names.add(docker_repo.split("/", 1)[1])
287        elif docker_repo:
288            names.add(docker_repo)
289
290    return sorted(names)

List connectors from the compiled cloud registry index with filtering.

When any filter is applied, reads the compiled cloud_registry.json index instead of globbing individual metadata blobs. This is significantly faster because the index is a single JSON file containing all connector entries.

When no filters are applied, falls back to the existing glob-based search which captures all connectors (including OSS-only connectors not in the Cloud index).

Arguments:
  • bucket_name: Name of the GCS bucket containing the registry.
  • support_level: Exact support level to match (e.g., SupportLevel.CERTIFIED).
  • min_support_level: Minimum support level threshold. Returns connectors at or above this level.
  • connector_type: Filter by connector type (ConnectorType.SOURCE or ConnectorType.DESTINATION).
  • language: Filter by implementation language (e.g., ConnectorLanguage.PYTHON).
  • prefix: Optional bucket prefix (e.g., "aj-test100").
Returns:

Sorted list of connector technical names (e.g., "source-github").

Raises:
  • ValueError: If support_level and min_support_level are both provided.
def publish_connector_metadata( connector_name: str, metadata: dict[str, typing.Any], bucket_name: str, version: str, update_latest: bool = True, dry_run: bool = False) -> MetadataPublishResult:
 97def publish_connector_metadata(
 98    connector_name: str,
 99    metadata: dict[str, Any],
100    bucket_name: str,
101    version: str,
102    update_latest: bool = True,
103    dry_run: bool = False,
104) -> MetadataPublishResult:
105    """Publish connector metadata to GCS.
106
107    Uploads the metadata to the registry bucket at a versioned path, and optionally
108    also updates the 'latest' pointer. Uses MD5 hash comparison to avoid re-uploading
109    unchanged files.
110
111    Requires GCS_CREDENTIALS environment variable to be set.
112    """
113    if not isinstance(metadata, dict):
114        raise ValueError("Metadata must be a dictionary")
115
116    if "data" not in metadata:
117        raise ValueError("Metadata must contain 'data' field")
118
119    # Construct GCS paths using airbyte/{connector_name} convention
120    versioned_blob_path = get_gcs_publish_path(connector_name, "metadata", version)
121    latest_blob_path = get_gcs_publish_path(
122        connector_name, "metadata", LATEST_GCS_FOLDER_NAME
123    )
124
125    if dry_run:
126        message = f"[DRY RUN] Would upload metadata to gs://{bucket_name}/{versioned_blob_path}"
127        if update_latest:
128            message += f" and gs://{bucket_name}/{latest_blob_path}"
129        logger.info(message)
130        return MetadataPublishResult(
131            connector_name=connector_name,
132            version=version,
133            bucket_name=bucket_name,
134            versioned_path=versioned_blob_path,
135            latest_path=latest_blob_path if update_latest else None,
136            versioned_uploaded=False,
137            latest_uploaded=False,
138            status="dry-run",
139            message=message,
140        )
141
142    # Get GCS client and bucket
143    storage_client = get_gcs_storage_client()
144    bucket = storage_client.bucket(bucket_name)
145
146    # Write metadata to temp file
147    with tempfile.NamedTemporaryFile(
148        mode="w", suffix=".yaml", delete=False
149    ) as tmp_file:
150        yaml.dump(metadata, tmp_file)
151        tmp_path = Path(tmp_file.name)
152
153    try:
154        # Upload versioned file
155        versioned_uploaded, _ = upload_file_if_changed(
156            local_file_path=tmp_path,
157            bucket=bucket,
158            blob_path=versioned_blob_path,
159            disable_cache=True,
160        )
161
162        if versioned_uploaded:
163            logger.info(
164                f"Uploaded metadata for {connector_name} v{version} to {versioned_blob_path}"
165            )
166        else:
167            logger.info(
168                f"Versioned metadata for {connector_name} v{version} is already up to date"
169            )
170
171        # Optionally update latest pointer
172        latest_uploaded = False
173        if update_latest:
174            latest_uploaded, _ = upload_file_if_changed(
175                local_file_path=tmp_path,
176                bucket=bucket,
177                blob_path=latest_blob_path,
178                disable_cache=True,
179            )
180            if latest_uploaded:
181                logger.info(f"Updated latest pointer for {connector_name}")
182            else:
183                logger.info(
184                    f"Latest pointer for {connector_name} is already up to date"
185                )
186    finally:
187        # Clean up temp file even if upload fails
188        tmp_path.unlink(missing_ok=True)
189
190    # Determine status
191    if versioned_uploaded or latest_uploaded:
192        status = "success"
193        message = f"Published metadata for {connector_name} v{version}"
194        if versioned_uploaded:
195            message += f" to {versioned_blob_path}"
196        if latest_uploaded:
197            message += " and updated latest"
198    else:
199        status = "already-up-to-date"
200        message = f"Metadata for {connector_name} v{version} is already up to date"
201
202    return MetadataPublishResult(
203        connector_name=connector_name,
204        version=version,
205        bucket_name=bucket_name,
206        versioned_path=versioned_blob_path,
207        latest_path=latest_blob_path if update_latest else None,
208        versioned_uploaded=versioned_uploaded,
209        latest_uploaded=latest_uploaded,
210        status=status,
211        message=message,
212    )

Publish connector metadata to GCS.

Uploads the metadata to the registry bucket at a versioned path, and optionally also updates the 'latest' pointer. Uses MD5 hash comparison to avoid re-uploading unchanged files.

Requires GCS_CREDENTIALS environment variable to be set.

def publish_version_artifacts( connector_name: str, version: str, artifacts_dir: pathlib.Path, store: RegistryStore, dry_run: bool = False, with_validate: bool = True) -> PublishArtifactsResult:
212def publish_version_artifacts(
213    connector_name: str,
214    version: str,
215    artifacts_dir: Path,
216    store: RegistryStore,
217    dry_run: bool = False,
218    with_validate: bool = True,
219) -> PublishArtifactsResult:
220    """Publish locally generated artifacts to a GCS registry bucket.
221
222    Uses `gcsfs.GCSFileSystem` to upload the local *artifacts_dir* to the
223    versioned path inside the target GCS bucket.
224
225    The target GCS path is:
226        `gs://<bucket>/[<prefix>/]metadata/airbyte/<connector>/<version>/`
227
228    Before uploading, this function validates that the `connector_name`
229    (derived from the connector directory) matches the `dockerRepository`
230    declared in `metadata.yaml`.  A mismatch would cause the registry
231    compile step to see duplicate definition-ID entries and fail.
232
233    Args:
234        connector_name: Connector name (e.g. `source-faker`).
235        version: Version string (e.g. `6.2.38`).
236        artifacts_dir: Local directory containing artifacts from `generate`.
237        store: Parsed store target containing bucket, prefix, and stage info.
238        dry_run: If `True`, report what would be uploaded without writing.
239        with_validate: If `True` (default), validate metadata before uploading.
240            Pass `False` (`--no-validate`) to skip.
241
242    Returns:
243        A `PublishArtifactsResult` describing what was published.
244
245    Raises:
246        ValueError: If the connector directory name does not match
247            `dockerRepository` in the generated metadata.
248    """
249    if not artifacts_dir.is_dir():
250        raise FileNotFoundError(f"Artifacts directory not found: {artifacts_dir}")
251
252    # Fail fast if the connector directory name doesn't match dockerRepository.
253    # A mismatch would publish artifacts under the wrong GCS path and corrupt
254    # the registry (duplicate definition-IDs under different directory names).
255    mismatch_error = _check_connector_name_matches_docker_repo(
256        connector_name, artifacts_dir
257    )
258    if mismatch_error:
259        raise ValueError(mismatch_error)
260
261    # Build the GCS destination path
262    bucket_name = store.bucket
263    prefix = store.prefix
264    blob_root = versioned_blob_root(
265        connector_name=connector_name, version=version, store=store
266    )
267    versioned_dest = f"gcs://{bucket_name}/{blob_root}"
268
269    target_label = f"{bucket_name}/{prefix}" if prefix else bucket_name
270    progressive_rollout_enabled = _metadata_enables_progressive_rollout(artifacts_dir)
271    rollout_overridden_by_breaking_change = (
272        progressive_rollout_enabled
273        and _metadata_declares_breaking_change(artifacts_dir, version)
274    )
275    published_latest_version = (
276        _published_latest_version(connector_name, store)
277        if progressive_rollout_enabled and not rollout_overridden_by_breaking_change
278        else None
279    )
280    rollout_overridden_by_published_ga = (
281        published_latest_version == version
282        if published_latest_version is not None
283        else False
284    )
285    if rollout_overridden_by_breaking_change:
286        logger.info(
287            "Ignoring progressive rollout settings for %s@%s; version is declared "
288            "as a breaking change. The rollout marker will not be published.",
289            connector_name,
290            version,
291        )
292    elif rollout_overridden_by_published_ga:
293        logger.info(
294            "Ignoring progressive rollout settings for %s@%s; version is already "
295            "the published Default GA. The rollout marker will not be published.",
296            connector_name,
297            version,
298        )
299    should_publish_rollout_marker = (
300        progressive_rollout_enabled
301        and not rollout_overridden_by_breaking_change
302        and not rollout_overridden_by_published_ga
303    )
304    result = PublishArtifactsResult(
305        connector_name=connector_name,
306        version=version,
307        target=target_label,
308        gcs_destination=versioned_dest,
309        progressive_rollout_overridden_by_breaking_change=rollout_overridden_by_breaking_change,
310        progressive_rollout_overridden_by_published_ga=rollout_overridden_by_published_ga,
311        dry_run=dry_run,
312    )
313
314    # --- Pre-publish validation ---
315    if with_validate:
316        metadata_file = artifacts_dir / "metadata.yaml"
317        if metadata_file.is_file():
318            raw_metadata = yaml.safe_load(metadata_file.read_text())
319            metadata_data = (raw_metadata or {}).get("data", {})
320            validation = validate_metadata(metadata_data=metadata_data)
321            if not validation.passed:
322                for err in validation.errors:
323                    logger.error("Pre-publish validation error: %s", err)
324                result.validation_errors = validation.errors
325                return result
326            logger.info(
327                "Pre-publish validation passed (%d validators).",
328                validation.validators_run,
329            )
330        else:
331            logger.warning("No metadata.yaml in artifacts dir; skipping validation.")
332
333    # Enumerate local files
334    local_files = sorted(f for f in artifacts_dir.rglob("*") if f.is_file())
335    if not local_files:
336        result.errors.append(f"No files found in {artifacts_dir}.")
337        return result
338
339    _log_progress(
340        "Publishing %d artifacts for %s@%s%s",
341        len(local_files),
342        connector_name,
343        version,
344        versioned_dest,
345    )
346
347    # Build references used by both dry-run and real upload paths
348    deps_file = artifacts_dir / CONNECTOR_DEPENDENCY_FILE_NAME
349    has_deps = deps_file.is_file()
350    deps_gcs_key = dependencies_blob_path(
351        connector_name=connector_name, version=version, store=store
352    )
353
354    sbom_file = artifacts_dir / SBOM_FILE_NAME
355    has_sbom = sbom_file.is_file()
356    rollout_marker_path = f"{blob_root}/{PROGRESSIVE_ROLLOUT_MARKER_FILE}"
357
358    if dry_run:
359        for f in local_files:
360            rel = f.relative_to(artifacts_dir)
361            result.files_uploaded.append(str(rel))
362            _log_progress("  [DRY RUN] would upload: %s", rel)
363        # Report the dual-load of dependencies.json to connector_dependencies/
364        if has_deps:
365            result.files_uploaded.append(deps_gcs_key)
366            _log_progress(
367                "  [DRY RUN] would also dual-load: %s → gs://%s/%s",
368                CONNECTOR_DEPENDENCY_FILE_NAME,
369                bucket_name,
370                deps_gcs_key,
371            )
372        # Report the separate sbom/ upload
373        if has_sbom:
374            sbom_gcs_key = sbom_blob_path(
375                connector_name=connector_name,
376                version=version,
377                store=store,
378            )
379            result.files_uploaded.append(sbom_gcs_key)
380            _log_progress(
381                "  [DRY RUN] would also upload: %s → gs://%s/%s",
382                SBOM_FILE_NAME,
383                bucket_name,
384                sbom_gcs_key,
385            )
386        if should_publish_rollout_marker:
387            result.files_uploaded.append(PROGRESSIVE_ROLLOUT_MARKER_FILE)
388            _log_progress(
389                "  [DRY RUN] would write active marker: gs://%s/%s",
390                bucket_name,
391                rollout_marker_path,
392            )
393        return result
394
395    # Authenticate
396    token = get_gcs_credentials_token()
397    fs = gcsfs.GCSFileSystem(token=token)
398
399    # Strip gcs:// prefix for gcsfs path
400    dest_path = versioned_dest.replace("gcs://", "")
401
402    # Upload all files to the versioned path
403    _log_progress("Uploading to: %s", versioned_dest)
404    for f in local_files:
405        rel = f.relative_to(artifacts_dir)
406        remote_path = f"{dest_path}/{rel}"
407        fs.put(str(f), remote_path)
408        result.files_uploaded.append(str(rel))
409        _log_progress("  Uploaded: %s", rel)
410
411    # Delete remote files that don't exist locally (sync semantics)
412    try:
413        remote_files = fs.ls(dest_path, detail=False)
414        local_rel_paths = {str(f.relative_to(artifacts_dir)) for f in local_files}
415        for remote_file in remote_files:
416            # Skip the directory entry itself if it appears in the listing
417            if remote_file == dest_path:
418                continue
419            # Derive the remote relative path, matching upload semantics
420            if remote_file.startswith(dest_path + "/"):
421                remote_rel = remote_file[len(dest_path) + 1 :]
422            else:
423                remote_rel = remote_file.split("/")[-1]
424            if is_registry_state_marker_file(Path(remote_rel).name):
425                continue
426            if remote_rel not in local_rel_paths:
427                fs.rm(remote_file)
428                _log_progress("  Deleted stale remote file: %s", remote_rel)
429    except FileNotFoundError:
430        pass  # Destination doesn't exist yet, nothing to clean
431
432    _log_progress("Uploaded %d files to %s", len(local_files), versioned_dest)
433
434    if should_publish_rollout_marker:
435        marker_remote = f"{bucket_name}/{rollout_marker_path}"
436        with fs.open(marker_remote, "w") as marker_file:
437            marker_file.write(_progressive_rollout_marker_content())
438        result.files_uploaded.append(PROGRESSIVE_ROLLOUT_MARKER_FILE)
439        _log_progress("Wrote active marker: gs://%s", marker_remote)
440
441    # --- Dual-load dependencies.json to the connector_dependencies/ path ---
442    if not has_deps:
443        logger.debug(
444            "No %s in artifacts dir — skipping dual-load.",
445            CONNECTOR_DEPENDENCY_FILE_NAME,
446        )
447    else:
448        deps_remote = f"{bucket_name}/{deps_gcs_key}"
449        _log_progress(
450            "Dual-loading %s to gs://%s",
451            CONNECTOR_DEPENDENCY_FILE_NAME,
452            deps_remote,
453        )
454        fs.put(str(deps_file), deps_remote)
455        result.files_uploaded.append(deps_gcs_key)
456        _log_progress("  Uploaded %s (dual-load)", CONNECTOR_DEPENDENCY_FILE_NAME)
457
458    # --- Upload SBOM to the dedicated sbom/ path in GCS ---
459    if not has_sbom:
460        logger.debug(
461            "No %s in artifacts dir — skipping SBOM dual-load.",
462            SBOM_FILE_NAME,
463        )
464    else:
465        sbom_gcs_uri = upload_sbom(
466            sbom_path=sbom_file,
467            connector_name=connector_name,
468            version=version,
469            store=store,
470            dry_run=dry_run,
471        )
472        result.files_uploaded.append(
473            sbom_blob_path(
474                connector_name=connector_name,
475                version=version,
476                store=store,
477            ),
478        )
479        _log_progress("Uploaded SBOM to dedicated path: %s", sbom_gcs_uri)
480
481    return result

Publish locally generated artifacts to a GCS registry bucket.

Uses gcsfs.GCSFileSystem to upload the local artifacts_dir to the versioned path inside the target GCS bucket.

The target GCS path is:

gs://<bucket>/[<prefix>/]metadata/airbyte/<connector>/<version>/

Before uploading, this function validates that the connector_name (derived from the connector directory) matches the dockerRepository declared in metadata.yaml. A mismatch would cause the registry compile step to see duplicate definition-ID entries and fail.

Arguments:
  • connector_name: Connector name (e.g. source-faker).
  • version: Version string (e.g. 6.2.38).
  • artifacts_dir: Local directory containing artifacts from generate.
  • store: Parsed store target containing bucket, prefix, and stage info.
  • dry_run: If True, report what would be uploaded without writing.
  • with_validate: If True (default), validate metadata before uploading. Pass False (--no-validate) to skip.
Returns:

A PublishArtifactsResult describing what was published.

Raises:
  • ValueError: If the connector directory name does not match dockerRepository in the generated metadata.
def purge_latest_dirs( *, store: RegistryStore, connector_name: list[str] | None = None, dry_run: bool = False) -> PurgeLatestResult:
1511def purge_latest_dirs(
1512    *,
1513    store: RegistryStore,
1514    connector_name: list[str] | None = None,
1515    dry_run: bool = False,
1516) -> PurgeLatestResult:
1517    """Delete all `latest/` directories from the registry store.
1518
1519    Discovers connector directories via glob, then deletes each
1520    `latest/` subdirectory in parallel using a thread pool.
1521
1522    Args:
1523        store: Registry store (bucket + optional prefix).
1524        connector_name: If provided, only purge these connectors.
1525        dry_run: If True, report what would be done without deleting.
1526
1527    Returns:
1528        A `PurgeLatestResult` describing what was done.
1529    """
1530    result = PurgeLatestResult(target=store.bucket_root, dry_run=dry_run)
1531
1532    token = get_gcs_credentials_token()
1533    fs = gcsfs.GCSFileSystem(token=token)
1534
1535    base = f"{store.bucket_root}/{METADATA_FOLDER}/airbyte"
1536
1537    # Discover latest/ dirs by listing connector directories that contain
1538    # a `latest/` subdirectory.
1539    _log_progress("Discovering latest/ directories...")
1540    base_with_slash = f"{base}/"
1541    if connector_name:
1542        # Check each requested connector for a latest/ dir
1543        seen: set[str] = set()
1544        connectors_with_latest: list[str] = []
1545        for name in connector_name:
1546            if name in seen:
1547                continue
1548            latest_path = f"{base}/{name}/latest"
1549            if fs.exists(latest_path):
1550                connectors_with_latest.append(name)
1551                seen.add(name)
1552    else:
1553        # Glob for all connectors, then filter to those with latest/
1554        all_connector_dirs = fs.glob(f"{base}/*/latest")
1555        seen = set()
1556        connectors_with_latest = []
1557        for path in all_connector_dirs:
1558            # Strip the known base prefix and take the first component
1559            if not path.startswith(base_with_slash):
1560                logger.warning("Could not parse latest path: %s", path)
1561                continue
1562            relative = path[len(base_with_slash) :]
1563            connector = relative.split("/")[0]
1564            if connector and connector not in seen:
1565                connectors_with_latest.append(connector)
1566                seen.add(connector)
1567
1568    result.connectors_found = len(connectors_with_latest)
1569    _log_progress(
1570        "Found %d connectors with latest/ directories",
1571        result.connectors_found,
1572    )
1573
1574    if not connectors_with_latest:
1575        _log_progress("Nothing to purge.")
1576        _log_progress(result.summary())
1577        return result
1578
1579    if dry_run:
1580        for connector in sorted(connectors_with_latest):
1581            _log_progress("  [DRY RUN] Would delete %s/latest/", connector)
1582        result.latest_dirs_deleted = len(connectors_with_latest)
1583        _log_progress(result.summary())
1584        return result
1585
1586    # Delete latest/ dirs in parallel using the shared helper.
1587    def _delete_one(connector: str) -> str | None:
1588        """Delete a single connector's latest/ dir. Returns error string or None."""
1589        try:
1590            _delete_latest_dir(
1591                fs,
1592                store=store,
1593                connector=connector,
1594            )
1595            return None
1596        except Exception as exc:
1597            return f"Failed to delete latest/ for {connector}: {exc}"
1598
1599    _log_progress(
1600        "Deleting %d latest/ directories (max_workers=%d)...",
1601        len(connectors_with_latest),
1602        _PURGE_LATEST_MAX_WORKERS,
1603    )
1604
1605    with ThreadPoolExecutor(max_workers=_PURGE_LATEST_MAX_WORKERS) as pool:
1606        futures = {
1607            pool.submit(_delete_one, c): c for c in sorted(connectors_with_latest)
1608        }
1609        for i, future in enumerate(as_completed(futures), 1):
1610            connector = futures[future]
1611            error = future.result()
1612            if error:
1613                logger.error(error)
1614                result.errors.append(error)
1615            else:
1616                result.latest_dirs_deleted += 1
1617            if i % 100 == 0:
1618                _log_progress("  Deleted %d / %d...", i, len(connectors_with_latest))
1619
1620    _log_progress(result.summary())
1621    return result

Delete all latest/ directories from the registry store.

Discovers connector directories via glob, then deletes each latest/ subdirectory in parallel using a thread pool.

Arguments:
  • store: Registry store (bucket + optional prefix).
  • connector_name: If provided, only purge these connectors.
  • dry_run: If True, report what would be done without deleting.
Returns:

A PurgeLatestResult describing what was done.

def rebuild_registry( source_bucket: str, output_mode: Literal['local', 'gcs', 's3'], output_path_root: str | None = None, gcs_bucket: str | None = None, s3_bucket: str | None = None, dry_run: bool = False, connector_name: list[str] | None = None) -> RebuildResult:
147def rebuild_registry(
148    source_bucket: str,
149    output_mode: OutputMode,
150    output_path_root: str | None = None,
151    gcs_bucket: str | None = None,
152    s3_bucket: str | None = None,
153    dry_run: bool = False,
154    connector_name: list[str] | None = None,
155) -> RebuildResult:
156    """Rebuild the entire registry from a source GCS bucket to an output target.
157
158    Reads all connector metadata blobs from the source GCS bucket and copies
159    them to the output target using fsspec for unified filesystem access.
160
161    The output targets are:
162    - local: Write to a local directory tree.
163    - gcs: Copy to a GCS bucket (must not be the prod bucket).
164    - s3: Copy to an S3 bucket.
165
166    Args:
167        source_bucket: The GCS bucket to read from (typically prod).
168        output_mode: Where to write: "local", "gcs", or "s3".
169        output_path_root: Root path/prefix for output. For local mode, if None
170            creates a temp directory. For GCS/S3, prepended to all blob paths.
171        gcs_bucket: Target GCS bucket name (required if output_mode="gcs").
172        s3_bucket: Target S3 bucket name (required if output_mode="s3").
173        dry_run: If True, report what would be done without writing.
174        connector_name: If provided, only rebuild these connector names
175            (e.g. ["source-faker", "destination-bigquery"]). If None, rebuilds all.
176
177    Returns:
178        RebuildResult with details of the operation.
179
180    Raises:
181        ValueError: If target is the prod bucket, or required bucket arg is missing.
182    """
183    if output_mode == "gcs" and gcs_bucket:
184        _validate_not_prod_bucket(gcs_bucket)
185
186    # Resolve output root for local mode
187    effective_output_root = output_path_root or ""
188    if output_mode == "local":
189        effective_output_root = _resolve_local_output_root(output_path_root)
190
191    result = RebuildResult(
192        source_bucket=source_bucket,
193        output_mode=output_mode,
194        output_root=effective_output_root,
195        dry_run=dry_run,
196    )
197
198    # Create source filesystem (always GCS)
199    source_fs, gcs_token = _make_source_fs()
200    source_base = f"{source_bucket}/{METADATA_FOLDER}"
201
202    # List files under metadata/ in the source bucket
203    if connector_name:
204        _log_progress(
205            "Listing blobs for %d connectors under gs://%s/...",
206            len(connector_name),
207            source_base,
208        )
209        source_paths: list[str] = []
210        for name in connector_name:
211            connector_prefix = f"{source_base}/airbyte/{name}"
212            found = source_fs.find(connector_prefix)
213            source_paths.extend(found)
214            _log_progress("  %s: %d blobs", name, len(found))
215    else:
216        _log_progress("Listing all blobs under gs://%s/...", source_base)
217        source_paths = source_fs.find(source_base)
218
219    if not source_paths:
220        _log_progress("No blobs found under gs://%s/", source_base)
221        return result
222
223    total_blobs = len(source_paths)
224    _log_progress("Found %d blobs to process", total_blobs)
225
226    # Create output filesystem
227    output_fs, output_base = _make_output_fs(
228        output_mode=output_mode,
229        output_root=effective_output_root,
230        gcs_bucket=gcs_bucket,
231        s3_bucket=s3_bucket,
232        gcs_token=gcs_token,
233    )
234
235    # Collect connector names and compute relative paths for all blobs.
236    bucket_prefix = f"{source_bucket}/"
237    blob_relative_paths: list[str] = []
238    connector_names: set[str] = set()
239
240    for source_path in source_paths:
241        relative_path = source_path
242        if source_path.startswith(bucket_prefix):
243            relative_path = source_path[len(bucket_prefix) :]
244        blob_relative_paths.append(relative_path)
245
246        parts = relative_path.split("/")
247        if len(parts) >= 3:
248            connector_names.add(parts[2])
249
250    if dry_run:
251        result.blobs_copied = total_blobs
252        result.connectors_processed = len(connector_names)
253        _log_progress(
254            "[DRY RUN] Would copy %d blobs (%d connectors)",
255            total_blobs,
256            len(connector_names),
257        )
258        _log_progress(result.summary())
259        return result
260
261    # Use GCS-native server-side copy for GCS→GCS mirrors.
262    if output_mode == "gcs" and gcs_bucket:
263        result = _gcs_native_copy(
264            source_bucket=source_bucket,
265            dest_bucket_name=gcs_bucket,
266            dest_prefix=effective_output_root,
267            blob_relative_paths=blob_relative_paths,
268            connector_names=connector_names,
269            gcs_token=gcs_token,
270            result=result,
271            connector_name_filter=connector_name,
272        )
273        return result
274
275    # Fallback: fsspec-based copy for local and S3 output modes.
276    result = _fsspec_copy(
277        source_fs=source_fs,
278        source_paths=source_paths,
279        output_fs=output_fs,
280        output_base=output_base,
281        output_mode=output_mode,
282        source_bucket=source_bucket,
283        blob_relative_paths=blob_relative_paths,
284        connector_names=connector_names,
285        result=result,
286    )
287    return result

Rebuild the entire registry from a source GCS bucket to an output target.

Reads all connector metadata blobs from the source GCS bucket and copies them to the output target using fsspec for unified filesystem access.

The output targets are:

  • local: Write to a local directory tree.
  • gcs: Copy to a GCS bucket (must not be the prod bucket).
  • s3: Copy to an S3 bucket.
Arguments:
  • source_bucket: The GCS bucket to read from (typically prod).
  • output_mode: Where to write: "local", "gcs", or "s3".
  • output_path_root: Root path/prefix for output. For local mode, if None creates a temp directory. For GCS/S3, prepended to all blob paths.
  • gcs_bucket: Target GCS bucket name (required if output_mode="gcs").
  • s3_bucket: Target S3 bucket name (required if output_mode="s3").
  • dry_run: If True, report what would be done without writing.
  • connector_name: If provided, only rebuild these connector names (e.g. ["source-faker", "destination-bigquery"]). If None, rebuilds all.
Returns:

RebuildResult with details of the operation.

Raises:
  • ValueError: If target is the prod bucket, or required bucket arg is missing.
def resolve_registry_store( store: str | None = None, connector_name: str | None = None, cwd: pathlib.Path | None = None, default_env: str = 'dev') -> RegistryStore:
229def resolve_registry_store(
230    store: str | None = None,
231    connector_name: str | None = None,
232    cwd: Path | None = None,
233    default_env: str = "dev",
234) -> RegistryStore:
235    """Resolve a `RegistryStore` from CLI inputs.
236
237    All applicable detection methods are evaluated.  Explicit sources
238    (`--store`, then the `AIRBYTE_REGISTRY_STORE` env var) take priority
239    and are returned directly.  When only auto-detected sources remain, they
240    are compared and a `ValueError` is raised if they disagree.
241
242    Priority (highest → lowest):
243
244    1. **Explicit** `--store` argument (e.g. `"coral:dev"`).
245    2. **Environment variable** -- `AIRBYTE_REGISTRY_STORE`.
246    3. **Auto-detected** -- connector name and/or working directory.
247       If both are present and disagree, a `ValueError` is raised.
248
249    Args:
250        store: Explicit store target string (e.g. `"coral:dev"`).
251        connector_name: Optional connector name for auto-detection.
252        cwd: Working directory for repo-based detection.
253        default_env: Environment to use when auto-detecting (default `"dev"`).
254
255    Returns:
256        A fully resolved `RegistryStore`.
257
258    Raises:
259        ValueError: If no detection method succeeds, or if auto-detected
260            methods produce conflicting store types.
261    """
262    # -- Collect all detection results ------------------------------------
263    # Explicit sources (take priority — no conflict checking needed).
264    explicit_target: RegistryStore | None = None
265    explicit_source: str | None = None
266
267    if store is not None:
268        explicit_target = RegistryStore.parse(store)
269        explicit_source = "--store"
270
271    env_store = os.environ.get(REGISTRY_STORE_ENV_VAR)
272    if env_store and explicit_target is None:
273        explicit_target = RegistryStore.parse(env_store)
274        explicit_source = REGISTRY_STORE_ENV_VAR
275
276    # Auto-detected sources (only consulted when no explicit source).
277    auto_detections: dict[str, StoreType] = {}
278
279    if connector_name is not None:
280        auto_detections["connector_name"] = StoreType.get_from_connector_name(
281            connector_name,
282        )
283
284    dir_type = StoreType.detect_from_repo_dir(cwd)
285    if dir_type is not None:
286        auto_detections["working_directory"] = dir_type
287
288    # -- Return explicit source if present --------------------------------
289    if explicit_target is not None:
290        logger.debug(
291            "Using explicit store target from %s: %s:%s",
292            explicit_source,
293            explicit_target.store_type.value,
294            explicit_target.env,
295        )
296        return explicit_target
297
298    # -- No explicit source: resolve from auto-detections -----------------
299    if not auto_detections:
300        raise ValueError(
301            "Cannot determine registry store. "
302            "Provide --store (e.g. 'coral:dev' or 'sonar:prod'), "
303            f"set ${REGISTRY_STORE_ENV_VAR}, "
304            "or run from a recognized repository directory."
305        )
306
307    distinct = set(auto_detections.values())
308
309    if len(distinct) > 1:
310        detail = ", ".join(f"{src}={st.value}" for src, st in auto_detections.items())
311        raise ValueError(
312            f"Conflicting store types detected: {detail}. "
313            "Provide an explicit --store to resolve the ambiguity."
314        )
315
316    resolved_type = distinct.pop()
317    logger.info(
318        "Auto-detected store type '%s' (sources: %s)",
319        resolved_type.value,
320        ", ".join(auto_detections),
321    )
322    return RegistryStore(store_type=resolved_type, env=default_env)

Resolve a RegistryStore from CLI inputs.

All applicable detection methods are evaluated. Explicit sources (--store, then the AIRBYTE_REGISTRY_STORE env var) take priority and are returned directly. When only auto-detected sources remain, they are compared and a ValueError is raised if they disagree.

Priority (highest → lowest):

  1. Explicit --store argument (e.g. "coral:dev").
  2. Environment variable -- AIRBYTE_REGISTRY_STORE.
  3. Auto-detected -- connector name and/or working directory. If both are present and disagree, a ValueError is raised.
Arguments:
  • store: Explicit store target string (e.g. "coral:dev").
  • connector_name: Optional connector name for auto-detection.
  • cwd: Working directory for repo-based detection.
  • default_env: Environment to use when auto-detecting (default "dev").
Returns:

A fully resolved RegistryStore.

Raises:
  • ValueError: If no detection method succeeds, or if auto-detected methods produce conflicting store types.
def unyank_connector_version( connector_name: str, version: str, bucket_name: str, dry_run: bool = False) -> YankResult:
322def unyank_connector_version(
323    connector_name: str,
324    version: str,
325    bucket_name: str,
326    dry_run: bool = False,
327) -> YankResult:
328    """Rename the active yank marker to an unyanked audit marker.
329
330    Moves the active version-yank.yml marker at:
331        metadata/airbyte/{connector_name}/{version}/version-yank.yml
332    to:
333        metadata/airbyte/{connector_name}/{version}/version-unyanked-yyyymmdd.yml
334
335    Args:
336        connector_name: The connector name (e.g., "source-faker").
337        version: The version to unyank (e.g., "1.2.3").
338        bucket_name: The GCS bucket name.
339        dry_run: If True, report what would be done without writing.
340
341    Returns:
342        YankResult with details of the operation.
343    """
344    yank_path = _get_yank_blob_path(connector_name, version)
345
346    storage_client = get_gcs_storage_client()
347    bucket = storage_client.bucket(bucket_name)
348
349    # Check if yank marker exists
350    yank_blob = bucket.blob(yank_path)
351    if not yank_blob.exists():
352        return YankResult(
353            connector_name=connector_name,
354            version=version,
355            bucket_name=bucket_name,
356            action="unyank",
357            success=False,
358            message=f"Version {version} of {connector_name} is not yanked.",
359            dry_run=dry_run,
360        )
361
362    if dry_run:
363        return YankResult(
364            connector_name=connector_name,
365            version=version,
366            bucket_name=bucket_name,
367            action="unyank",
368            success=True,
369            message=f"[DRY RUN] Would unyank {connector_name} {version}.",
370            dry_run=True,
371        )
372
373    unyanked_path = _get_yank_blob_path(connector_name, version).replace(
374        YANK_FILE_NAME,
375        unyanked_marker_file(),
376    )
377    bucket.copy_blob(yank_blob, bucket, new_name=unyanked_path)
378    yank_blob.delete()
379
380    logger.info("Unyanked %s version %s in %s", connector_name, version, bucket_name)
381
382    return YankResult(
383        connector_name=connector_name,
384        version=version,
385        bucket_name=bucket_name,
386        action="unyank",
387        success=True,
388        message=f"Successfully unyanked {connector_name} {version}.",
389    )

Rename the active yank marker to an unyanked audit marker.

Moves the active version-yank.yml marker at: metadata/airbyte/{connector_name}/{version}/version-yank.yml to: metadata/airbyte/{connector_name}/{version}/version-unyanked-yyyymmdd.yml

Arguments:
  • connector_name: The connector name (e.g., "source-faker").
  • version: The version to unyank (e.g., "1.2.3").
  • bucket_name: The GCS bucket name.
  • dry_run: If True, report what would be done without writing.
Returns:

YankResult with details of the operation.

def validate_metadata( metadata_data: dict[str, typing.Any], opts: ValidateOptions | None = None) -> ValidationResult:
270def validate_metadata(
271    metadata_data: dict[str, Any],
272    opts: ValidateOptions | None = None,
273) -> ValidationResult:
274    """Run all pre-publish validators against raw `metadata.data`.
275
276    Args:
277        metadata_data: The `data` section of a parsed `metadata.yaml`.
278        opts: Options influencing validation behaviour.
279
280    Returns:
281        A `ValidationResult` with aggregate pass/fail and error list.
282    """
283    if opts is None:
284        opts = ValidateOptions()
285
286    result = ValidationResult()
287
288    for validator in PRE_PUBLISH_VALIDATORS:
289        result.validators_run += 1
290        logger.info("Running validator: %s", validator.__name__)
291        passed, error = validator(metadata_data, opts)
292        if not passed and error:
293            logger.error("Validation failed: %s", error)
294            result.add_error(error)
295
296    return result

Run all pre-publish validators against raw metadata.data.

Arguments:
  • metadata_data: The data section of a parsed metadata.yaml.
  • opts: Options influencing validation behaviour.
Returns:

A ValidationResult with aggregate pass/fail and error list.

def yank_connector_version( connector_name: str, version: str, bucket_name: str, reason: str = '', approval_url: str = '', dry_run: bool = False) -> YankResult:
226def yank_connector_version(
227    connector_name: str,
228    version: str,
229    bucket_name: str,
230    reason: str = "",
231    approval_url: str = "",
232    dry_run: bool = False,
233) -> YankResult:
234    """Mark a connector version as yanked by writing a version-yank.yml marker.
235
236    The marker file is placed at:
237        metadata/airbyte/{connector_name}/{version}/version-yank.yml
238
239    Args:
240        connector_name: The connector name (e.g., "source-faker").
241        version: The version to yank (e.g., "1.2.3").
242        bucket_name: The GCS bucket name.
243        reason: Optional reason for yanking the version.
244        approval_url: Optional approval evidence URL to record in the marker.
245        dry_run: If True, report what would be done without writing.
246
247    Returns:
248        YankResult with details of the operation.
249
250    Raises:
251        ValueError: If the bucket is the production bucket and no override is set,
252            or if the version does not exist.
253    """
254    yank_path = _get_yank_blob_path(connector_name, version)
255    metadata_path = _get_metadata_blob_path(connector_name, version)
256
257    storage_client = get_gcs_storage_client()
258    bucket = storage_client.bucket(bucket_name)
259
260    # Verify the version exists
261    metadata_blob = bucket.blob(metadata_path)
262    if not metadata_blob.exists():
263        return YankResult(
264            connector_name=connector_name,
265            version=version,
266            bucket_name=bucket_name,
267            action="yank",
268            success=False,
269            message=f"Version {version} not found for {connector_name} in {bucket_name}.",
270            dry_run=dry_run,
271        )
272
273    # Check if already yanked
274    yank_blob = bucket.blob(yank_path)
275    if yank_blob.exists():
276        return YankResult(
277            connector_name=connector_name,
278            version=version,
279            bucket_name=bucket_name,
280            action="yank",
281            success=False,
282            message=f"Version {version} of {connector_name} is already yanked.",
283            dry_run=dry_run,
284        )
285
286    if dry_run:
287        return YankResult(
288            connector_name=connector_name,
289            version=version,
290            bucket_name=bucket_name,
291            action="yank",
292            success=True,
293            message=f"[DRY RUN] Would yank {connector_name} {version}.",
294            dry_run=True,
295        )
296
297    # Write the yank marker file
298    yank_content: dict[str, Any] = {
299        "yanked": True,
300        "yanked_at": datetime.now(tz=timezone.utc).isoformat(),
301    }
302    if reason:
303        yank_content["reason"] = reason
304    if approval_url:
305        yank_content["approval_url"] = approval_url
306
307    yank_yaml = yaml.dump(yank_content, default_flow_style=False)
308    yank_blob.upload_from_string(yank_yaml, content_type="application/x-yaml")
309
310    logger.info("Yanked %s version %s in %s", connector_name, version, bucket_name)
311
312    return YankResult(
313        connector_name=connector_name,
314        version=version,
315        bucket_name=bucket_name,
316        action="yank",
317        success=True,
318        message=f"Successfully yanked {connector_name} {version}.",
319    )

Mark a connector version as yanked by writing a version-yank.yml marker.

The marker file is placed at:

metadata/airbyte/{connector_name}/{version}/version-yank.yml

Arguments:
  • connector_name: The connector name (e.g., "source-faker").
  • version: The version to yank (e.g., "1.2.3").
  • bucket_name: The GCS bucket name.
  • reason: Optional reason for yanking the version.
  • approval_url: Optional approval evidence URL to record in the marker.
  • dry_run: If True, report what would be done without writing.
Returns:

YankResult with details of the operation.

Raises:
  • ValueError: If the bucket is the production bucket and no override is set, or if the version does not exist.