airbyte_ops_mcp.airbyte_repo
Airbyte repository operations.
This package provides tools for working with the Airbyte monorepo, including:
- Detecting changed connectors
- Filtering connectors by certification status
- Format checking and fixing
- CI matrix planning
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Airbyte repository operations. 3 4This package provides tools for working with the Airbyte monorepo, including: 5- Detecting changed connectors 6- Filtering connectors by certification status 7- Format checking and fixing 8- CI matrix planning 9""" 10 11from __future__ import annotations 12 13from airbyte_ops_mcp.airbyte_repo.bump_version import ( 14 BumpType, 15 ChangelogEntry, 16 ChangelogParsingError, 17 ConnectorNotFoundError, 18 ConnectorVersionError, 19 InvalidVersionError, 20 VersionBumpResult, 21 VersionNotFoundError, 22 bump_connector_version, 23 strip_prerelease_suffix, 24) 25from airbyte_ops_mcp.airbyte_repo.list_connectors import ( 26 ConnectorLanguage, 27 ConnectorListResult, 28 get_all_connectors, 29 get_certified_connectors, 30 get_changed_files, 31 get_connectors_by_language, 32 get_connectors_with_local_cdk, 33 get_file_at_merge_base, 34 get_merge_base, 35 get_modified_connectors, 36 list_connectors, 37) 38from airbyte_ops_mcp.airbyte_repo.utils import ( 39 detect_env_pr_info, 40 parse_pr_info, 41 resolve_diff_range, 42) 43 44__all__ = [ 45 "BumpType", 46 "ChangelogEntry", 47 "ChangelogParsingError", 48 "ConnectorLanguage", 49 "ConnectorListResult", 50 "ConnectorNotFoundError", 51 "ConnectorVersionError", 52 "InvalidVersionError", 53 "VersionBumpResult", 54 "VersionNotFoundError", 55 "bump_connector_version", 56 "detect_env_pr_info", 57 "get_all_connectors", 58 "get_certified_connectors", 59 "get_changed_files", 60 "get_connectors_by_language", 61 "get_connectors_with_local_cdk", 62 "get_file_at_merge_base", 63 "get_merge_base", 64 "get_modified_connectors", 65 "list_connectors", 66 "parse_pr_info", 67 "resolve_diff_range", 68 "strip_prerelease_suffix", 69]
41class BumpType(StrEnum): 42 """Supported version bump types.""" 43 44 PATCH = "patch" 45 MINOR = "minor" 46 MAJOR = "major" 47 PATCH_RC = "patch_rc" 48 MINOR_RC = "minor_rc" 49 MAJOR_RC = "major_rc" 50 RC = "rc" 51 PROMOTE = "promote"
Supported version bump types.
95@dataclass(frozen=True) 96class ChangelogEntry: 97 """A single changelog entry.""" 98 99 date: datetime.date 100 version: semver.Version 101 pr_number: int | str 102 comment: str 103 104 def to_markdown(self, github_repo: str = AIRBYTE_GITHUB_REPO) -> str: 105 """Convert entry to markdown table row.""" 106 return ( 107 f"| {self.version} | {self.date.strftime('%Y-%m-%d')} | " 108 f"[{self.pr_number}](https://github.com/{github_repo}/pull/{self.pr_number}) | " 109 f"{self.comment} |" 110 )
A single changelog entry.
104 def to_markdown(self, github_repo: str = AIRBYTE_GITHUB_REPO) -> str: 105 """Convert entry to markdown table row.""" 106 return ( 107 f"| {self.version} | {self.date.strftime('%Y-%m-%d')} | " 108 f"[{self.pr_number}](https://github.com/{github_repo}/pull/{self.pr_number}) | " 109 f"{self.comment} |" 110 )
Convert entry to markdown table row.
78class ChangelogParsingError(ConnectorVersionError): 79 """Raised when changelog cannot be parsed.""" 80 81 pass
Raised when changelog cannot be parsed.
88class ConnectorLanguage(StrEnum): 89 """Connector implementation languages.""" 90 91 PYTHON = "python" 92 JAVA = "java" 93 LOW_CODE = "low-code" 94 MANIFEST_ONLY = "manifest-only" 95 96 @classmethod 97 def parse(cls, value: str) -> ConnectorLanguage: 98 """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch.""" 99 try: 100 return cls(value) 101 except ValueError: 102 valid = ", ".join(f"`{m.value}`" for m in cls) 103 raise ValueError( 104 f"Unrecognized language: {value!r}. Expected one of: {valid}." 105 ) from None
Connector implementation languages.
96 @classmethod 97 def parse(cls, value: str) -> ConnectorLanguage: 98 """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch.""" 99 try: 100 return cls(value) 101 except ValueError: 102 valid = ", ".join(f"`{m.value}`" for m in cls) 103 raise ValueError( 104 f"Unrecognized language: {value!r}. Expected one of: {valid}." 105 ) from None
Parse a string into a ConnectorLanguage, raising ValueError on mismatch.
679@dataclass 680class ConnectorListResult: 681 """Result of listing connectors with filters.""" 682 683 connectors: list[str] 684 count: int
Result of listing connectors with filters.
60class ConnectorNotFoundError(ConnectorVersionError): 61 """Raised when a connector is not found.""" 62 63 pass
Raised when a connector is not found.
54class ConnectorVersionError(Exception): 55 """Base exception for connector version operations.""" 56 57 pass
Base exception for connector version operations.
72class InvalidVersionError(ConnectorVersionError): 73 """Raised when a version string is invalid.""" 74 75 pass
Raised when a version string is invalid.
84@dataclass 85class VersionBumpResult: 86 """Result of a version bump operation.""" 87 88 connector: str 89 previous_version: str 90 new_version: str 91 files_modified: list[str] 92 dry_run: bool
Result of a version bump operation.
66class VersionNotFoundError(ConnectorVersionError): 67 """Raised when version cannot be found in a file.""" 68 69 pass
Raised when version cannot be found in a file.
685def bump_connector_version( 686 repo_path: str | Path, 687 connector_name: str, 688 bump_type: Literal[ 689 "patch", "minor", "major", "patch_rc", "minor_rc", "major_rc", "rc", "promote" 690 ] 691 | None = None, 692 new_version: str | None = None, 693 changelog_message: str | None = None, 694 pr_number: int | str | None = None, 695 dry_run: bool = False, 696 no_changelog: bool = False, 697 progressive_rollout_enabled: bool | None = None, 698) -> VersionBumpResult: 699 """Bump a connector's version across all relevant files. 700 701 This function updates the version in: 702 - metadata.yaml (always) 703 - pyproject.toml (if exists, for Python connectors) 704 - Documentation changelog (if changelog_message provided, doc exists, and 705 no_changelog is False) 706 707 For RC bump types (`patch_rc`, `minor_rc`, `major_rc`, `rc` from a 708 stable version), this also sets `enableProgressiveRollout: true` in 709 metadata so the registry compile step excludes this version from 710 default/latest evaluation while the rollout is in progress. 711 712 For `promote`, this clears `enableProgressiveRollout` to false so the 713 version becomes eligible for default/latest again — unless the connector's 714 `defaultRolloutMode` is `autopilot`, in which case the flag is left `true` 715 (autopilot connectors keep progressive rollout on as their default). 716 717 Args: 718 repo_path: Path to the Airbyte monorepo 719 connector_name: Technical name of the connector (e.g., "source-github") 720 bump_type: Type of version bump (patch, minor, major, patch_rc, minor_rc, major_rc, rc, promote) 721 new_version: Explicit new version (overrides bump_type) 722 changelog_message: Message to add to changelog (optional) 723 pr_number: PR number for changelog entry (optional) 724 dry_run: If True, don't actually modify files 725 no_changelog: If True, skip changelog updates even if changelog_message 726 is provided. Useful for ephemeral version bumps (e.g. pre-release 727 artifact generation) where the version is temporary. 728 progressive_rollout_enabled: Explicit override for the 729 `enableProgressiveRollout` flag in metadata.yaml. When `True` or 730 `False`, the flag is set to that value (without touching registry 731 overrides). When `None` (the default), the automatic behaviour 732 based on `bump_type` is used instead. 733 734 Returns: 735 VersionBumpResult with details of the operation 736 737 Raises: 738 ConnectorNotFoundError: If connector doesn't exist 739 VersionNotFoundError: If current version cannot be found 740 InvalidVersionError: If version format is invalid 741 ValueError: If neither bump_type nor new_version is provided 742 """ 743 repo_path = Path(repo_path) 744 connector_path = get_connector_path(repo_path, connector_name) 745 746 # Get current version 747 current_version = get_current_version(connector_path) 748 749 # Calculate new version 750 bump_type_enum = BumpType(bump_type) if bump_type else None 751 calculated_version = calculate_new_version( 752 current_version, bump_type_enum, new_version 753 ) 754 755 if ( 756 changelog_message 757 and not no_changelog 758 and not semver.Version.is_valid(calculated_version) 759 ): 760 raise InvalidVersionError( 761 f"Changelog updates are not supported for non-semver pre-release " 762 f"version {calculated_version}; pass --no-changelog." 763 ) 764 765 files_modified: list[str] = [] 766 metadata_file = connector_path / METADATA_FILE_NAME 767 if not metadata_file.exists(): 768 raise ConnectorNotFoundError(f"metadata.yaml not found at {metadata_file}") 769 metadata_file_rel = str(metadata_file.relative_to(repo_path)) 770 771 # Update metadata.yaml 772 if update_metadata_version(connector_path, calculated_version, dry_run): 773 files_modified.append(metadata_file_rel) 774 775 # Update pyproject.toml if it exists 776 if update_pyproject_version(connector_path, calculated_version, dry_run): 777 files_modified.append( 778 f"{CONNECTOR_PATH_PREFIX}/{connector_name}/{PYPROJECT_FILE_NAME}" 779 ) 780 781 # Update changelog if message provided and not explicitly skipped 782 if changelog_message and not no_changelog: 783 doc_path = get_connector_doc_path(repo_path, connector_name) 784 if doc_path and update_changelog( 785 doc_path, 786 calculated_version, 787 changelog_message, 788 pr_number, 789 dry_run, 790 ): 791 files_modified.append(str(doc_path.relative_to(repo_path))) 792 793 # Progressive rollout handling 794 795 # Automatic behaviour based on bump_type. 796 # Configure progressive rollout metadata for initial RC bumps 797 # (skip if current version is already an RC — flag is already set) 798 is_current_rc = semver.Version.is_valid(current_version) and _is_rc( 799 semver.Version.parse(current_version) 800 ) 801 if ( 802 bump_type_enum in _RC_BUMP_TYPES 803 and not is_current_rc 804 and _setup_progressive_rollout_metadata( 805 connector_path=connector_path, 806 dry_run=dry_run, 807 ) 808 and metadata_file_rel not in files_modified 809 ): 810 files_modified.append(metadata_file_rel) 811 812 # Clean up progressive rollout metadata on promote (RC → GA) 813 if ( 814 bump_type_enum == BumpType.PROMOTE 815 and _cleanup_progressive_rollout_metadata( 816 connector_path=connector_path, dry_run=dry_run 817 ) 818 and metadata_file_rel not in files_modified 819 ): 820 files_modified.append(metadata_file_rel) 821 822 # Explicit override — applied last so it takes precedence over 823 # the automatic steps above (e.g. preview builds can force the 824 # flag off even when bump_type would normally enable it). 825 if progressive_rollout_enabled is not None: 826 metadata = load_metadata_yaml(metadata_file) 827 data = metadata.get("data", {}) 828 if set_progressive_rollout_flag(data, progressive_rollout_enabled): 829 metadata["data"] = data 830 if not dry_run: 831 write_metadata_yaml(metadata, metadata_file) 832 if metadata_file_rel not in files_modified: 833 files_modified.append(metadata_file_rel) 834 835 return VersionBumpResult( 836 connector=connector_name, 837 previous_version=current_version, 838 new_version=calculated_version, 839 files_modified=files_modified, 840 dry_run=dry_run, 841 )
Bump a connector's version across all relevant files.
This function updates the version in:
- metadata.yaml (always)
- pyproject.toml (if exists, for Python connectors)
- Documentation changelog (if changelog_message provided, doc exists, and no_changelog is False)
For RC bump types (patch_rc, minor_rc, major_rc, rc from a
stable version), this also sets enableProgressiveRollout: true in
metadata so the registry compile step excludes this version from
default/latest evaluation while the rollout is in progress.
For promote, this clears enableProgressiveRollout to false so the
version becomes eligible for default/latest again — unless the connector's
defaultRolloutMode is autopilot, in which case the flag is left true
(autopilot connectors keep progressive rollout on as their default).
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_name: Technical name of the connector (e.g., "source-github")
- bump_type: Type of version bump (patch, minor, major, patch_rc, minor_rc, major_rc, rc, promote)
- new_version: Explicit new version (overrides bump_type)
- changelog_message: Message to add to changelog (optional)
- pr_number: PR number for changelog entry (optional)
- dry_run: If True, don't actually modify files
- no_changelog: If True, skip changelog updates even if changelog_message is provided. Useful for ephemeral version bumps (e.g. pre-release artifact generation) where the version is temporary.
- progressive_rollout_enabled: Explicit override for the
enableProgressiveRolloutflag in metadata.yaml. WhenTrueorFalse, the flag is set to that value (without touching registry overrides). WhenNone(the default), the automatic behaviour based onbump_typeis used instead.
Returns:
VersionBumpResult with details of the operation
Raises:
- ConnectorNotFoundError: If connector doesn't exist
- VersionNotFoundError: If current version cannot be found
- InvalidVersionError: If version format is invalid
- ValueError: If neither bump_type nor new_version is provided
39def detect_env_pr_info() -> tuple[int | None, str | None, str | None]: 40 """Detect PR number, owner, and repo from GitHub Actions environment variables. 41 42 Returns: 43 Tuple of (pr_number, pr_owner, pr_repo) 44 """ 45 # Try to extract PR number from GITHUB_REF (e.g., "refs/pull/123/merge") 46 github_ref = os.getenv("GITHUB_REF", "") 47 m = re.match(r"refs/pull/(\d+)/", github_ref) 48 pr_number = int(m.group(1)) if m else None 49 50 # Try to extract owner and repo from GITHUB_REPOSITORY (e.g., "airbytehq/airbyte") 51 pr_owner = None 52 pr_repo = None 53 github_repo = os.getenv("GITHUB_REPOSITORY", "") 54 if github_repo and "/" in github_repo: 55 parts = github_repo.split("/", 1) 56 pr_owner = parts[0] 57 pr_repo = parts[1] 58 59 return pr_number, pr_owner, pr_repo
Detect PR number, owner, and repo from GitHub Actions environment variables.
Returns:
Tuple of (pr_number, pr_owner, pr_repo)
334def get_all_connectors(repo_path: str | Path) -> set[str]: 335 """Get set of all connector IDs in the repository. 336 337 Args: 338 repo_path: Path to the Airbyte monorepo 339 340 Returns: 341 Set of all connector technical names 342 343 Example: 344 >>> all_connectors = get_all_connectors("/path/to/airbyte") 345 >>> "source-faker" in all_connectors 346 True 347 """ 348 repo_path = Path(repo_path) 349 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 350 351 if not connectors_dir.exists(): 352 raise ValueError(f"Connectors directory not found: {connectors_dir}") 353 354 return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
Get set of all connector IDs in the repository.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of all connector technical names
Example:
>>> all_connectors = get_all_connectors("/path/to/airbyte") >>> "source-faker" in all_connectors True
282@lru_cache(maxsize=128) 283def get_certified_connectors(repo_path: str | Path) -> set[str]: 284 """Get set of all certified connector IDs from metadata. 285 286 This function reads the metadata.yaml file for each connector to determine 287 which connectors have "certified" support level. 288 289 Args: 290 repo_path: Path to the Airbyte monorepo 291 292 Returns: 293 Set of certified connector technical names 294 295 Example: 296 >>> certified = get_certified_connectors("/path/to/airbyte") 297 >>> "source-postgres" in certified 298 True 299 """ 300 repo_path = Path(repo_path) 301 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 302 303 if not connectors_dir.exists(): 304 raise ValueError(f"Connectors directory not found: {connectors_dir}") 305 306 certified_connectors = set() 307 308 # Iterate through all connector directories 309 for connector_dir in connectors_dir.iterdir(): 310 if not connector_dir.is_dir(): 311 continue 312 313 metadata_file = connector_dir / METADATA_FILE_NAME 314 if not metadata_file.exists(): 315 continue 316 317 # Read metadata to check support level 318 try: 319 import yaml 320 321 with open(metadata_file) as f: 322 metadata = yaml.safe_load(f) 323 324 support_level = metadata.get("data", {}).get("supportLevel") 325 if support_level == "certified": 326 certified_connectors.add(connector_dir.name) 327 except Exception: 328 # Skip connectors with invalid metadata 329 continue 330 331 return certified_connectors
Get set of all certified connector IDs from metadata.
This function reads the metadata.yaml file for each connector to determine which connectors have "certified" support level.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of certified connector technical names
Example:
>>> certified = get_certified_connectors("/path/to/airbyte") >>> "source-postgres" in certified True
148def get_changed_files( 149 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 150) -> list[str]: 151 """List the files under a directory that changed since a base git reference. 152 153 Compares the working tree, untracked files included, with the merge base of 154 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 155 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 156 so a caller that skips work based on the result learns when the changes are 157 unknown. 158 159 Args: 160 path: A directory inside the git checkout. Only changes under it are listed. 161 base_ref: Base git reference to compare against, e.g. `origin/master`, 162 or a merge base from `get_merge_base`. 163 164 Returns: 165 Sorted paths relative to `path`, exactly as stored (git does not quote 166 them). A move lists both the old and the new path. 167 168 Raises: 169 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 170 git checkout, or git times out. 171 OSError: If git cannot be run or `path` does not exist. 172 """ 173 merge_base = get_merge_base(path, base_ref) 174 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 175 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 176 # file in or out of a directory shows up as a change on both sides. 177 changed = _git( 178 path, 179 "diff", 180 "--name-only", 181 "-z", 182 "--no-renames", 183 "--relative", 184 merge_base, 185 "--", 186 ).split("\0") 187 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 188 "\0" 189 ) 190 return sorted({file_path for file_path in changed + untracked if file_path})
List the files under a directory that changed since a base git reference.
Compares the working tree, untracked files included, with the merge base of
base_ref and HEAD, so committed, uncommitted and untracked changes all
count. Unlike get_modified_connectors, it never falls back to a plain diff,
so a caller that skips work based on the result learns when the changes are
unknown.
Arguments:
- path: A directory inside the git checkout. Only changes under it are listed.
- base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
Sorted paths relative to
path, exactly as stored (git does not quote them). A move lists both the old and the new path.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
390def get_connectors_by_language( 391 repo_path: str | Path, 392 language: ConnectorLanguage, 393) -> set[str]: 394 """Get set of all connector IDs for a specific language. 395 396 This function reads connector directories to determine which connectors 397 use the specified language. 398 399 Args: 400 repo_path: Path to the Airbyte monorepo 401 language: Language to filter by 402 403 Returns: 404 Set of connector technical names using the specified language 405 406 Example: 407 >>> python_connectors = get_connectors_by_language( 408 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 409 ... ) 410 >>> "source-faker" in python_connectors 411 True 412 """ 413 repo_path = Path(repo_path) 414 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 415 416 if not connectors_dir.exists(): 417 raise ValueError(f"Connectors directory not found: {connectors_dir}") 418 419 language_value = language.value 420 connectors_by_language = set() 421 422 # Iterate through all connector directories 423 for connector_dir in connectors_dir.iterdir(): 424 if not connector_dir.is_dir(): 425 continue 426 427 # Determine language based on file structure 428 connector_name = connector_dir.name 429 detected_language = _detect_connector_language(connector_dir, connector_name) 430 431 if detected_language == language_value: 432 connectors_by_language.add(connector_dir.name) 433 434 return connectors_by_language
Get set of all connector IDs for a specific language.
This function reads connector directories to determine which connectors use the specified language.
Arguments:
- repo_path: Path to the Airbyte monorepo
- language: Language to filter by
Returns:
Set of connector technical names using the specified language
Example:
>>> python_connectors = get_connectors_by_language( ... "/path/to/airbyte", ConnectorLanguage.PYTHON ... ) >>> "source-faker" in python_connectors True
572@lru_cache(maxsize=128) 573def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 574 """Get set of connectors using local CDK reference instead of published version. 575 576 This function detects connectors that are configured to use a local CDK 577 checkout rather than a published CDK version. The detection method differs 578 by connector language: 579 - Python: Checks for path-based dependency in pyproject.toml 580 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 581 582 Args: 583 repo_path: Path to the Airbyte monorepo 584 585 Returns: 586 Set of connector technical names using local CDK reference 587 588 Example: 589 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 590 >>> "destination-motherduck" in local_cdk_connectors 591 True 592 """ 593 repo_path = Path(repo_path) 594 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 595 596 if not connectors_dir.exists(): 597 raise ValueError(f"Connectors directory not found: {connectors_dir}") 598 599 local_cdk_connectors = set() 600 601 # Iterate through all connector directories 602 for connector_dir in connectors_dir.iterdir(): 603 if not connector_dir.is_dir(): 604 continue 605 606 connector_name = connector_dir.name 607 detected_language = _detect_connector_language(connector_dir, connector_name) 608 609 # Check for local CDK based on language 610 if detected_language in ("python", "low-code") and _has_local_cdk_python( 611 connector_dir 612 ): 613 # Python connectors: check pyproject.toml for path-based CDK dependency 614 local_cdk_connectors.add(connector_name) 615 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 616 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 617 local_cdk_connectors.add(connector_name) 618 619 return local_cdk_connectors
Get set of connectors using local CDK reference instead of published version.
This function detects connectors that are configured to use a local CDK checkout rather than a published CDK version. The detection method differs by connector language:
- Python: Checks for path-based dependency in pyproject.toml
- Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of connector technical names using local CDK reference
Example:
>>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") >>> "destination-motherduck" in local_cdk_connectors True
193def get_file_at_merge_base( 194 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 195) -> str | None: 196 """Read a file as it was at the merge base of a base git reference and `HEAD`. 197 198 This is the base side of the diff that `get_changed_files` lists, so the two 199 can be used together to tell what changed inside a file. 200 201 Args: 202 path: A directory inside the git checkout. 203 file_name: The file's path relative to `path`. 204 base_ref: Base git reference to compare against, e.g. `origin/master`, 205 or a merge base from `get_merge_base`. 206 207 Returns: 208 The file's content, or None if it did not exist at the merge base. 209 210 Raises: 211 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 212 git checkout, or git times out. 213 OSError: If git cannot be run or `path` does not exist. 214 """ 215 merge_base = get_merge_base(path, base_ref) 216 try: 217 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 218 return _git(path, "show", f"{merge_base}:./{file_name}") 219 except subprocess.CalledProcessError: 220 return None
Read a file as it was at the merge base of a base git reference and HEAD.
This is the base side of the diff that get_changed_files lists, so the two
can be used together to tell what changed inside a file.
Arguments:
- path: A directory inside the git checkout.
- file_name: The file's path relative to
path. - base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
The file's content, or None if it did not exist at the merge base.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
127def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 128 """Find the merge base of a base git reference and `HEAD`. 129 130 Resolve it once and pass it as `base_ref` to `get_changed_files` and 131 `get_file_at_merge_base` so both compare against the same commit. 132 133 Args: 134 path: A directory inside the git checkout. 135 base_ref: Base git reference, e.g. `origin/master`. 136 137 Returns: 138 The merge base commit's SHA. 139 140 Raises: 141 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 142 git checkout, or git times out. 143 OSError: If git cannot be run or `path` does not exist. 144 """ 145 return _git(path, "merge-base", base_ref, "HEAD").strip()
Find the merge base of a base git reference and HEAD.
Resolve it once and pass it as base_ref to get_changed_files and
get_file_at_merge_base so both compare against the same commit.
Arguments:
- path: A directory inside the git checkout.
- base_ref: Base git reference, e.g.
origin/master.
Returns:
The merge base commit's SHA.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
61def get_modified_connectors( 62 repo_path: str | Path, 63 base_ref: str = GIT_DEFAULT_BRANCH, 64 head_ref: str = "HEAD", 65) -> list[str]: 66 """Get list of connector IDs modified according to a local git diff. 67 68 This is the *local-only* fallback used when the GitHub API path is not 69 available (e.g. no PR number or no token). It uses `git merge-base` 70 when possible so that only changes introduced on the head branch are 71 reported, falling back to a plain two-ref diff when the merge-base 72 cannot be determined (shallow clones without enough history). 73 74 Prefer `get_modified_connectors_from_github` whenever a PR number 75 is known, as it avoids shallow-clone pitfalls entirely. 76 77 Args: 78 repo_path: Path to the Airbyte monorepo. 79 base_ref: Base git reference to compare against. 80 head_ref: Head git reference to compare. 81 82 Returns: 83 Sorted list of connector technical names. 84 """ 85 repo_path = Path(repo_path) 86 87 # Try to find the merge-base so the diff only contains changes introduced 88 # on the head branch, avoiding false positives from unrelated base-branch 89 # commits. Falls back to a plain two-ref diff when unavailable. 90 effective_base = base_ref 91 try: 92 mb_result = subprocess.run( 93 ["git", "merge-base", base_ref, head_ref], 94 cwd=repo_path, 95 capture_output=True, 96 text=True, 97 check=True, 98 timeout=30, 99 ) 100 effective_base = mb_result.stdout.strip() 101 logger.debug("Using merge-base %s for diff", effective_base) 102 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 103 logger.debug( 104 "Could not determine merge-base for %s..%s; falling back to plain diff", 105 base_ref, 106 head_ref, 107 ) 108 109 try: 110 result = subprocess.run( 111 ["git", "diff", "--name-only", effective_base, head_ref], 112 cwd=repo_path, 113 capture_output=True, 114 text=True, 115 check=True, 116 timeout=30, 117 ) 118 changed_files = result.stdout.strip().split("\n") 119 except subprocess.CalledProcessError as e: 120 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 121 except subprocess.TimeoutExpired as e: 122 raise RuntimeError("Git diff command timed out after 30 seconds") from e 123 124 return _extract_connector_names(changed_files)
Get list of connector IDs modified according to a local git diff.
This is the local-only fallback used when the GitHub API path is not
available (e.g. no PR number or no token). It uses git merge-base
when possible so that only changes introduced on the head branch are
reported, falling back to a plain two-ref diff when the merge-base
cannot be determined (shallow clones without enough history).
Prefer get_modified_connectors_from_github whenever a PR number
is known, as it avoids shallow-clone pitfalls entirely.
Arguments:
- repo_path: Path to the Airbyte monorepo.
- base_ref: Base git reference to compare against.
- head_ref: Head git reference to compare.
Returns:
Sorted list of connector technical names.
687def list_connectors( 688 repo_path: str | Path, 689 certified: bool | None = None, 690 modified: bool | None = None, 691 language_filter: set[str] | None = None, 692 language_exclude: set[str] | None = None, 693 connector_type: str | None = None, 694 connector_subtype: str | None = None, 695 base_ref: str | None = None, 696 head_ref: str | None = None, 697 pr_number: int | None = None, 698 pr_owner: str | None = None, 699 pr_repo: str | None = None, 700 gh_token: str | None = None, 701) -> ConnectorListResult: 702 """List connectors in the Airbyte monorepo with flexible filtering. 703 704 This is the core capability function that encapsulates all filtering logic. 705 Both CLI and MCP layers should use this function. 706 707 When `modified` filtering is requested and both `gh_token` and full PR 708 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 709 GitHub API is used to determine changed files — this is the most reliable 710 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 711 when the token is not provided or PR context is incomplete. 712 713 Args: 714 repo_path: Path to the Airbyte monorepo 715 certified: Filter by certification status (True=certified only, 716 False=non-certified only, None=all) 717 modified: Filter by modification status (True=modified only, 718 False=not-modified only, None=all) 719 language_filter: Set of languages to include (python, java, low-code, manifest-only) 720 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 721 connector_type: Filter by connector type (source, destination, or None for all) 722 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 723 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 724 head_ref: Head git reference for modification detection (default: "HEAD") 725 pr_number: Pull request number (enables GitHub API path for modification detection) 726 pr_owner: Repository owner for PR (e.g. "airbytehq") 727 pr_repo: Repository name for PR (e.g. "airbyte") 728 gh_token: GitHub API token. When provided together with PR context, 729 the GitHub API is used instead of local git diff. 730 731 Returns: 732 ConnectorListResult with sorted list of connector names and count 733 734 Raises: 735 ValueError: If both language_filter and language_exclude are provided 736 """ 737 # Validate mutual exclusivity of language filters 738 if language_filter is not None and language_exclude is not None: 739 raise ValueError( 740 "Cannot specify both language_filter and language_exclude. " 741 "Only one language filter parameter is accepted." 742 ) 743 744 # Start with all connectors 745 result = get_all_connectors(repo_path) 746 747 # Apply certified filter 748 if certified is not None: 749 certified_set = get_certified_connectors(repo_path) 750 if certified: 751 # Include only certified 752 result &= certified_set 753 else: 754 # Include only non-certified 755 result -= certified_set 756 757 # Apply modified filter 758 if modified is not None: 759 changed_set: set[str] | None = None 760 761 # Use GitHub API when token and full PR context are explicitly provided 762 if ( 763 gh_token is not None 764 and pr_number is not None 765 and pr_owner is not None 766 and pr_repo is not None 767 ): 768 try: 769 changed_set = set( 770 get_modified_connectors_from_github( 771 pr_number, pr_owner, pr_repo, gh_token=gh_token 772 ) 773 ) 774 except (GitHubAPIError, requests.RequestException): 775 logger.warning( 776 "GitHub API call failed for PR %s/%s#%d; " 777 "falling back to local git diff", 778 pr_owner, 779 pr_repo, 780 pr_number, 781 exc_info=True, 782 ) 783 784 # Fallback to local git diff 785 if changed_set is None: 786 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 787 head = head_ref if head_ref is not None else "HEAD" 788 changed_set = set(get_modified_connectors(repo_path, base, head)) 789 790 if modified: 791 # Include only modified 792 result &= changed_set 793 else: 794 # Include only not-modified 795 result -= changed_set 796 797 # Apply language include filter 798 if language_filter: 799 # Get connectors for all specified languages and union them 800 lang_result: set[str] = set() 801 for lang in language_filter: 802 lang_result |= get_connectors_by_language( 803 repo_path, ConnectorLanguage(lang) 804 ) 805 result &= lang_result 806 807 # Apply language exclude filter 808 if language_exclude: 809 # Get connectors for all specified languages and exclude them 810 for lang in language_exclude: 811 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 812 result -= excluded 813 814 # Apply connector type filter 815 if connector_type is not None: 816 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 817 result &= type_set 818 819 # Apply connector subtype filter 820 if connector_subtype is not None: 821 subtype_set = get_connectors_by_subtype( 822 repo_path, ConnectorSubtype(connector_subtype) 823 ) 824 result &= subtype_set 825 826 return ConnectorListResult( 827 connectors=sorted(result), 828 count=len(result), 829 )
List connectors in the Airbyte monorepo with flexible filtering.
This is the core capability function that encapsulates all filtering logic. Both CLI and MCP layers should use this function.
When modified filtering is requested and both gh_token and full PR
context (pr_number, pr_owner, pr_repo) are provided, the
GitHub API is used to determine changed files — this is the most reliable
method and avoids shallow-clone pitfalls. Falls back to local git diff
when the token is not provided or PR context is incomplete.
Arguments:
- repo_path: Path to the Airbyte monorepo
- certified: Filter by certification status (True=certified only, False=non-certified only, None=all)
- modified: Filter by modification status (True=modified only, False=not-modified only, None=all)
- language_filter: Set of languages to include (python, java, low-code, manifest-only)
- language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
- connector_type: Filter by connector type (source, destination, or None for all)
- connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
- base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
- head_ref: Head git reference for modification detection (default: "HEAD")
- pr_number: Pull request number (enables GitHub API path for modification detection)
- pr_owner: Repository owner for PR (e.g. "airbytehq")
- pr_repo: Repository name for PR (e.g. "airbyte")
- gh_token: GitHub API token. When provided together with PR context, the GitHub API is used instead of local git diff.
Returns:
ConnectorListResult with sorted list of connector names and count
Raises:
- ValueError: If both language_filter and language_exclude are provided
15def parse_pr_info(pr_num_or_url: str) -> tuple[int | None, str | None, str | None]: 16 """Parse PR number, owner, and repo from string or URL. 17 18 Args: 19 pr_num_or_url: PR number (e.g., "123") or URL (e.g., "https://github.com/airbytehq/airbyte/pull/123") 20 21 Returns: 22 Tuple of (pr_number, pr_owner, pr_repo) 23 """ 24 # If it's all digits, just return PR number 25 if pr_num_or_url.isdigit(): 26 return int(pr_num_or_url), None, None 27 28 # Parse URL: https://github.com/airbytehq/airbyte-enterprise/pull/123 29 m = re.match(r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)", pr_num_or_url) 30 if m: 31 pr_owner = m.group(1) 32 pr_repo = m.group(2) 33 pr_number = int(m.group(3)) 34 return pr_number, pr_owner, pr_repo 35 36 return None, None, None
Parse PR number, owner, and repo from string or URL.
Arguments:
- pr_num_or_url: PR number (e.g., "123") or URL (e.g., "https://github.com/airbytehq/airbyte/pull/123")
Returns:
Tuple of (pr_number, pr_owner, pr_repo)
62def resolve_diff_range( 63 pr_num_or_url: str | None, 64) -> tuple[str, str, int | None, str | None, str | None]: 65 """Resolve PR info to base_ref and head_ref for git diff. 66 67 Args: 68 pr_num_or_url: PR number, URL, or None to auto-detect from environment 69 70 Returns: 71 Tuple of (base_ref, head_ref, pr_number, pr_owner, pr_repo) 72 """ 73 pr_number = None 74 pr_owner = None 75 pr_repo = None 76 77 # Try to get PR info from parameter or environment 78 if pr_num_or_url: 79 pr_number, pr_owner, pr_repo = parse_pr_info(pr_num_or_url) 80 else: 81 pr_number, pr_owner, pr_repo = detect_env_pr_info() 82 83 # Backfill owner/repo from GITHUB_REPOSITORY when only PR number is known 84 # (e.g. --pr 75600 without a full URL, running inside GitHub Actions) 85 if pr_number is not None and (pr_owner is None or pr_repo is None): 86 _env_number, env_owner, env_repo = detect_env_pr_info() 87 if env_owner and env_repo: 88 pr_owner = pr_owner or env_owner 89 pr_repo = pr_repo or env_repo 90 91 # Determine base_ref and head_ref based on PR detection 92 if pr_number is not None: 93 # PR detected - use origin/{base_branch} vs HEAD (assumes CI checked out the PR) 94 # Use GITHUB_BASE_REF if available (set by GitHub Actions for PRs) 95 # This handles repos with different default branches (main, master, etc.) 96 base_branch = os.getenv("GITHUB_BASE_REF") or "master" 97 base_ref = f"origin/{base_branch}" 98 head_ref = "HEAD" 99 else: 100 # No PR detected - fallback to HEAD~1 vs HEAD (post-merge use case) 101 base_ref = "HEAD~1" 102 head_ref = "HEAD" 103 104 return base_ref, head_ref, pr_number, pr_owner, pr_repo
Resolve PR info to base_ref and head_ref for git diff.
Arguments:
- pr_num_or_url: PR number, URL, or None to auto-detect from environment
Returns:
Tuple of (base_ref, head_ref, pr_number, pr_owner, pr_repo)
165def strip_prerelease_suffix(version_string: str) -> str: 166 """Strip any pre-release suffix from a version string. 167 168 Returns the base version (major.minor.patch) without any pre-release 169 or build metadata suffixes. This is useful when transforming a version 170 from one pre-release state to another (e.g., from -rc.1 to -preview.{sha}). 171 172 Args: 173 version_string: A semver version string, possibly with a pre-release suffix 174 (e.g., "2.23.16-rc.1", "1.2.3-preview.abc1234", "1.0.0") 175 176 Returns: 177 The base version without any pre-release suffix (e.g., "2.23.16", "1.2.3", "1.0.0") 178 179 Raises: 180 InvalidVersionError: If the version string is not valid semver 181 """ 182 try: 183 version = semver.Version.parse(version_string) 184 except ValueError as e: 185 raise InvalidVersionError(f"Cannot parse version: {version_string}") from e 186 187 return str(version.finalize_version())
Strip any pre-release suffix from a version string.
Returns the base version (major.minor.patch) without any pre-release or build metadata suffixes. This is useful when transforming a version from one pre-release state to another (e.g., from -rc.1 to -preview.{sha}).
Arguments:
- version_string: A semver version string, possibly with a pre-release suffix (e.g., "2.23.16-rc.1", "1.2.3-preview.abc1234", "1.0.0")
Returns:
The base version without any pre-release suffix (e.g., "2.23.16", "1.2.3", "1.0.0")
Raises:
- InvalidVersionError: If the version string is not valid semver