airbyte_ops_mcp.airbyte_repo
Airbyte repository operations.
This package provides tools for working with the Airbyte monorepo, including:
- Detecting changed connectors
- Filtering connectors by certification status
- Format checking and fixing
- CI matrix planning
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Airbyte repository operations. 3 4This package provides tools for working with the Airbyte monorepo, including: 5- Detecting changed connectors 6- Filtering connectors by certification status 7- Format checking and fixing 8- CI matrix planning 9""" 10 11from __future__ import annotations 12 13from airbyte_ops_mcp.airbyte_repo.bump_version import ( 14 BumpType, 15 ChangelogEntry, 16 ChangelogParsingError, 17 ConnectorNotFoundError, 18 ConnectorVersionError, 19 InvalidVersionError, 20 VersionBumpResult, 21 VersionNotFoundError, 22 bump_connector_version, 23 strip_prerelease_suffix, 24) 25from airbyte_ops_mcp.airbyte_repo.list_connectors import ( 26 ConnectorLanguage, 27 ConnectorListResult, 28 get_all_connectors, 29 get_certified_connectors, 30 get_changed_files, 31 get_connectors_by_language, 32 get_connectors_with_local_cdk, 33 get_file_at_merge_base, 34 get_merge_base, 35 get_modified_connectors, 36 list_connectors, 37) 38from airbyte_ops_mcp.airbyte_repo.utils import ( 39 detect_env_pr_info, 40 parse_pr_info, 41 resolve_diff_range, 42) 43 44__all__ = [ 45 "BumpType", 46 "ChangelogEntry", 47 "ChangelogParsingError", 48 "ConnectorLanguage", 49 "ConnectorListResult", 50 "ConnectorNotFoundError", 51 "ConnectorVersionError", 52 "InvalidVersionError", 53 "VersionBumpResult", 54 "VersionNotFoundError", 55 "bump_connector_version", 56 "detect_env_pr_info", 57 "get_all_connectors", 58 "get_certified_connectors", 59 "get_changed_files", 60 "get_connectors_by_language", 61 "get_connectors_with_local_cdk", 62 "get_file_at_merge_base", 63 "get_merge_base", 64 "get_modified_connectors", 65 "list_connectors", 66 "parse_pr_info", 67 "resolve_diff_range", 68 "strip_prerelease_suffix", 69]
41class BumpType(StrEnum): 42 """Supported version bump types.""" 43 44 PATCH = "patch" 45 MINOR = "minor" 46 MAJOR = "major" 47 PATCH_RC = "patch_rc" 48 MINOR_RC = "minor_rc" 49 MAJOR_RC = "major_rc" 50 RC = "rc" 51 PROMOTE = "promote"
Supported version bump types.
95@dataclass(frozen=True) 96class ChangelogEntry: 97 """A single changelog entry.""" 98 99 date: datetime.date 100 version: semver.Version 101 pr_number: int | str 102 comment: str 103 104 def to_markdown(self, github_repo: str = AIRBYTE_GITHUB_REPO) -> str: 105 """Convert entry to markdown table row.""" 106 return ( 107 f"| {self.version} | {self.date.strftime('%Y-%m-%d')} | " 108 f"[{self.pr_number}](https://github.com/{github_repo}/pull/{self.pr_number}) | " 109 f"{self.comment} |" 110 )
A single changelog entry.
104 def to_markdown(self, github_repo: str = AIRBYTE_GITHUB_REPO) -> str: 105 """Convert entry to markdown table row.""" 106 return ( 107 f"| {self.version} | {self.date.strftime('%Y-%m-%d')} | " 108 f"[{self.pr_number}](https://github.com/{github_repo}/pull/{self.pr_number}) | " 109 f"{self.comment} |" 110 )
Convert entry to markdown table row.
78class ChangelogParsingError(ConnectorVersionError): 79 """Raised when changelog cannot be parsed.""" 80 81 pass
Raised when changelog cannot be parsed.
88class ConnectorLanguage(StrEnum): 89 """Connector implementation languages.""" 90 91 PYTHON = "python" 92 JAVA = "java" 93 LOW_CODE = "low-code" 94 MANIFEST_ONLY = "manifest-only" 95 96 @classmethod 97 def parse(cls, value: str) -> ConnectorLanguage: 98 """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch.""" 99 try: 100 return cls(value) 101 except ValueError: 102 valid = ", ".join(f"`{m.value}`" for m in cls) 103 raise ValueError( 104 f"Unrecognized language: {value!r}. Expected one of: {valid}." 105 ) from None
Connector implementation languages.
96 @classmethod 97 def parse(cls, value: str) -> ConnectorLanguage: 98 """Parse a string into a `ConnectorLanguage`, raising `ValueError` on mismatch.""" 99 try: 100 return cls(value) 101 except ValueError: 102 valid = ", ".join(f"`{m.value}`" for m in cls) 103 raise ValueError( 104 f"Unrecognized language: {value!r}. Expected one of: {valid}." 105 ) from None
Parse a string into a ConnectorLanguage, raising ValueError on mismatch.
704@dataclass 705class ConnectorListResult: 706 """Result of listing connectors with filters.""" 707 708 connectors: list[str] 709 count: int
Result of listing connectors with filters.
60class ConnectorNotFoundError(ConnectorVersionError): 61 """Raised when a connector is not found.""" 62 63 pass
Raised when a connector is not found.
54class ConnectorVersionError(Exception): 55 """Base exception for connector version operations.""" 56 57 pass
Base exception for connector version operations.
72class InvalidVersionError(ConnectorVersionError): 73 """Raised when a version string is invalid.""" 74 75 pass
Raised when a version string is invalid.
84@dataclass 85class VersionBumpResult: 86 """Result of a version bump operation.""" 87 88 connector: str 89 previous_version: str 90 new_version: str 91 files_modified: list[str] 92 dry_run: bool
Result of a version bump operation.
66class VersionNotFoundError(ConnectorVersionError): 67 """Raised when version cannot be found in a file.""" 68 69 pass
Raised when version cannot be found in a file.
685def bump_connector_version( 686 repo_path: str | Path, 687 connector_name: str, 688 bump_type: Literal[ 689 "patch", "minor", "major", "patch_rc", "minor_rc", "major_rc", "rc", "promote" 690 ] 691 | None = None, 692 new_version: str | None = None, 693 changelog_message: str | None = None, 694 pr_number: int | str | None = None, 695 dry_run: bool = False, 696 no_changelog: bool = False, 697 progressive_rollout_enabled: bool | None = None, 698) -> VersionBumpResult: 699 """Bump a connector's version across all relevant files. 700 701 This function updates the version in: 702 - metadata.yaml (always) 703 - pyproject.toml (if exists, for Python connectors) 704 - Documentation changelog (if changelog_message provided, doc exists, and 705 no_changelog is False) 706 707 For RC bump types (`patch_rc`, `minor_rc`, `major_rc`, `rc` from a 708 stable version), this also sets `enableProgressiveRollout: true` in 709 metadata so the registry compile step excludes this version from 710 default/latest evaluation while the rollout is in progress. 711 712 For `promote`, this clears `enableProgressiveRollout` to false so the 713 version becomes eligible for default/latest again — unless the connector's 714 `defaultRolloutMode` is `autopilot`, in which case the flag is left `true` 715 (autopilot connectors keep progressive rollout on as their default). 716 717 Args: 718 repo_path: Path to the Airbyte monorepo 719 connector_name: Technical name of the connector (e.g., "source-github") 720 bump_type: Type of version bump (patch, minor, major, patch_rc, minor_rc, major_rc, rc, promote) 721 new_version: Explicit new version (overrides bump_type) 722 changelog_message: Message to add to changelog (optional) 723 pr_number: PR number for changelog entry (optional) 724 dry_run: If True, don't actually modify files 725 no_changelog: If True, skip changelog updates even if changelog_message 726 is provided. Useful for ephemeral version bumps (e.g. pre-release 727 artifact generation) where the version is temporary. 728 progressive_rollout_enabled: Explicit override for the 729 `enableProgressiveRollout` flag in metadata.yaml. When `True` or 730 `False`, the flag is set to that value (without touching registry 731 overrides). When `None` (the default), the automatic behaviour 732 based on `bump_type` is used instead. 733 734 Returns: 735 VersionBumpResult with details of the operation 736 737 Raises: 738 ConnectorNotFoundError: If connector doesn't exist 739 VersionNotFoundError: If current version cannot be found 740 InvalidVersionError: If version format is invalid 741 ValueError: If neither bump_type nor new_version is provided 742 """ 743 repo_path = Path(repo_path) 744 connector_path = get_connector_path(repo_path, connector_name) 745 746 # Get current version 747 current_version = get_current_version(connector_path) 748 749 # Calculate new version 750 bump_type_enum = BumpType(bump_type) if bump_type else None 751 calculated_version = calculate_new_version( 752 current_version, bump_type_enum, new_version 753 ) 754 755 if ( 756 changelog_message 757 and not no_changelog 758 and not semver.Version.is_valid(calculated_version) 759 ): 760 raise InvalidVersionError( 761 f"Changelog updates are not supported for non-semver pre-release " 762 f"version {calculated_version}; pass --no-changelog." 763 ) 764 765 files_modified: list[str] = [] 766 metadata_file = connector_path / METADATA_FILE_NAME 767 if not metadata_file.exists(): 768 raise ConnectorNotFoundError(f"metadata.yaml not found at {metadata_file}") 769 metadata_file_rel = str(metadata_file.relative_to(repo_path)) 770 771 # Update metadata.yaml 772 if update_metadata_version(connector_path, calculated_version, dry_run): 773 files_modified.append(metadata_file_rel) 774 775 # Update pyproject.toml if it exists 776 if update_pyproject_version(connector_path, calculated_version, dry_run): 777 files_modified.append( 778 f"{CONNECTOR_PATH_PREFIX}/{connector_name}/{PYPROJECT_FILE_NAME}" 779 ) 780 781 # Update changelog if message provided and not explicitly skipped 782 if changelog_message and not no_changelog: 783 doc_path = get_connector_doc_path(repo_path, connector_name) 784 if doc_path and update_changelog( 785 doc_path, 786 calculated_version, 787 changelog_message, 788 pr_number, 789 dry_run, 790 ): 791 files_modified.append(str(doc_path.relative_to(repo_path))) 792 793 # Progressive rollout handling 794 795 # Automatic behaviour based on bump_type. 796 # Configure progressive rollout metadata for initial RC bumps 797 # (skip if current version is already an RC — flag is already set) 798 is_current_rc = semver.Version.is_valid(current_version) and _is_rc( 799 semver.Version.parse(current_version) 800 ) 801 if ( 802 bump_type_enum in _RC_BUMP_TYPES 803 and not is_current_rc 804 and _setup_progressive_rollout_metadata( 805 connector_path=connector_path, 806 dry_run=dry_run, 807 ) 808 and metadata_file_rel not in files_modified 809 ): 810 files_modified.append(metadata_file_rel) 811 812 # Clean up progressive rollout metadata on promote (RC → GA) 813 if ( 814 bump_type_enum == BumpType.PROMOTE 815 and _cleanup_progressive_rollout_metadata( 816 connector_path=connector_path, dry_run=dry_run 817 ) 818 and metadata_file_rel not in files_modified 819 ): 820 files_modified.append(metadata_file_rel) 821 822 # Explicit override — applied last so it takes precedence over 823 # the automatic steps above (e.g. preview builds can force the 824 # flag off even when bump_type would normally enable it). 825 if progressive_rollout_enabled is not None: 826 metadata = load_metadata_yaml(metadata_file) 827 data = metadata.get("data", {}) 828 if set_progressive_rollout_flag(data, progressive_rollout_enabled): 829 metadata["data"] = data 830 if not dry_run: 831 write_metadata_yaml(metadata, metadata_file) 832 if metadata_file_rel not in files_modified: 833 files_modified.append(metadata_file_rel) 834 835 return VersionBumpResult( 836 connector=connector_name, 837 previous_version=current_version, 838 new_version=calculated_version, 839 files_modified=files_modified, 840 dry_run=dry_run, 841 )
Bump a connector's version across all relevant files.
This function updates the version in:
- metadata.yaml (always)
- pyproject.toml (if exists, for Python connectors)
- Documentation changelog (if changelog_message provided, doc exists, and no_changelog is False)
For RC bump types (patch_rc, minor_rc, major_rc, rc from a
stable version), this also sets enableProgressiveRollout: true in
metadata so the registry compile step excludes this version from
default/latest evaluation while the rollout is in progress.
For promote, this clears enableProgressiveRollout to false so the
version becomes eligible for default/latest again — unless the connector's
defaultRolloutMode is autopilot, in which case the flag is left true
(autopilot connectors keep progressive rollout on as their default).
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_name: Technical name of the connector (e.g., "source-github")
- bump_type: Type of version bump (patch, minor, major, patch_rc, minor_rc, major_rc, rc, promote)
- new_version: Explicit new version (overrides bump_type)
- changelog_message: Message to add to changelog (optional)
- pr_number: PR number for changelog entry (optional)
- dry_run: If True, don't actually modify files
- no_changelog: If True, skip changelog updates even if changelog_message is provided. Useful for ephemeral version bumps (e.g. pre-release artifact generation) where the version is temporary.
- progressive_rollout_enabled: Explicit override for the
enableProgressiveRolloutflag in metadata.yaml. WhenTrueorFalse, the flag is set to that value (without touching registry overrides). WhenNone(the default), the automatic behaviour based onbump_typeis used instead.
Returns:
VersionBumpResult with details of the operation
Raises:
- ConnectorNotFoundError: If connector doesn't exist
- VersionNotFoundError: If current version cannot be found
- InvalidVersionError: If version format is invalid
- ValueError: If neither bump_type nor new_version is provided
39def detect_env_pr_info() -> tuple[int | None, str | None, str | None]: 40 """Detect PR number, owner, and repo from GitHub Actions environment variables. 41 42 Returns: 43 Tuple of (pr_number, pr_owner, pr_repo) 44 """ 45 # Try to extract PR number from GITHUB_REF (e.g., "refs/pull/123/merge") 46 github_ref = os.getenv("GITHUB_REF", "") 47 m = re.match(r"refs/pull/(\d+)/", github_ref) 48 pr_number = int(m.group(1)) if m else None 49 50 # Try to extract owner and repo from GITHUB_REPOSITORY (e.g., "airbytehq/airbyte") 51 pr_owner = None 52 pr_repo = None 53 github_repo = os.getenv("GITHUB_REPOSITORY", "") 54 if github_repo and "/" in github_repo: 55 parts = github_repo.split("/", 1) 56 pr_owner = parts[0] 57 pr_repo = parts[1] 58 59 return pr_number, pr_owner, pr_repo
Detect PR number, owner, and repo from GitHub Actions environment variables.
Returns:
Tuple of (pr_number, pr_owner, pr_repo)
359def get_all_connectors(repo_path: str | Path) -> set[str]: 360 """Get set of all connector IDs in the repository. 361 362 Args: 363 repo_path: Path to the Airbyte monorepo 364 365 Returns: 366 Set of all connector technical names 367 368 Example: 369 >>> all_connectors = get_all_connectors("/path/to/airbyte") 370 >>> "source-faker" in all_connectors 371 True 372 """ 373 repo_path = Path(repo_path) 374 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 375 376 if not connectors_dir.exists(): 377 raise ValueError(f"Connectors directory not found: {connectors_dir}") 378 379 return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
Get set of all connector IDs in the repository.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of all connector technical names
Example:
>>> all_connectors = get_all_connectors("/path/to/airbyte") >>> "source-faker" in all_connectors True
307@lru_cache(maxsize=128) 308def get_certified_connectors(repo_path: str | Path) -> set[str]: 309 """Get set of all certified connector IDs from metadata. 310 311 This function reads the metadata.yaml file for each connector to determine 312 which connectors have "certified" support level. 313 314 Args: 315 repo_path: Path to the Airbyte monorepo 316 317 Returns: 318 Set of certified connector technical names 319 320 Example: 321 >>> certified = get_certified_connectors("/path/to/airbyte") 322 >>> "source-postgres" in certified 323 True 324 """ 325 repo_path = Path(repo_path) 326 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 327 328 if not connectors_dir.exists(): 329 raise ValueError(f"Connectors directory not found: {connectors_dir}") 330 331 certified_connectors = set() 332 333 # Iterate through all connector directories 334 for connector_dir in connectors_dir.iterdir(): 335 if not connector_dir.is_dir(): 336 continue 337 338 metadata_file = connector_dir / METADATA_FILE_NAME 339 if not metadata_file.exists(): 340 continue 341 342 # Read metadata to check support level 343 try: 344 import yaml 345 346 with open(metadata_file) as f: 347 metadata = yaml.safe_load(f) 348 349 support_level = metadata.get("data", {}).get("supportLevel") 350 if support_level == "certified": 351 certified_connectors.add(connector_dir.name) 352 except Exception: 353 # Skip connectors with invalid metadata 354 continue 355 356 return certified_connectors
Get set of all certified connector IDs from metadata.
This function reads the metadata.yaml file for each connector to determine which connectors have "certified" support level.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of certified connector technical names
Example:
>>> certified = get_certified_connectors("/path/to/airbyte") >>> "source-postgres" in certified True
173def get_changed_files( 174 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 175) -> list[str]: 176 """List the files under a directory that changed since a base git reference. 177 178 Compares the working tree, untracked files included, with the merge base of 179 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 180 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 181 so a caller that skips work based on the result learns when the changes are 182 unknown. 183 184 Args: 185 path: A directory inside the git checkout. Only changes under it are listed. 186 base_ref: Base git reference to compare against, e.g. `origin/master`, 187 or a merge base from `get_merge_base`. 188 189 Returns: 190 Sorted paths relative to `path`, exactly as stored (git does not quote 191 them). A move lists both the old and the new path. 192 193 Raises: 194 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 195 git checkout, or git times out. 196 OSError: If git cannot be run or `path` does not exist. 197 """ 198 merge_base = get_merge_base(path, base_ref) 199 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 200 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 201 # file in or out of a directory shows up as a change on both sides. 202 changed = _git( 203 path, 204 "diff", 205 "--name-only", 206 "-z", 207 "--no-renames", 208 "--relative", 209 merge_base, 210 "--", 211 ).split("\0") 212 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 213 "\0" 214 ) 215 return sorted({file_path for file_path in changed + untracked if file_path})
List the files under a directory that changed since a base git reference.
Compares the working tree, untracked files included, with the merge base of
base_ref and HEAD, so committed, uncommitted and untracked changes all
count. Unlike get_modified_connectors, it never falls back to a plain diff,
so a caller that skips work based on the result learns when the changes are
unknown.
Arguments:
- path: A directory inside the git checkout. Only changes under it are listed.
- base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
Sorted paths relative to
path, exactly as stored (git does not quote them). A move lists both the old and the new path.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
415def get_connectors_by_language( 416 repo_path: str | Path, 417 language: ConnectorLanguage, 418) -> set[str]: 419 """Get set of all connector IDs for a specific language. 420 421 This function reads connector directories to determine which connectors 422 use the specified language. 423 424 Args: 425 repo_path: Path to the Airbyte monorepo 426 language: Language to filter by 427 428 Returns: 429 Set of connector technical names using the specified language 430 431 Example: 432 >>> python_connectors = get_connectors_by_language( 433 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 434 ... ) 435 >>> "source-faker" in python_connectors 436 True 437 """ 438 repo_path = Path(repo_path) 439 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 440 441 if not connectors_dir.exists(): 442 raise ValueError(f"Connectors directory not found: {connectors_dir}") 443 444 language_value = language.value 445 connectors_by_language = set() 446 447 # Iterate through all connector directories 448 for connector_dir in connectors_dir.iterdir(): 449 if not connector_dir.is_dir(): 450 continue 451 452 # Determine language based on file structure 453 connector_name = connector_dir.name 454 detected_language = _detect_connector_language(connector_dir, connector_name) 455 456 if detected_language == language_value: 457 connectors_by_language.add(connector_dir.name) 458 459 return connectors_by_language
Get set of all connector IDs for a specific language.
This function reads connector directories to determine which connectors use the specified language.
Arguments:
- repo_path: Path to the Airbyte monorepo
- language: Language to filter by
Returns:
Set of connector technical names using the specified language
Example:
>>> python_connectors = get_connectors_by_language( ... "/path/to/airbyte", ConnectorLanguage.PYTHON ... ) >>> "source-faker" in python_connectors True
597@lru_cache(maxsize=128) 598def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 599 """Get set of connectors using local CDK reference instead of published version. 600 601 This function detects connectors that are configured to use a local CDK 602 checkout rather than a published CDK version. The detection method differs 603 by connector language: 604 - Python: Checks for path-based dependency in pyproject.toml 605 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 606 607 Args: 608 repo_path: Path to the Airbyte monorepo 609 610 Returns: 611 Set of connector technical names using local CDK reference 612 613 Example: 614 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 615 >>> "destination-motherduck" in local_cdk_connectors 616 True 617 """ 618 repo_path = Path(repo_path) 619 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 620 621 if not connectors_dir.exists(): 622 raise ValueError(f"Connectors directory not found: {connectors_dir}") 623 624 local_cdk_connectors = set() 625 626 # Iterate through all connector directories 627 for connector_dir in connectors_dir.iterdir(): 628 if not connector_dir.is_dir(): 629 continue 630 631 connector_name = connector_dir.name 632 detected_language = _detect_connector_language(connector_dir, connector_name) 633 634 # Check for local CDK based on language 635 if detected_language in ("python", "low-code") and _has_local_cdk_python( 636 connector_dir 637 ): 638 # Python connectors: check pyproject.toml for path-based CDK dependency 639 local_cdk_connectors.add(connector_name) 640 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 641 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 642 local_cdk_connectors.add(connector_name) 643 644 return local_cdk_connectors
Get set of connectors using local CDK reference instead of published version.
This function detects connectors that are configured to use a local CDK checkout rather than a published CDK version. The detection method differs by connector language:
- Python: Checks for path-based dependency in pyproject.toml
- Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of connector technical names using local CDK reference
Example:
>>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") >>> "destination-motherduck" in local_cdk_connectors True
218def get_file_at_merge_base( 219 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 220) -> str | None: 221 """Read a file as it was at the merge base of a base git reference and `HEAD`. 222 223 This is the base side of the diff that `get_changed_files` lists, so the two 224 can be used together to tell what changed inside a file. 225 226 Args: 227 path: A directory inside the git checkout. 228 file_name: The file's path relative to `path`. 229 base_ref: Base git reference to compare against, e.g. `origin/master`, 230 or a merge base from `get_merge_base`. 231 232 Returns: 233 The file's content, or None if it did not exist at the merge base. 234 235 Raises: 236 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 237 git checkout, or git times out. 238 OSError: If git cannot be run or `path` does not exist. 239 """ 240 merge_base = get_merge_base(path, base_ref) 241 try: 242 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 243 return _git(path, "show", f"{merge_base}:./{file_name}") 244 except subprocess.CalledProcessError: 245 return None
Read a file as it was at the merge base of a base git reference and HEAD.
This is the base side of the diff that get_changed_files lists, so the two
can be used together to tell what changed inside a file.
Arguments:
- path: A directory inside the git checkout.
- file_name: The file's path relative to
path. - base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
The file's content, or None if it did not exist at the merge base.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
152def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 153 """Find the merge base of a base git reference and `HEAD`. 154 155 Resolve it once and pass it as `base_ref` to `get_changed_files` and 156 `get_file_at_merge_base` so both compare against the same commit. 157 158 Args: 159 path: A directory inside the git checkout. 160 base_ref: Base git reference, e.g. `origin/master`. 161 162 Returns: 163 The merge base commit's SHA. 164 165 Raises: 166 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 167 git checkout, or git times out. 168 OSError: If git cannot be run or `path` does not exist. 169 """ 170 return _git(path, "merge-base", base_ref, "HEAD").strip()
Find the merge base of a base git reference and HEAD.
Resolve it once and pass it as base_ref to get_changed_files and
get_file_at_merge_base so both compare against the same commit.
Arguments:
- path: A directory inside the git checkout.
- base_ref: Base git reference, e.g.
origin/master.
Returns:
The merge base commit's SHA.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
86def get_modified_connectors( 87 repo_path: str | Path, 88 base_ref: str = GIT_DEFAULT_BRANCH, 89 head_ref: str = "HEAD", 90) -> list[str]: 91 """Get list of connector IDs modified according to a local git diff. 92 93 This is the *local-only* fallback used when the GitHub API path is not 94 available (e.g. no PR number or no token). It uses `git merge-base` 95 when possible so that only changes introduced on the head branch are 96 reported, falling back to a plain two-ref diff when the merge-base 97 cannot be determined (shallow clones without enough history). 98 99 Prefer `get_modified_connectors_from_github` whenever a PR number 100 is known, as it avoids shallow-clone pitfalls entirely. 101 102 Args: 103 repo_path: Path to the Airbyte monorepo. 104 base_ref: Base git reference to compare against. 105 head_ref: Head git reference to compare. 106 107 Returns: 108 Sorted list of connector technical names. 109 """ 110 repo_path = Path(repo_path) 111 112 # Try to find the merge-base so the diff only contains changes introduced 113 # on the head branch, avoiding false positives from unrelated base-branch 114 # commits. Falls back to a plain two-ref diff when unavailable. 115 effective_base = base_ref 116 try: 117 mb_result = subprocess.run( 118 ["git", "merge-base", base_ref, head_ref], 119 cwd=repo_path, 120 capture_output=True, 121 text=True, 122 check=True, 123 timeout=30, 124 ) 125 effective_base = mb_result.stdout.strip() 126 logger.debug("Using merge-base %s for diff", effective_base) 127 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 128 logger.debug( 129 "Could not determine merge-base for %s..%s; falling back to plain diff", 130 base_ref, 131 head_ref, 132 ) 133 134 try: 135 result = subprocess.run( 136 ["git", "diff", "--name-only", effective_base, head_ref], 137 cwd=repo_path, 138 capture_output=True, 139 text=True, 140 check=True, 141 timeout=30, 142 ) 143 changed_files = result.stdout.strip().split("\n") 144 except subprocess.CalledProcessError as e: 145 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 146 except subprocess.TimeoutExpired as e: 147 raise RuntimeError("Git diff command timed out after 30 seconds") from e 148 149 return _extract_connector_names(changed_files)
Get list of connector IDs modified according to a local git diff.
This is the local-only fallback used when the GitHub API path is not
available (e.g. no PR number or no token). It uses git merge-base
when possible so that only changes introduced on the head branch are
reported, falling back to a plain two-ref diff when the merge-base
cannot be determined (shallow clones without enough history).
Prefer get_modified_connectors_from_github whenever a PR number
is known, as it avoids shallow-clone pitfalls entirely.
Arguments:
- repo_path: Path to the Airbyte monorepo.
- base_ref: Base git reference to compare against.
- head_ref: Head git reference to compare.
Returns:
Sorted list of connector technical names.
712def list_connectors( 713 repo_path: str | Path, 714 certified: bool | None = None, 715 modified: bool | None = None, 716 language_filter: set[str] | None = None, 717 language_exclude: set[str] | None = None, 718 connector_type: str | None = None, 719 connector_subtype: str | None = None, 720 base_ref: str | None = None, 721 head_ref: str | None = None, 722 pr_number: int | None = None, 723 pr_owner: str | None = None, 724 pr_repo: str | None = None, 725 gh_token: str | None = None, 726) -> ConnectorListResult: 727 """List connectors in the Airbyte monorepo with flexible filtering. 728 729 This is the core capability function that encapsulates all filtering logic. 730 Both CLI and MCP layers should use this function. 731 732 When `modified` filtering is requested and both `gh_token` and full PR 733 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 734 GitHub API is used to determine changed files — this is the most reliable 735 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 736 when the token is not provided or PR context is incomplete. 737 738 Args: 739 repo_path: Path to the Airbyte monorepo 740 certified: Filter by certification status (True=certified only, 741 False=non-certified only, None=all) 742 modified: Filter by modification status (True=modified only, 743 False=not-modified only, None=all) 744 language_filter: Set of languages to include (python, java, low-code, manifest-only) 745 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 746 connector_type: Filter by connector type (source, destination, or None for all) 747 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 748 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 749 head_ref: Head git reference for modification detection (default: "HEAD") 750 pr_number: Pull request number (enables GitHub API path for modification detection) 751 pr_owner: Repository owner for PR (e.g. "airbytehq") 752 pr_repo: Repository name for PR (e.g. "airbyte") 753 gh_token: GitHub API token. When provided together with PR context, 754 the GitHub API is used instead of local git diff. 755 756 Returns: 757 ConnectorListResult with sorted list of connector names and count 758 759 Raises: 760 ValueError: If both language_filter and language_exclude are provided 761 """ 762 # Validate mutual exclusivity of language filters 763 if language_filter is not None and language_exclude is not None: 764 raise ValueError( 765 "Cannot specify both language_filter and language_exclude. " 766 "Only one language filter parameter is accepted." 767 ) 768 769 # Start with all connectors 770 result = get_all_connectors(repo_path) 771 772 # Apply certified filter 773 if certified is not None: 774 certified_set = get_certified_connectors(repo_path) 775 if certified: 776 # Include only certified 777 result &= certified_set 778 else: 779 # Include only non-certified 780 result -= certified_set 781 782 # Apply modified filter 783 if modified is not None: 784 changed_set: set[str] | None = None 785 786 # Use GitHub API when token and full PR context are explicitly provided 787 if ( 788 gh_token is not None 789 and pr_number is not None 790 and pr_owner is not None 791 and pr_repo is not None 792 ): 793 try: 794 changed_set = set( 795 get_modified_connectors_from_github( 796 pr_number, pr_owner, pr_repo, gh_token=gh_token 797 ) 798 ) 799 except (GitHubAPIError, requests.RequestException): 800 logger.warning( 801 "GitHub API call failed for PR %s/%s#%d; " 802 "falling back to local git diff", 803 pr_owner, 804 pr_repo, 805 pr_number, 806 exc_info=True, 807 ) 808 809 # Fallback to local git diff 810 if changed_set is None: 811 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 812 head = head_ref if head_ref is not None else "HEAD" 813 changed_set = set(get_modified_connectors(repo_path, base, head)) 814 815 if modified: 816 # Include only modified 817 result &= changed_set 818 else: 819 # Include only not-modified 820 result -= changed_set 821 822 # Apply language include filter 823 if language_filter: 824 # Get connectors for all specified languages and union them 825 lang_result: set[str] = set() 826 for lang in language_filter: 827 lang_result |= get_connectors_by_language( 828 repo_path, ConnectorLanguage(lang) 829 ) 830 result &= lang_result 831 832 # Apply language exclude filter 833 if language_exclude: 834 # Get connectors for all specified languages and exclude them 835 for lang in language_exclude: 836 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 837 result -= excluded 838 839 # Apply connector type filter 840 if connector_type is not None: 841 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 842 result &= type_set 843 844 # Apply connector subtype filter 845 if connector_subtype is not None: 846 subtype_set = get_connectors_by_subtype( 847 repo_path, ConnectorSubtype(connector_subtype) 848 ) 849 result &= subtype_set 850 851 return ConnectorListResult( 852 connectors=sorted(result), 853 count=len(result), 854 )
List connectors in the Airbyte monorepo with flexible filtering.
This is the core capability function that encapsulates all filtering logic. Both CLI and MCP layers should use this function.
When modified filtering is requested and both gh_token and full PR
context (pr_number, pr_owner, pr_repo) are provided, the
GitHub API is used to determine changed files — this is the most reliable
method and avoids shallow-clone pitfalls. Falls back to local git diff
when the token is not provided or PR context is incomplete.
Arguments:
- repo_path: Path to the Airbyte monorepo
- certified: Filter by certification status (True=certified only, False=non-certified only, None=all)
- modified: Filter by modification status (True=modified only, False=not-modified only, None=all)
- language_filter: Set of languages to include (python, java, low-code, manifest-only)
- language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
- connector_type: Filter by connector type (source, destination, or None for all)
- connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
- base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
- head_ref: Head git reference for modification detection (default: "HEAD")
- pr_number: Pull request number (enables GitHub API path for modification detection)
- pr_owner: Repository owner for PR (e.g. "airbytehq")
- pr_repo: Repository name for PR (e.g. "airbyte")
- gh_token: GitHub API token. When provided together with PR context, the GitHub API is used instead of local git diff.
Returns:
ConnectorListResult with sorted list of connector names and count
Raises:
- ValueError: If both language_filter and language_exclude are provided
15def parse_pr_info(pr_num_or_url: str) -> tuple[int | None, str | None, str | None]: 16 """Parse PR number, owner, and repo from string or URL. 17 18 Args: 19 pr_num_or_url: PR number (e.g., "123") or URL (e.g., "https://github.com/airbytehq/airbyte/pull/123") 20 21 Returns: 22 Tuple of (pr_number, pr_owner, pr_repo) 23 """ 24 # If it's all digits, just return PR number 25 if pr_num_or_url.isdigit(): 26 return int(pr_num_or_url), None, None 27 28 # Parse URL: https://github.com/airbytehq/airbyte-enterprise/pull/123 29 m = re.match(r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)", pr_num_or_url) 30 if m: 31 pr_owner = m.group(1) 32 pr_repo = m.group(2) 33 pr_number = int(m.group(3)) 34 return pr_number, pr_owner, pr_repo 35 36 return None, None, None
Parse PR number, owner, and repo from string or URL.
Arguments:
- pr_num_or_url: PR number (e.g., "123") or URL (e.g., "https://github.com/airbytehq/airbyte/pull/123")
Returns:
Tuple of (pr_number, pr_owner, pr_repo)
62def resolve_diff_range( 63 pr_num_or_url: str | None, 64) -> tuple[str, str, int | None, str | None, str | None]: 65 """Resolve PR info to base_ref and head_ref for git diff. 66 67 Args: 68 pr_num_or_url: PR number, URL, or None to auto-detect from environment 69 70 Returns: 71 Tuple of (base_ref, head_ref, pr_number, pr_owner, pr_repo) 72 """ 73 pr_number = None 74 pr_owner = None 75 pr_repo = None 76 77 # Try to get PR info from parameter or environment 78 if pr_num_or_url: 79 pr_number, pr_owner, pr_repo = parse_pr_info(pr_num_or_url) 80 else: 81 pr_number, pr_owner, pr_repo = detect_env_pr_info() 82 83 # Backfill owner/repo from GITHUB_REPOSITORY when only PR number is known 84 # (e.g. --pr 75600 without a full URL, running inside GitHub Actions) 85 if pr_number is not None and (pr_owner is None or pr_repo is None): 86 _env_number, env_owner, env_repo = detect_env_pr_info() 87 if env_owner and env_repo: 88 pr_owner = pr_owner or env_owner 89 pr_repo = pr_repo or env_repo 90 91 # Determine base_ref and head_ref based on PR detection 92 if pr_number is not None: 93 # PR detected - use origin/{base_branch} vs HEAD (assumes CI checked out the PR) 94 # Use GITHUB_BASE_REF if available (set by GitHub Actions for PRs) 95 # This handles repos with different default branches (main, master, etc.) 96 base_branch = os.getenv("GITHUB_BASE_REF") or "master" 97 base_ref = f"origin/{base_branch}" 98 head_ref = "HEAD" 99 else: 100 # No PR detected - fallback to HEAD~1 vs HEAD (post-merge use case) 101 base_ref = "HEAD~1" 102 head_ref = "HEAD" 103 104 return base_ref, head_ref, pr_number, pr_owner, pr_repo
Resolve PR info to base_ref and head_ref for git diff.
Arguments:
- pr_num_or_url: PR number, URL, or None to auto-detect from environment
Returns:
Tuple of (base_ref, head_ref, pr_number, pr_owner, pr_repo)
165def strip_prerelease_suffix(version_string: str) -> str: 166 """Strip any pre-release suffix from a version string. 167 168 Returns the base version (major.minor.patch) without any pre-release 169 or build metadata suffixes. This is useful when transforming a version 170 from one pre-release state to another (e.g., from -rc.1 to -preview.{sha}). 171 172 Args: 173 version_string: A semver version string, possibly with a pre-release suffix 174 (e.g., "2.23.16-rc.1", "1.2.3-preview.abc1234", "1.0.0") 175 176 Returns: 177 The base version without any pre-release suffix (e.g., "2.23.16", "1.2.3", "1.0.0") 178 179 Raises: 180 InvalidVersionError: If the version string is not valid semver 181 """ 182 try: 183 version = semver.Version.parse(version_string) 184 except ValueError as e: 185 raise InvalidVersionError(f"Cannot parse version: {version_string}") from e 186 187 return str(version.finalize_version())
Strip any pre-release suffix from a version string.
Returns the base version (major.minor.patch) without any pre-release or build metadata suffixes. This is useful when transforming a version from one pre-release state to another (e.g., from -rc.1 to -preview.{sha}).
Arguments:
- version_string: A semver version string, possibly with a pre-release suffix (e.g., "2.23.16-rc.1", "1.2.3-preview.abc1234", "1.0.0")
Returns:
The base version without any pre-release suffix (e.g., "2.23.16", "1.2.3", "1.0.0")
Raises:
- InvalidVersionError: If the version string is not valid semver