airbyte_ops_mcp.airbyte_repo.list_connectors
Detect changed connectors in the Airbyte monorepo.
This module provides functionality to detect which connectors have been modified by comparing git diffs between branches.
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Detect changed connectors in the Airbyte monorepo. 3 4This module provides functionality to detect which connectors have been modified 5by comparing git diffs between branches. 6""" 7 8from __future__ import annotations 9 10import logging 11import re 12import subprocess 13from dataclasses import dataclass 14from enum import StrEnum 15from functools import lru_cache 16from pathlib import Path 17from typing import Any 18 19import requests 20import yaml 21 22from airbyte_ops_mcp.github_api import GitHubAPIError, get_pr_changed_files 23from airbyte_ops_mcp.registry._enums import ConnectorLanguage, ConnectorType 24 25CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" 26METADATA_FILE_NAME = "metadata.yaml" 27GIT_DEFAULT_BRANCH = "origin/master" 28 29logger = logging.getLogger(__name__) 30 31 32class ConnectorSubtype(StrEnum): 33 """Connector subtypes based on data source category.""" 34 35 API = "api" 36 DATABASE = "database" 37 FILE = "file" 38 CUSTOM = "custom" 39 40 41def _extract_connector_names(changed_files: list[str]) -> list[str]: 42 """Extract unique connector names from a list of changed file paths. 43 44 Args: 45 changed_files: File paths relative to the repo root. 46 47 Returns: 48 Sorted list of connector technical names. 49 """ 50 changed_connectors: set[str] = set() 51 for file_path in changed_files: 52 if file_path.startswith(CONNECTOR_PATH_PREFIX + "/"): 53 # e.g. "airbyte-integrations/connectors/source-faker/..." -> "source-faker" 54 path_parts = file_path.split("/") 55 if len(path_parts) >= 3: 56 changed_connectors.add(path_parts[2]) 57 return sorted(changed_connectors) 58 59 60def get_modified_connectors( 61 repo_path: str | Path, 62 base_ref: str = GIT_DEFAULT_BRANCH, 63 head_ref: str = "HEAD", 64) -> list[str]: 65 """Get list of connector IDs modified according to a local git diff. 66 67 This is the *local-only* fallback used when the GitHub API path is not 68 available (e.g. no PR number or no token). It uses `git merge-base` 69 when possible so that only changes introduced on the head branch are 70 reported, falling back to a plain two-ref diff when the merge-base 71 cannot be determined (shallow clones without enough history). 72 73 Prefer `get_modified_connectors_from_github` whenever a PR number 74 is known, as it avoids shallow-clone pitfalls entirely. 75 76 Args: 77 repo_path: Path to the Airbyte monorepo. 78 base_ref: Base git reference to compare against. 79 head_ref: Head git reference to compare. 80 81 Returns: 82 Sorted list of connector technical names. 83 """ 84 repo_path = Path(repo_path) 85 86 # Try to find the merge-base so the diff only contains changes introduced 87 # on the head branch, avoiding false positives from unrelated base-branch 88 # commits. Falls back to a plain two-ref diff when unavailable. 89 effective_base = base_ref 90 try: 91 mb_result = subprocess.run( 92 ["git", "merge-base", base_ref, head_ref], 93 cwd=repo_path, 94 capture_output=True, 95 text=True, 96 check=True, 97 timeout=30, 98 ) 99 effective_base = mb_result.stdout.strip() 100 logger.debug("Using merge-base %s for diff", effective_base) 101 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 102 logger.debug( 103 "Could not determine merge-base for %s..%s; falling back to plain diff", 104 base_ref, 105 head_ref, 106 ) 107 108 try: 109 result = subprocess.run( 110 ["git", "diff", "--name-only", effective_base, head_ref], 111 cwd=repo_path, 112 capture_output=True, 113 text=True, 114 check=True, 115 timeout=30, 116 ) 117 changed_files = result.stdout.strip().split("\n") 118 except subprocess.CalledProcessError as e: 119 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 120 except subprocess.TimeoutExpired as e: 121 raise RuntimeError("Git diff command timed out after 30 seconds") from e 122 123 return _extract_connector_names(changed_files) 124 125 126def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 127 """Find the merge base of a base git reference and `HEAD`. 128 129 Resolve it once and pass it as `base_ref` to `get_changed_files` and 130 `get_file_at_merge_base` so both compare against the same commit. 131 132 Args: 133 path: A directory inside the git checkout. 134 base_ref: Base git reference, e.g. `origin/master`. 135 136 Returns: 137 The merge base commit's SHA. 138 139 Raises: 140 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 141 git checkout, or git times out. 142 OSError: If git cannot be run or `path` does not exist. 143 """ 144 return _git(path, "merge-base", base_ref, "HEAD").strip() 145 146 147def get_changed_files( 148 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 149) -> list[str]: 150 """List the files under a directory that changed since a base git reference. 151 152 Compares the working tree, untracked files included, with the merge base of 153 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 154 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 155 so a caller that skips work based on the result learns when the changes are 156 unknown. 157 158 Args: 159 path: A directory inside the git checkout. Only changes under it are listed. 160 base_ref: Base git reference to compare against, e.g. `origin/master`, 161 or a merge base from `get_merge_base`. 162 163 Returns: 164 Sorted paths relative to `path`, exactly as stored (git does not quote 165 them). A move lists both the old and the new path. 166 167 Raises: 168 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 169 git checkout, or git times out. 170 OSError: If git cannot be run or `path` does not exist. 171 """ 172 merge_base = get_merge_base(path, base_ref) 173 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 174 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 175 # file in or out of a directory shows up as a change on both sides. 176 changed = _git( 177 path, 178 "diff", 179 "--name-only", 180 "-z", 181 "--no-renames", 182 "--relative", 183 merge_base, 184 "--", 185 ).split("\0") 186 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 187 "\0" 188 ) 189 return sorted({file_path for file_path in changed + untracked if file_path}) 190 191 192def get_file_at_merge_base( 193 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 194) -> str | None: 195 """Read a file as it was at the merge base of a base git reference and `HEAD`. 196 197 This is the base side of the diff that `get_changed_files` lists, so the two 198 can be used together to tell what changed inside a file. 199 200 Args: 201 path: A directory inside the git checkout. 202 file_name: The file's path relative to `path`. 203 base_ref: Base git reference to compare against, e.g. `origin/master`, 204 or a merge base from `get_merge_base`. 205 206 Returns: 207 The file's content, or None if it did not exist at the merge base. 208 209 Raises: 210 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 211 git checkout, or git times out. 212 OSError: If git cannot be run or `path` does not exist. 213 """ 214 merge_base = get_merge_base(path, base_ref) 215 try: 216 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 217 return _git(path, "show", f"{merge_base}:./{file_name}") 218 except subprocess.CalledProcessError: 219 return None 220 221 222def _git(cwd: str | Path, *args: str) -> str: 223 """Run a git command in `cwd` and return its output, raising on failure.""" 224 return subprocess.run( 225 ["git", *args], 226 cwd=cwd, 227 capture_output=True, 228 encoding="utf-8", 229 errors="surrogateescape", 230 check=True, 231 timeout=30, 232 ).stdout 233 234 235def format_github_actions_connector_matrix( 236 connectors: list[str], 237) -> dict[str, list[str]]: 238 """Format connector names as a GitHub Actions matrix. 239 240 GitHub Actions matrix generation needs at least one value. Return an empty 241 connector placeholder when no connectors changed so callers can pair this 242 with an `if` guard on the matrix job. 243 """ 244 return {"connector": connectors or [""]} 245 246 247def get_modified_connectors_from_github( 248 pr_number: int, 249 pr_owner: str, 250 pr_repo: str, 251 gh_token: str, 252) -> list[str]: 253 """Get list of connector IDs modified in a PR via the GitHub API. 254 255 This is the preferred path when a PR number is known because it is 256 immune to shallow-clone depth issues and always returns an accurate 257 diff (only the files the PR actually changes). 258 259 Args: 260 pr_number: Pull request number. 261 pr_owner: Repository owner (e.g. `"airbytehq"`). 262 pr_repo: Repository name (e.g. `"airbyte"`). 263 gh_token: GitHub API token (must be provided explicitly). 264 265 Returns: 266 Sorted list of connector technical names. 267 268 Raises: 269 GitHubAPIError: If the API request fails. 270 """ 271 logger.info( 272 "Fetching changed files for PR %s/%s#%d via GitHub API", 273 pr_owner, 274 pr_repo, 275 pr_number, 276 ) 277 changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token) 278 return _extract_connector_names(changed_files) 279 280 281@lru_cache(maxsize=128) 282def get_certified_connectors(repo_path: str | Path) -> set[str]: 283 """Get set of all certified connector IDs from metadata. 284 285 This function reads the metadata.yaml file for each connector to determine 286 which connectors have "certified" support level. 287 288 Args: 289 repo_path: Path to the Airbyte monorepo 290 291 Returns: 292 Set of certified connector technical names 293 294 Example: 295 >>> certified = get_certified_connectors("/path/to/airbyte") 296 >>> "source-postgres" in certified 297 True 298 """ 299 repo_path = Path(repo_path) 300 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 301 302 if not connectors_dir.exists(): 303 raise ValueError(f"Connectors directory not found: {connectors_dir}") 304 305 certified_connectors = set() 306 307 # Iterate through all connector directories 308 for connector_dir in connectors_dir.iterdir(): 309 if not connector_dir.is_dir(): 310 continue 311 312 metadata_file = connector_dir / METADATA_FILE_NAME 313 if not metadata_file.exists(): 314 continue 315 316 # Read metadata to check support level 317 try: 318 import yaml 319 320 with open(metadata_file) as f: 321 metadata = yaml.safe_load(f) 322 323 support_level = metadata.get("data", {}).get("supportLevel") 324 if support_level == "certified": 325 certified_connectors.add(connector_dir.name) 326 except Exception: 327 # Skip connectors with invalid metadata 328 continue 329 330 return certified_connectors 331 332 333def get_all_connectors(repo_path: str | Path) -> set[str]: 334 """Get set of all connector IDs in the repository. 335 336 Args: 337 repo_path: Path to the Airbyte monorepo 338 339 Returns: 340 Set of all connector technical names 341 342 Example: 343 >>> all_connectors = get_all_connectors("/path/to/airbyte") 344 >>> "source-faker" in all_connectors 345 True 346 """ 347 repo_path = Path(repo_path) 348 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 349 350 if not connectors_dir.exists(): 351 raise ValueError(f"Connectors directory not found: {connectors_dir}") 352 353 return {p.name for p in connectors_dir.iterdir() if p.is_dir()} 354 355 356def get_connector_metadata( 357 repo_path: str | Path, 358 connector_name: str, 359) -> dict[str, Any] | None: 360 """Get metadata for a specific connector. 361 362 Args: 363 repo_path: Path to the Airbyte monorepo 364 connector_name: Technical name of the connector (e.g., "source-faker") 365 366 Returns: 367 The connector's metadata dict (the 'data' section), or None if not found 368 369 Example: 370 >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") 371 >>> metadata.get("supportLevel") 372 'certified' 373 """ 374 repo_path = Path(repo_path) 375 connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name 376 metadata_file = connector_dir / METADATA_FILE_NAME 377 378 if not metadata_file.exists(): 379 return None 380 381 try: 382 with open(metadata_file) as f: 383 metadata = yaml.safe_load(f) 384 return metadata.get("data", {}) 385 except Exception: 386 return None 387 388 389def get_connectors_by_language( 390 repo_path: str | Path, 391 language: ConnectorLanguage, 392) -> set[str]: 393 """Get set of all connector IDs for a specific language. 394 395 This function reads connector directories to determine which connectors 396 use the specified language. 397 398 Args: 399 repo_path: Path to the Airbyte monorepo 400 language: Language to filter by 401 402 Returns: 403 Set of connector technical names using the specified language 404 405 Example: 406 >>> python_connectors = get_connectors_by_language( 407 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 408 ... ) 409 >>> "source-faker" in python_connectors 410 True 411 """ 412 repo_path = Path(repo_path) 413 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 414 415 if not connectors_dir.exists(): 416 raise ValueError(f"Connectors directory not found: {connectors_dir}") 417 418 language_value = language.value 419 connectors_by_language = set() 420 421 # Iterate through all connector directories 422 for connector_dir in connectors_dir.iterdir(): 423 if not connector_dir.is_dir(): 424 continue 425 426 # Determine language based on file structure 427 connector_name = connector_dir.name 428 detected_language = _detect_connector_language(connector_dir, connector_name) 429 430 if detected_language == language_value: 431 connectors_by_language.add(connector_dir.name) 432 433 return connectors_by_language 434 435 436def get_connectors_by_type( 437 repo_path: str | Path, 438 connector_type: ConnectorType, 439) -> set[str]: 440 """Get set of all connector IDs for a specific type (source or destination). 441 442 This function reads connector directories to determine which connectors 443 match the specified type based on their metadata.yaml or name prefix. 444 445 Args: 446 repo_path: Path to the Airbyte monorepo 447 connector_type: Type to filter by (source or destination) 448 449 Returns: 450 Set of connector technical names matching the specified type 451 452 Example: 453 >>> source_connectors = get_connectors_by_type( 454 ... "/path/to/airbyte", ConnectorType.SOURCE 455 ... ) 456 >>> "source-postgres" in source_connectors 457 True 458 """ 459 repo_path = Path(repo_path) 460 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 461 462 if not connectors_dir.exists(): 463 raise ValueError(f"Connectors directory not found: {connectors_dir}") 464 465 type_value = connector_type.value 466 connectors_by_type = set() 467 468 for connector_dir in connectors_dir.iterdir(): 469 if not connector_dir.is_dir(): 470 continue 471 472 connector_name = connector_dir.name 473 474 # First try to get type from metadata.yaml 475 metadata = get_connector_metadata(repo_path, connector_name) 476 if metadata: 477 metadata_type = metadata.get("connectorType") 478 if metadata_type == type_value: 479 connectors_by_type.add(connector_name) 480 continue 481 482 # Fallback to name prefix detection 483 if connector_name.startswith(f"{type_value}-"): 484 connectors_by_type.add(connector_name) 485 486 return connectors_by_type 487 488 489def get_connectors_by_subtype( 490 repo_path: str | Path, 491 connector_subtype: ConnectorSubtype, 492) -> set[str]: 493 """Get set of all connector IDs for a specific subtype (api, database, file, etc.). 494 495 This function reads connector metadata.yaml files to determine which connectors 496 match the specified subtype. 497 498 Args: 499 repo_path: Path to the Airbyte monorepo 500 connector_subtype: Subtype to filter by (api, database, file, custom) 501 502 Returns: 503 Set of connector technical names matching the specified subtype 504 505 Example: 506 >>> database_connectors = get_connectors_by_subtype( 507 ... "/path/to/airbyte", ConnectorSubtype.DATABASE 508 ... ) 509 >>> "source-postgres" in database_connectors 510 True 511 """ 512 repo_path = Path(repo_path) 513 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 514 515 if not connectors_dir.exists(): 516 raise ValueError(f"Connectors directory not found: {connectors_dir}") 517 518 subtype_value = connector_subtype.value 519 connectors_by_subtype = set() 520 521 for connector_dir in connectors_dir.iterdir(): 522 if not connector_dir.is_dir(): 523 continue 524 525 connector_name = connector_dir.name 526 metadata = get_connector_metadata(repo_path, connector_name) 527 528 if metadata: 529 metadata_subtype = metadata.get("connectorSubtype") 530 if metadata_subtype == subtype_value: 531 connectors_by_subtype.add(connector_name) 532 533 return connectors_by_subtype 534 535 536@lru_cache(maxsize=1024) 537def _detect_connector_language(connector_dir: Path, connector_name: str) -> str | None: 538 """Detect the language of a connector based on its file structure. 539 540 Args: 541 connector_dir: Path to the connector directory 542 connector_name: Technical name of the connector 543 544 Returns: 545 Language string (python, java, low-code, manifest-only) or None 546 """ 547 # Check for manifest-only (manifest.yaml at root) 548 if (connector_dir / "manifest.yaml").is_file(): 549 return "manifest-only" 550 551 # Check for low-code (manifest.yaml in source directory) 552 source_dir = connector_dir / connector_name.replace("-", "_") 553 if (source_dir / "manifest.yaml").is_file(): 554 return "low-code" 555 556 # Check for Python (setup.py or pyproject.toml) 557 if (connector_dir / "setup.py").is_file() or ( 558 connector_dir / "pyproject.toml" 559 ).is_file(): 560 return "python" 561 562 # Check for Java/Kotlin (src/main/java or src/main/kotlin) 563 if (connector_dir / "src" / "main" / "java").exists() or ( 564 connector_dir / "src" / "main" / "kotlin" 565 ).exists(): 566 return "java" 567 568 return None 569 570 571@lru_cache(maxsize=128) 572def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 573 """Get set of connectors using local CDK reference instead of published version. 574 575 This function detects connectors that are configured to use a local CDK 576 checkout rather than a published CDK version. The detection method differs 577 by connector language: 578 - Python: Checks for path-based dependency in pyproject.toml 579 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 580 581 Args: 582 repo_path: Path to the Airbyte monorepo 583 584 Returns: 585 Set of connector technical names using local CDK reference 586 587 Example: 588 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 589 >>> "destination-motherduck" in local_cdk_connectors 590 True 591 """ 592 repo_path = Path(repo_path) 593 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 594 595 if not connectors_dir.exists(): 596 raise ValueError(f"Connectors directory not found: {connectors_dir}") 597 598 local_cdk_connectors = set() 599 600 # Iterate through all connector directories 601 for connector_dir in connectors_dir.iterdir(): 602 if not connector_dir.is_dir(): 603 continue 604 605 connector_name = connector_dir.name 606 detected_language = _detect_connector_language(connector_dir, connector_name) 607 608 # Check for local CDK based on language 609 if detected_language in ("python", "low-code") and _has_local_cdk_python( 610 connector_dir 611 ): 612 # Python connectors: check pyproject.toml for path-based CDK dependency 613 local_cdk_connectors.add(connector_name) 614 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 615 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 616 local_cdk_connectors.add(connector_name) 617 618 return local_cdk_connectors 619 620 621def _has_local_cdk_python(connector_dir: Path) -> bool: 622 """Check if a Python connector uses local CDK reference. 623 624 Args: 625 connector_dir: Path to the connector directory 626 627 Returns: 628 True if connector uses local CDK reference 629 """ 630 pyproject_file = connector_dir / "pyproject.toml" 631 if not pyproject_file.exists(): 632 return False 633 634 try: 635 content = pyproject_file.read_text() 636 # Look for path-based airbyte-cdk dependency 637 # Pattern: airbyte-cdk = { path = "..." } or airbyte-cdk = {path = "..."} 638 return bool(re.search(r"airbyte-cdk\s*=\s*\{[^}]*path\s*=", content)) 639 except Exception: 640 return False 641 642 643def _has_local_cdk_java(connector_dir: Path) -> bool: 644 """Check if a Java/Kotlin connector uses local CDK reference. 645 646 Mirrors the implementation from airbyte-ci/connectors/connector_ops/connector_ops/utils.py 647 648 Args: 649 connector_dir: Path to the connector directory 650 651 Returns: 652 True if connector uses local CDK reference 653 """ 654 # Check both build.gradle and build.gradle.kts 655 for build_file_name in ("build.gradle", "build.gradle.kts"): 656 build_file = connector_dir / build_file_name 657 if not build_file.exists(): 658 continue 659 660 try: 661 # Read file and strip inline comments 662 contents = "\n".join( 663 [ 664 line.split("//")[0] # Remove inline comments 665 for line in build_file.read_text().split("\n") 666 ] 667 ) 668 # Remove spaces and check for useLocalCdk=true 669 contents = contents.replace(" ", "") 670 if "useLocalCdk=true" in contents: 671 return True 672 except Exception: 673 continue 674 675 return False 676 677 678@dataclass 679class ConnectorListResult: 680 """Result of listing connectors with filters.""" 681 682 connectors: list[str] 683 count: int 684 685 686def list_connectors( 687 repo_path: str | Path, 688 certified: bool | None = None, 689 modified: bool | None = None, 690 language_filter: set[str] | None = None, 691 language_exclude: set[str] | None = None, 692 connector_type: str | None = None, 693 connector_subtype: str | None = None, 694 base_ref: str | None = None, 695 head_ref: str | None = None, 696 pr_number: int | None = None, 697 pr_owner: str | None = None, 698 pr_repo: str | None = None, 699 gh_token: str | None = None, 700) -> ConnectorListResult: 701 """List connectors in the Airbyte monorepo with flexible filtering. 702 703 This is the core capability function that encapsulates all filtering logic. 704 Both CLI and MCP layers should use this function. 705 706 When `modified` filtering is requested and both `gh_token` and full PR 707 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 708 GitHub API is used to determine changed files — this is the most reliable 709 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 710 when the token is not provided or PR context is incomplete. 711 712 Args: 713 repo_path: Path to the Airbyte monorepo 714 certified: Filter by certification status (True=certified only, 715 False=non-certified only, None=all) 716 modified: Filter by modification status (True=modified only, 717 False=not-modified only, None=all) 718 language_filter: Set of languages to include (python, java, low-code, manifest-only) 719 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 720 connector_type: Filter by connector type (source, destination, or None for all) 721 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 722 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 723 head_ref: Head git reference for modification detection (default: "HEAD") 724 pr_number: Pull request number (enables GitHub API path for modification detection) 725 pr_owner: Repository owner for PR (e.g. "airbytehq") 726 pr_repo: Repository name for PR (e.g. "airbyte") 727 gh_token: GitHub API token. When provided together with PR context, 728 the GitHub API is used instead of local git diff. 729 730 Returns: 731 ConnectorListResult with sorted list of connector names and count 732 733 Raises: 734 ValueError: If both language_filter and language_exclude are provided 735 """ 736 # Validate mutual exclusivity of language filters 737 if language_filter is not None and language_exclude is not None: 738 raise ValueError( 739 "Cannot specify both language_filter and language_exclude. " 740 "Only one language filter parameter is accepted." 741 ) 742 743 # Start with all connectors 744 result = get_all_connectors(repo_path) 745 746 # Apply certified filter 747 if certified is not None: 748 certified_set = get_certified_connectors(repo_path) 749 if certified: 750 # Include only certified 751 result &= certified_set 752 else: 753 # Include only non-certified 754 result -= certified_set 755 756 # Apply modified filter 757 if modified is not None: 758 changed_set: set[str] | None = None 759 760 # Use GitHub API when token and full PR context are explicitly provided 761 if ( 762 gh_token is not None 763 and pr_number is not None 764 and pr_owner is not None 765 and pr_repo is not None 766 ): 767 try: 768 changed_set = set( 769 get_modified_connectors_from_github( 770 pr_number, pr_owner, pr_repo, gh_token=gh_token 771 ) 772 ) 773 except (GitHubAPIError, requests.RequestException): 774 logger.warning( 775 "GitHub API call failed for PR %s/%s#%d; " 776 "falling back to local git diff", 777 pr_owner, 778 pr_repo, 779 pr_number, 780 exc_info=True, 781 ) 782 783 # Fallback to local git diff 784 if changed_set is None: 785 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 786 head = head_ref if head_ref is not None else "HEAD" 787 changed_set = set(get_modified_connectors(repo_path, base, head)) 788 789 if modified: 790 # Include only modified 791 result &= changed_set 792 else: 793 # Include only not-modified 794 result -= changed_set 795 796 # Apply language include filter 797 if language_filter: 798 # Get connectors for all specified languages and union them 799 lang_result: set[str] = set() 800 for lang in language_filter: 801 lang_result |= get_connectors_by_language( 802 repo_path, ConnectorLanguage(lang) 803 ) 804 result &= lang_result 805 806 # Apply language exclude filter 807 if language_exclude: 808 # Get connectors for all specified languages and exclude them 809 for lang in language_exclude: 810 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 811 result -= excluded 812 813 # Apply connector type filter 814 if connector_type is not None: 815 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 816 result &= type_set 817 818 # Apply connector subtype filter 819 if connector_subtype is not None: 820 subtype_set = get_connectors_by_subtype( 821 repo_path, ConnectorSubtype(connector_subtype) 822 ) 823 result &= subtype_set 824 825 return ConnectorListResult( 826 connectors=sorted(result), 827 count=len(result), 828 )
33class ConnectorSubtype(StrEnum): 34 """Connector subtypes based on data source category.""" 35 36 API = "api" 37 DATABASE = "database" 38 FILE = "file" 39 CUSTOM = "custom"
Connector subtypes based on data source category.
61def get_modified_connectors( 62 repo_path: str | Path, 63 base_ref: str = GIT_DEFAULT_BRANCH, 64 head_ref: str = "HEAD", 65) -> list[str]: 66 """Get list of connector IDs modified according to a local git diff. 67 68 This is the *local-only* fallback used when the GitHub API path is not 69 available (e.g. no PR number or no token). It uses `git merge-base` 70 when possible so that only changes introduced on the head branch are 71 reported, falling back to a plain two-ref diff when the merge-base 72 cannot be determined (shallow clones without enough history). 73 74 Prefer `get_modified_connectors_from_github` whenever a PR number 75 is known, as it avoids shallow-clone pitfalls entirely. 76 77 Args: 78 repo_path: Path to the Airbyte monorepo. 79 base_ref: Base git reference to compare against. 80 head_ref: Head git reference to compare. 81 82 Returns: 83 Sorted list of connector technical names. 84 """ 85 repo_path = Path(repo_path) 86 87 # Try to find the merge-base so the diff only contains changes introduced 88 # on the head branch, avoiding false positives from unrelated base-branch 89 # commits. Falls back to a plain two-ref diff when unavailable. 90 effective_base = base_ref 91 try: 92 mb_result = subprocess.run( 93 ["git", "merge-base", base_ref, head_ref], 94 cwd=repo_path, 95 capture_output=True, 96 text=True, 97 check=True, 98 timeout=30, 99 ) 100 effective_base = mb_result.stdout.strip() 101 logger.debug("Using merge-base %s for diff", effective_base) 102 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 103 logger.debug( 104 "Could not determine merge-base for %s..%s; falling back to plain diff", 105 base_ref, 106 head_ref, 107 ) 108 109 try: 110 result = subprocess.run( 111 ["git", "diff", "--name-only", effective_base, head_ref], 112 cwd=repo_path, 113 capture_output=True, 114 text=True, 115 check=True, 116 timeout=30, 117 ) 118 changed_files = result.stdout.strip().split("\n") 119 except subprocess.CalledProcessError as e: 120 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 121 except subprocess.TimeoutExpired as e: 122 raise RuntimeError("Git diff command timed out after 30 seconds") from e 123 124 return _extract_connector_names(changed_files)
Get list of connector IDs modified according to a local git diff.
This is the local-only fallback used when the GitHub API path is not
available (e.g. no PR number or no token). It uses git merge-base
when possible so that only changes introduced on the head branch are
reported, falling back to a plain two-ref diff when the merge-base
cannot be determined (shallow clones without enough history).
Prefer get_modified_connectors_from_github whenever a PR number
is known, as it avoids shallow-clone pitfalls entirely.
Arguments:
- repo_path: Path to the Airbyte monorepo.
- base_ref: Base git reference to compare against.
- head_ref: Head git reference to compare.
Returns:
Sorted list of connector technical names.
127def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 128 """Find the merge base of a base git reference and `HEAD`. 129 130 Resolve it once and pass it as `base_ref` to `get_changed_files` and 131 `get_file_at_merge_base` so both compare against the same commit. 132 133 Args: 134 path: A directory inside the git checkout. 135 base_ref: Base git reference, e.g. `origin/master`. 136 137 Returns: 138 The merge base commit's SHA. 139 140 Raises: 141 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 142 git checkout, or git times out. 143 OSError: If git cannot be run or `path` does not exist. 144 """ 145 return _git(path, "merge-base", base_ref, "HEAD").strip()
Find the merge base of a base git reference and HEAD.
Resolve it once and pass it as base_ref to get_changed_files and
get_file_at_merge_base so both compare against the same commit.
Arguments:
- path: A directory inside the git checkout.
- base_ref: Base git reference, e.g.
origin/master.
Returns:
The merge base commit's SHA.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
148def get_changed_files( 149 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 150) -> list[str]: 151 """List the files under a directory that changed since a base git reference. 152 153 Compares the working tree, untracked files included, with the merge base of 154 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 155 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 156 so a caller that skips work based on the result learns when the changes are 157 unknown. 158 159 Args: 160 path: A directory inside the git checkout. Only changes under it are listed. 161 base_ref: Base git reference to compare against, e.g. `origin/master`, 162 or a merge base from `get_merge_base`. 163 164 Returns: 165 Sorted paths relative to `path`, exactly as stored (git does not quote 166 them). A move lists both the old and the new path. 167 168 Raises: 169 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 170 git checkout, or git times out. 171 OSError: If git cannot be run or `path` does not exist. 172 """ 173 merge_base = get_merge_base(path, base_ref) 174 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 175 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 176 # file in or out of a directory shows up as a change on both sides. 177 changed = _git( 178 path, 179 "diff", 180 "--name-only", 181 "-z", 182 "--no-renames", 183 "--relative", 184 merge_base, 185 "--", 186 ).split("\0") 187 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 188 "\0" 189 ) 190 return sorted({file_path for file_path in changed + untracked if file_path})
List the files under a directory that changed since a base git reference.
Compares the working tree, untracked files included, with the merge base of
base_ref and HEAD, so committed, uncommitted and untracked changes all
count. Unlike get_modified_connectors, it never falls back to a plain diff,
so a caller that skips work based on the result learns when the changes are
unknown.
Arguments:
- path: A directory inside the git checkout. Only changes under it are listed.
- base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
Sorted paths relative to
path, exactly as stored (git does not quote them). A move lists both the old and the new path.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
193def get_file_at_merge_base( 194 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 195) -> str | None: 196 """Read a file as it was at the merge base of a base git reference and `HEAD`. 197 198 This is the base side of the diff that `get_changed_files` lists, so the two 199 can be used together to tell what changed inside a file. 200 201 Args: 202 path: A directory inside the git checkout. 203 file_name: The file's path relative to `path`. 204 base_ref: Base git reference to compare against, e.g. `origin/master`, 205 or a merge base from `get_merge_base`. 206 207 Returns: 208 The file's content, or None if it did not exist at the merge base. 209 210 Raises: 211 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 212 git checkout, or git times out. 213 OSError: If git cannot be run or `path` does not exist. 214 """ 215 merge_base = get_merge_base(path, base_ref) 216 try: 217 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 218 return _git(path, "show", f"{merge_base}:./{file_name}") 219 except subprocess.CalledProcessError: 220 return None
Read a file as it was at the merge base of a base git reference and HEAD.
This is the base side of the diff that get_changed_files lists, so the two
can be used together to tell what changed inside a file.
Arguments:
- path: A directory inside the git checkout.
- file_name: The file's path relative to
path. - base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
The file's content, or None if it did not exist at the merge base.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
236def format_github_actions_connector_matrix( 237 connectors: list[str], 238) -> dict[str, list[str]]: 239 """Format connector names as a GitHub Actions matrix. 240 241 GitHub Actions matrix generation needs at least one value. Return an empty 242 connector placeholder when no connectors changed so callers can pair this 243 with an `if` guard on the matrix job. 244 """ 245 return {"connector": connectors or [""]}
Format connector names as a GitHub Actions matrix.
GitHub Actions matrix generation needs at least one value. Return an empty
connector placeholder when no connectors changed so callers can pair this
with an if guard on the matrix job.
248def get_modified_connectors_from_github( 249 pr_number: int, 250 pr_owner: str, 251 pr_repo: str, 252 gh_token: str, 253) -> list[str]: 254 """Get list of connector IDs modified in a PR via the GitHub API. 255 256 This is the preferred path when a PR number is known because it is 257 immune to shallow-clone depth issues and always returns an accurate 258 diff (only the files the PR actually changes). 259 260 Args: 261 pr_number: Pull request number. 262 pr_owner: Repository owner (e.g. `"airbytehq"`). 263 pr_repo: Repository name (e.g. `"airbyte"`). 264 gh_token: GitHub API token (must be provided explicitly). 265 266 Returns: 267 Sorted list of connector technical names. 268 269 Raises: 270 GitHubAPIError: If the API request fails. 271 """ 272 logger.info( 273 "Fetching changed files for PR %s/%s#%d via GitHub API", 274 pr_owner, 275 pr_repo, 276 pr_number, 277 ) 278 changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token) 279 return _extract_connector_names(changed_files)
Get list of connector IDs modified in a PR via the GitHub API.
This is the preferred path when a PR number is known because it is immune to shallow-clone depth issues and always returns an accurate diff (only the files the PR actually changes).
Arguments:
- pr_number: Pull request number.
- pr_owner: Repository owner (e.g.
"airbytehq"). - pr_repo: Repository name (e.g.
"airbyte"). - gh_token: GitHub API token (must be provided explicitly).
Returns:
Sorted list of connector technical names.
Raises:
- GitHubAPIError: If the API request fails.
282@lru_cache(maxsize=128) 283def get_certified_connectors(repo_path: str | Path) -> set[str]: 284 """Get set of all certified connector IDs from metadata. 285 286 This function reads the metadata.yaml file for each connector to determine 287 which connectors have "certified" support level. 288 289 Args: 290 repo_path: Path to the Airbyte monorepo 291 292 Returns: 293 Set of certified connector technical names 294 295 Example: 296 >>> certified = get_certified_connectors("/path/to/airbyte") 297 >>> "source-postgres" in certified 298 True 299 """ 300 repo_path = Path(repo_path) 301 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 302 303 if not connectors_dir.exists(): 304 raise ValueError(f"Connectors directory not found: {connectors_dir}") 305 306 certified_connectors = set() 307 308 # Iterate through all connector directories 309 for connector_dir in connectors_dir.iterdir(): 310 if not connector_dir.is_dir(): 311 continue 312 313 metadata_file = connector_dir / METADATA_FILE_NAME 314 if not metadata_file.exists(): 315 continue 316 317 # Read metadata to check support level 318 try: 319 import yaml 320 321 with open(metadata_file) as f: 322 metadata = yaml.safe_load(f) 323 324 support_level = metadata.get("data", {}).get("supportLevel") 325 if support_level == "certified": 326 certified_connectors.add(connector_dir.name) 327 except Exception: 328 # Skip connectors with invalid metadata 329 continue 330 331 return certified_connectors
Get set of all certified connector IDs from metadata.
This function reads the metadata.yaml file for each connector to determine which connectors have "certified" support level.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of certified connector technical names
Example:
>>> certified = get_certified_connectors("/path/to/airbyte") >>> "source-postgres" in certified True
334def get_all_connectors(repo_path: str | Path) -> set[str]: 335 """Get set of all connector IDs in the repository. 336 337 Args: 338 repo_path: Path to the Airbyte monorepo 339 340 Returns: 341 Set of all connector technical names 342 343 Example: 344 >>> all_connectors = get_all_connectors("/path/to/airbyte") 345 >>> "source-faker" in all_connectors 346 True 347 """ 348 repo_path = Path(repo_path) 349 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 350 351 if not connectors_dir.exists(): 352 raise ValueError(f"Connectors directory not found: {connectors_dir}") 353 354 return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
Get set of all connector IDs in the repository.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of all connector technical names
Example:
>>> all_connectors = get_all_connectors("/path/to/airbyte") >>> "source-faker" in all_connectors True
357def get_connector_metadata( 358 repo_path: str | Path, 359 connector_name: str, 360) -> dict[str, Any] | None: 361 """Get metadata for a specific connector. 362 363 Args: 364 repo_path: Path to the Airbyte monorepo 365 connector_name: Technical name of the connector (e.g., "source-faker") 366 367 Returns: 368 The connector's metadata dict (the 'data' section), or None if not found 369 370 Example: 371 >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") 372 >>> metadata.get("supportLevel") 373 'certified' 374 """ 375 repo_path = Path(repo_path) 376 connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name 377 metadata_file = connector_dir / METADATA_FILE_NAME 378 379 if not metadata_file.exists(): 380 return None 381 382 try: 383 with open(metadata_file) as f: 384 metadata = yaml.safe_load(f) 385 return metadata.get("data", {}) 386 except Exception: 387 return None
Get metadata for a specific connector.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_name: Technical name of the connector (e.g., "source-faker")
Returns:
The connector's metadata dict (the 'data' section), or None if not found
Example:
>>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") >>> metadata.get("supportLevel") 'certified'
390def get_connectors_by_language( 391 repo_path: str | Path, 392 language: ConnectorLanguage, 393) -> set[str]: 394 """Get set of all connector IDs for a specific language. 395 396 This function reads connector directories to determine which connectors 397 use the specified language. 398 399 Args: 400 repo_path: Path to the Airbyte monorepo 401 language: Language to filter by 402 403 Returns: 404 Set of connector technical names using the specified language 405 406 Example: 407 >>> python_connectors = get_connectors_by_language( 408 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 409 ... ) 410 >>> "source-faker" in python_connectors 411 True 412 """ 413 repo_path = Path(repo_path) 414 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 415 416 if not connectors_dir.exists(): 417 raise ValueError(f"Connectors directory not found: {connectors_dir}") 418 419 language_value = language.value 420 connectors_by_language = set() 421 422 # Iterate through all connector directories 423 for connector_dir in connectors_dir.iterdir(): 424 if not connector_dir.is_dir(): 425 continue 426 427 # Determine language based on file structure 428 connector_name = connector_dir.name 429 detected_language = _detect_connector_language(connector_dir, connector_name) 430 431 if detected_language == language_value: 432 connectors_by_language.add(connector_dir.name) 433 434 return connectors_by_language
Get set of all connector IDs for a specific language.
This function reads connector directories to determine which connectors use the specified language.
Arguments:
- repo_path: Path to the Airbyte monorepo
- language: Language to filter by
Returns:
Set of connector technical names using the specified language
Example:
>>> python_connectors = get_connectors_by_language( ... "/path/to/airbyte", ConnectorLanguage.PYTHON ... ) >>> "source-faker" in python_connectors True
437def get_connectors_by_type( 438 repo_path: str | Path, 439 connector_type: ConnectorType, 440) -> set[str]: 441 """Get set of all connector IDs for a specific type (source or destination). 442 443 This function reads connector directories to determine which connectors 444 match the specified type based on their metadata.yaml or name prefix. 445 446 Args: 447 repo_path: Path to the Airbyte monorepo 448 connector_type: Type to filter by (source or destination) 449 450 Returns: 451 Set of connector technical names matching the specified type 452 453 Example: 454 >>> source_connectors = get_connectors_by_type( 455 ... "/path/to/airbyte", ConnectorType.SOURCE 456 ... ) 457 >>> "source-postgres" in source_connectors 458 True 459 """ 460 repo_path = Path(repo_path) 461 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 462 463 if not connectors_dir.exists(): 464 raise ValueError(f"Connectors directory not found: {connectors_dir}") 465 466 type_value = connector_type.value 467 connectors_by_type = set() 468 469 for connector_dir in connectors_dir.iterdir(): 470 if not connector_dir.is_dir(): 471 continue 472 473 connector_name = connector_dir.name 474 475 # First try to get type from metadata.yaml 476 metadata = get_connector_metadata(repo_path, connector_name) 477 if metadata: 478 metadata_type = metadata.get("connectorType") 479 if metadata_type == type_value: 480 connectors_by_type.add(connector_name) 481 continue 482 483 # Fallback to name prefix detection 484 if connector_name.startswith(f"{type_value}-"): 485 connectors_by_type.add(connector_name) 486 487 return connectors_by_type
Get set of all connector IDs for a specific type (source or destination).
This function reads connector directories to determine which connectors match the specified type based on their metadata.yaml or name prefix.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_type: Type to filter by (source or destination)
Returns:
Set of connector technical names matching the specified type
Example:
>>> source_connectors = get_connectors_by_type( ... "/path/to/airbyte", ConnectorType.SOURCE ... ) >>> "source-postgres" in source_connectors True
490def get_connectors_by_subtype( 491 repo_path: str | Path, 492 connector_subtype: ConnectorSubtype, 493) -> set[str]: 494 """Get set of all connector IDs for a specific subtype (api, database, file, etc.). 495 496 This function reads connector metadata.yaml files to determine which connectors 497 match the specified subtype. 498 499 Args: 500 repo_path: Path to the Airbyte monorepo 501 connector_subtype: Subtype to filter by (api, database, file, custom) 502 503 Returns: 504 Set of connector technical names matching the specified subtype 505 506 Example: 507 >>> database_connectors = get_connectors_by_subtype( 508 ... "/path/to/airbyte", ConnectorSubtype.DATABASE 509 ... ) 510 >>> "source-postgres" in database_connectors 511 True 512 """ 513 repo_path = Path(repo_path) 514 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 515 516 if not connectors_dir.exists(): 517 raise ValueError(f"Connectors directory not found: {connectors_dir}") 518 519 subtype_value = connector_subtype.value 520 connectors_by_subtype = set() 521 522 for connector_dir in connectors_dir.iterdir(): 523 if not connector_dir.is_dir(): 524 continue 525 526 connector_name = connector_dir.name 527 metadata = get_connector_metadata(repo_path, connector_name) 528 529 if metadata: 530 metadata_subtype = metadata.get("connectorSubtype") 531 if metadata_subtype == subtype_value: 532 connectors_by_subtype.add(connector_name) 533 534 return connectors_by_subtype
Get set of all connector IDs for a specific subtype (api, database, file, etc.).
This function reads connector metadata.yaml files to determine which connectors match the specified subtype.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_subtype: Subtype to filter by (api, database, file, custom)
Returns:
Set of connector technical names matching the specified subtype
Example:
>>> database_connectors = get_connectors_by_subtype( ... "/path/to/airbyte", ConnectorSubtype.DATABASE ... ) >>> "source-postgres" in database_connectors True
572@lru_cache(maxsize=128) 573def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 574 """Get set of connectors using local CDK reference instead of published version. 575 576 This function detects connectors that are configured to use a local CDK 577 checkout rather than a published CDK version. The detection method differs 578 by connector language: 579 - Python: Checks for path-based dependency in pyproject.toml 580 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 581 582 Args: 583 repo_path: Path to the Airbyte monorepo 584 585 Returns: 586 Set of connector technical names using local CDK reference 587 588 Example: 589 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 590 >>> "destination-motherduck" in local_cdk_connectors 591 True 592 """ 593 repo_path = Path(repo_path) 594 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 595 596 if not connectors_dir.exists(): 597 raise ValueError(f"Connectors directory not found: {connectors_dir}") 598 599 local_cdk_connectors = set() 600 601 # Iterate through all connector directories 602 for connector_dir in connectors_dir.iterdir(): 603 if not connector_dir.is_dir(): 604 continue 605 606 connector_name = connector_dir.name 607 detected_language = _detect_connector_language(connector_dir, connector_name) 608 609 # Check for local CDK based on language 610 if detected_language in ("python", "low-code") and _has_local_cdk_python( 611 connector_dir 612 ): 613 # Python connectors: check pyproject.toml for path-based CDK dependency 614 local_cdk_connectors.add(connector_name) 615 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 616 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 617 local_cdk_connectors.add(connector_name) 618 619 return local_cdk_connectors
Get set of connectors using local CDK reference instead of published version.
This function detects connectors that are configured to use a local CDK checkout rather than a published CDK version. The detection method differs by connector language:
- Python: Checks for path-based dependency in pyproject.toml
- Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of connector technical names using local CDK reference
Example:
>>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") >>> "destination-motherduck" in local_cdk_connectors True
679@dataclass 680class ConnectorListResult: 681 """Result of listing connectors with filters.""" 682 683 connectors: list[str] 684 count: int
Result of listing connectors with filters.
687def list_connectors( 688 repo_path: str | Path, 689 certified: bool | None = None, 690 modified: bool | None = None, 691 language_filter: set[str] | None = None, 692 language_exclude: set[str] | None = None, 693 connector_type: str | None = None, 694 connector_subtype: str | None = None, 695 base_ref: str | None = None, 696 head_ref: str | None = None, 697 pr_number: int | None = None, 698 pr_owner: str | None = None, 699 pr_repo: str | None = None, 700 gh_token: str | None = None, 701) -> ConnectorListResult: 702 """List connectors in the Airbyte monorepo with flexible filtering. 703 704 This is the core capability function that encapsulates all filtering logic. 705 Both CLI and MCP layers should use this function. 706 707 When `modified` filtering is requested and both `gh_token` and full PR 708 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 709 GitHub API is used to determine changed files — this is the most reliable 710 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 711 when the token is not provided or PR context is incomplete. 712 713 Args: 714 repo_path: Path to the Airbyte monorepo 715 certified: Filter by certification status (True=certified only, 716 False=non-certified only, None=all) 717 modified: Filter by modification status (True=modified only, 718 False=not-modified only, None=all) 719 language_filter: Set of languages to include (python, java, low-code, manifest-only) 720 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 721 connector_type: Filter by connector type (source, destination, or None for all) 722 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 723 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 724 head_ref: Head git reference for modification detection (default: "HEAD") 725 pr_number: Pull request number (enables GitHub API path for modification detection) 726 pr_owner: Repository owner for PR (e.g. "airbytehq") 727 pr_repo: Repository name for PR (e.g. "airbyte") 728 gh_token: GitHub API token. When provided together with PR context, 729 the GitHub API is used instead of local git diff. 730 731 Returns: 732 ConnectorListResult with sorted list of connector names and count 733 734 Raises: 735 ValueError: If both language_filter and language_exclude are provided 736 """ 737 # Validate mutual exclusivity of language filters 738 if language_filter is not None and language_exclude is not None: 739 raise ValueError( 740 "Cannot specify both language_filter and language_exclude. " 741 "Only one language filter parameter is accepted." 742 ) 743 744 # Start with all connectors 745 result = get_all_connectors(repo_path) 746 747 # Apply certified filter 748 if certified is not None: 749 certified_set = get_certified_connectors(repo_path) 750 if certified: 751 # Include only certified 752 result &= certified_set 753 else: 754 # Include only non-certified 755 result -= certified_set 756 757 # Apply modified filter 758 if modified is not None: 759 changed_set: set[str] | None = None 760 761 # Use GitHub API when token and full PR context are explicitly provided 762 if ( 763 gh_token is not None 764 and pr_number is not None 765 and pr_owner is not None 766 and pr_repo is not None 767 ): 768 try: 769 changed_set = set( 770 get_modified_connectors_from_github( 771 pr_number, pr_owner, pr_repo, gh_token=gh_token 772 ) 773 ) 774 except (GitHubAPIError, requests.RequestException): 775 logger.warning( 776 "GitHub API call failed for PR %s/%s#%d; " 777 "falling back to local git diff", 778 pr_owner, 779 pr_repo, 780 pr_number, 781 exc_info=True, 782 ) 783 784 # Fallback to local git diff 785 if changed_set is None: 786 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 787 head = head_ref if head_ref is not None else "HEAD" 788 changed_set = set(get_modified_connectors(repo_path, base, head)) 789 790 if modified: 791 # Include only modified 792 result &= changed_set 793 else: 794 # Include only not-modified 795 result -= changed_set 796 797 # Apply language include filter 798 if language_filter: 799 # Get connectors for all specified languages and union them 800 lang_result: set[str] = set() 801 for lang in language_filter: 802 lang_result |= get_connectors_by_language( 803 repo_path, ConnectorLanguage(lang) 804 ) 805 result &= lang_result 806 807 # Apply language exclude filter 808 if language_exclude: 809 # Get connectors for all specified languages and exclude them 810 for lang in language_exclude: 811 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 812 result -= excluded 813 814 # Apply connector type filter 815 if connector_type is not None: 816 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 817 result &= type_set 818 819 # Apply connector subtype filter 820 if connector_subtype is not None: 821 subtype_set = get_connectors_by_subtype( 822 repo_path, ConnectorSubtype(connector_subtype) 823 ) 824 result &= subtype_set 825 826 return ConnectorListResult( 827 connectors=sorted(result), 828 count=len(result), 829 )
List connectors in the Airbyte monorepo with flexible filtering.
This is the core capability function that encapsulates all filtering logic. Both CLI and MCP layers should use this function.
When modified filtering is requested and both gh_token and full PR
context (pr_number, pr_owner, pr_repo) are provided, the
GitHub API is used to determine changed files — this is the most reliable
method and avoids shallow-clone pitfalls. Falls back to local git diff
when the token is not provided or PR context is incomplete.
Arguments:
- repo_path: Path to the Airbyte monorepo
- certified: Filter by certification status (True=certified only, False=non-certified only, None=all)
- modified: Filter by modification status (True=modified only, False=not-modified only, None=all)
- language_filter: Set of languages to include (python, java, low-code, manifest-only)
- language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
- connector_type: Filter by connector type (source, destination, or None for all)
- connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
- base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
- head_ref: Head git reference for modification detection (default: "HEAD")
- pr_number: Pull request number (enables GitHub API path for modification detection)
- pr_owner: Repository owner for PR (e.g. "airbytehq")
- pr_repo: Repository name for PR (e.g. "airbyte")
- gh_token: GitHub API token. When provided together with PR context, the GitHub API is used instead of local git diff.
Returns:
ConnectorListResult with sorted list of connector names and count
Raises:
- ValueError: If both language_filter and language_exclude are provided