airbyte_ops_mcp.airbyte_repo.list_connectors
Detect changed connectors in the Airbyte monorepo.
This module provides functionality to detect which connectors have been modified by comparing git diffs between branches.
1# Copyright (c) 2025 Airbyte, Inc., all rights reserved. 2"""Detect changed connectors in the Airbyte monorepo. 3 4This module provides functionality to detect which connectors have been modified 5by comparing git diffs between branches. 6""" 7 8from __future__ import annotations 9 10import logging 11import re 12import subprocess 13from dataclasses import dataclass 14from enum import StrEnum 15from functools import lru_cache 16from pathlib import Path 17from typing import Any 18 19import requests 20import yaml 21 22from airbyte_ops_mcp.github_api import GitHubAPIError, get_pr_changed_files 23from airbyte_ops_mcp.registry._enums import ConnectorLanguage, ConnectorType 24 25CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" 26# Mirrors the root-level documentation exclusions in airbytehq/airbyte's `poe-tasks/get-modified-connectors.sh`. 27IGNORED_CONNECTOR_FILE_NAMES = frozenset( 28 {"README.md", "CONTRIBUTING.md", "AGENTS.md", "CLAUDE.md"} 29) 30METADATA_FILE_NAME = "metadata.yaml" 31GIT_DEFAULT_BRANCH = "origin/master" 32 33logger = logging.getLogger(__name__) 34 35 36class ConnectorSubtype(StrEnum): 37 """Connector subtypes based on data source category.""" 38 39 API = "api" 40 DATABASE = "database" 41 FILE = "file" 42 CUSTOM = "custom" 43 44 45def _extract_connector_names(changed_files: list[str]) -> list[str]: 46 """Extract unique connector names from a list of changed file paths. 47 48 Only include directories prefixed with `source-` or `destination-`; 49 entries directly under the connectors directory that do not match are not 50 connectors. 51 Ignore root-level `README.md`, `CONTRIBUTING.md`, `AGENTS.md`, and 52 `CLAUDE.md` files inside connector directories; nested files with those 53 names still count as connector changes. 54 55 Args: 56 changed_files: File paths relative to the repo root. 57 58 Returns: 59 Sorted list of connector technical names. 60 """ 61 changed_connectors: set[str] = set() 62 for file_path in changed_files: 63 if not file_path.startswith(CONNECTOR_PATH_PREFIX + "/"): 64 continue 65 66 # e.g. "airbyte-integrations/connectors/source-faker/..." -> "source-faker" 67 path_parts = file_path.split("/") 68 if len(path_parts) < 3: 69 continue 70 71 connector_name = path_parts[2] 72 if not connector_name.startswith(("source-", "destination-")): 73 continue 74 75 is_root_doc = ( 76 len(path_parts) == 4 and path_parts[3] in IGNORED_CONNECTOR_FILE_NAMES 77 ) 78 if is_root_doc: 79 continue 80 81 changed_connectors.add(connector_name) 82 return sorted(changed_connectors) 83 84 85def get_modified_connectors( 86 repo_path: str | Path, 87 base_ref: str = GIT_DEFAULT_BRANCH, 88 head_ref: str = "HEAD", 89) -> list[str]: 90 """Get list of connector IDs modified according to a local git diff. 91 92 This is the *local-only* fallback used when the GitHub API path is not 93 available (e.g. no PR number or no token). It uses `git merge-base` 94 when possible so that only changes introduced on the head branch are 95 reported, falling back to a plain two-ref diff when the merge-base 96 cannot be determined (shallow clones without enough history). 97 98 Prefer `get_modified_connectors_from_github` whenever a PR number 99 is known, as it avoids shallow-clone pitfalls entirely. 100 101 Args: 102 repo_path: Path to the Airbyte monorepo. 103 base_ref: Base git reference to compare against. 104 head_ref: Head git reference to compare. 105 106 Returns: 107 Sorted list of connector technical names. 108 """ 109 repo_path = Path(repo_path) 110 111 # Try to find the merge-base so the diff only contains changes introduced 112 # on the head branch, avoiding false positives from unrelated base-branch 113 # commits. Falls back to a plain two-ref diff when unavailable. 114 effective_base = base_ref 115 try: 116 mb_result = subprocess.run( 117 ["git", "merge-base", base_ref, head_ref], 118 cwd=repo_path, 119 capture_output=True, 120 text=True, 121 check=True, 122 timeout=30, 123 ) 124 effective_base = mb_result.stdout.strip() 125 logger.debug("Using merge-base %s for diff", effective_base) 126 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 127 logger.debug( 128 "Could not determine merge-base for %s..%s; falling back to plain diff", 129 base_ref, 130 head_ref, 131 ) 132 133 try: 134 result = subprocess.run( 135 ["git", "diff", "--name-only", effective_base, head_ref], 136 cwd=repo_path, 137 capture_output=True, 138 text=True, 139 check=True, 140 timeout=30, 141 ) 142 changed_files = result.stdout.strip().split("\n") 143 except subprocess.CalledProcessError as e: 144 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 145 except subprocess.TimeoutExpired as e: 146 raise RuntimeError("Git diff command timed out after 30 seconds") from e 147 148 return _extract_connector_names(changed_files) 149 150 151def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 152 """Find the merge base of a base git reference and `HEAD`. 153 154 Resolve it once and pass it as `base_ref` to `get_changed_files` and 155 `get_file_at_merge_base` so both compare against the same commit. 156 157 Args: 158 path: A directory inside the git checkout. 159 base_ref: Base git reference, e.g. `origin/master`. 160 161 Returns: 162 The merge base commit's SHA. 163 164 Raises: 165 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 166 git checkout, or git times out. 167 OSError: If git cannot be run or `path` does not exist. 168 """ 169 return _git(path, "merge-base", base_ref, "HEAD").strip() 170 171 172def get_changed_files( 173 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 174) -> list[str]: 175 """List the files under a directory that changed since a base git reference. 176 177 Compares the working tree, untracked files included, with the merge base of 178 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 179 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 180 so a caller that skips work based on the result learns when the changes are 181 unknown. 182 183 Args: 184 path: A directory inside the git checkout. Only changes under it are listed. 185 base_ref: Base git reference to compare against, e.g. `origin/master`, 186 or a merge base from `get_merge_base`. 187 188 Returns: 189 Sorted paths relative to `path`, exactly as stored (git does not quote 190 them). A move lists both the old and the new path. 191 192 Raises: 193 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 194 git checkout, or git times out. 195 OSError: If git cannot be run or `path` does not exist. 196 """ 197 merge_base = get_merge_base(path, base_ref) 198 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 199 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 200 # file in or out of a directory shows up as a change on both sides. 201 changed = _git( 202 path, 203 "diff", 204 "--name-only", 205 "-z", 206 "--no-renames", 207 "--relative", 208 merge_base, 209 "--", 210 ).split("\0") 211 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 212 "\0" 213 ) 214 return sorted({file_path for file_path in changed + untracked if file_path}) 215 216 217def get_file_at_merge_base( 218 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 219) -> str | None: 220 """Read a file as it was at the merge base of a base git reference and `HEAD`. 221 222 This is the base side of the diff that `get_changed_files` lists, so the two 223 can be used together to tell what changed inside a file. 224 225 Args: 226 path: A directory inside the git checkout. 227 file_name: The file's path relative to `path`. 228 base_ref: Base git reference to compare against, e.g. `origin/master`, 229 or a merge base from `get_merge_base`. 230 231 Returns: 232 The file's content, or None if it did not exist at the merge base. 233 234 Raises: 235 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 236 git checkout, or git times out. 237 OSError: If git cannot be run or `path` does not exist. 238 """ 239 merge_base = get_merge_base(path, base_ref) 240 try: 241 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 242 return _git(path, "show", f"{merge_base}:./{file_name}") 243 except subprocess.CalledProcessError: 244 return None 245 246 247def _git(cwd: str | Path, *args: str) -> str: 248 """Run a git command in `cwd` and return its output, raising on failure.""" 249 return subprocess.run( 250 ["git", *args], 251 cwd=cwd, 252 capture_output=True, 253 encoding="utf-8", 254 errors="surrogateescape", 255 check=True, 256 timeout=30, 257 ).stdout 258 259 260def format_github_actions_connector_matrix( 261 connectors: list[str], 262) -> dict[str, list[str]]: 263 """Format connector names as a GitHub Actions matrix. 264 265 GitHub Actions matrix generation needs at least one value. Return an empty 266 connector placeholder when no connectors changed so callers can pair this 267 with an `if` guard on the matrix job. 268 """ 269 return {"connector": connectors or [""]} 270 271 272def get_modified_connectors_from_github( 273 pr_number: int, 274 pr_owner: str, 275 pr_repo: str, 276 gh_token: str, 277) -> list[str]: 278 """Get list of connector IDs modified in a PR via the GitHub API. 279 280 This is the preferred path when a PR number is known because it is 281 immune to shallow-clone depth issues and always returns an accurate 282 diff (only the files the PR actually changes). 283 284 Args: 285 pr_number: Pull request number. 286 pr_owner: Repository owner (e.g. `"airbytehq"`). 287 pr_repo: Repository name (e.g. `"airbyte"`). 288 gh_token: GitHub API token (must be provided explicitly). 289 290 Returns: 291 Sorted list of connector technical names. 292 293 Raises: 294 GitHubAPIError: If the API request fails. 295 """ 296 logger.info( 297 "Fetching changed files for PR %s/%s#%d via GitHub API", 298 pr_owner, 299 pr_repo, 300 pr_number, 301 ) 302 changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token) 303 return _extract_connector_names(changed_files) 304 305 306@lru_cache(maxsize=128) 307def get_certified_connectors(repo_path: str | Path) -> set[str]: 308 """Get set of all certified connector IDs from metadata. 309 310 This function reads the metadata.yaml file for each connector to determine 311 which connectors have "certified" support level. 312 313 Args: 314 repo_path: Path to the Airbyte monorepo 315 316 Returns: 317 Set of certified connector technical names 318 319 Example: 320 >>> certified = get_certified_connectors("/path/to/airbyte") 321 >>> "source-postgres" in certified 322 True 323 """ 324 repo_path = Path(repo_path) 325 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 326 327 if not connectors_dir.exists(): 328 raise ValueError(f"Connectors directory not found: {connectors_dir}") 329 330 certified_connectors = set() 331 332 # Iterate through all connector directories 333 for connector_dir in connectors_dir.iterdir(): 334 if not connector_dir.is_dir(): 335 continue 336 337 metadata_file = connector_dir / METADATA_FILE_NAME 338 if not metadata_file.exists(): 339 continue 340 341 # Read metadata to check support level 342 try: 343 import yaml 344 345 with open(metadata_file) as f: 346 metadata = yaml.safe_load(f) 347 348 support_level = metadata.get("data", {}).get("supportLevel") 349 if support_level == "certified": 350 certified_connectors.add(connector_dir.name) 351 except Exception: 352 # Skip connectors with invalid metadata 353 continue 354 355 return certified_connectors 356 357 358def get_all_connectors(repo_path: str | Path) -> set[str]: 359 """Get set of all connector IDs in the repository. 360 361 Args: 362 repo_path: Path to the Airbyte monorepo 363 364 Returns: 365 Set of all connector technical names 366 367 Example: 368 >>> all_connectors = get_all_connectors("/path/to/airbyte") 369 >>> "source-faker" in all_connectors 370 True 371 """ 372 repo_path = Path(repo_path) 373 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 374 375 if not connectors_dir.exists(): 376 raise ValueError(f"Connectors directory not found: {connectors_dir}") 377 378 return {p.name for p in connectors_dir.iterdir() if p.is_dir()} 379 380 381def get_connector_metadata( 382 repo_path: str | Path, 383 connector_name: str, 384) -> dict[str, Any] | None: 385 """Get metadata for a specific connector. 386 387 Args: 388 repo_path: Path to the Airbyte monorepo 389 connector_name: Technical name of the connector (e.g., "source-faker") 390 391 Returns: 392 The connector's metadata dict (the 'data' section), or None if not found 393 394 Example: 395 >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") 396 >>> metadata.get("supportLevel") 397 'certified' 398 """ 399 repo_path = Path(repo_path) 400 connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name 401 metadata_file = connector_dir / METADATA_FILE_NAME 402 403 if not metadata_file.exists(): 404 return None 405 406 try: 407 with open(metadata_file) as f: 408 metadata = yaml.safe_load(f) 409 return metadata.get("data", {}) 410 except Exception: 411 return None 412 413 414def get_connectors_by_language( 415 repo_path: str | Path, 416 language: ConnectorLanguage, 417) -> set[str]: 418 """Get set of all connector IDs for a specific language. 419 420 This function reads connector directories to determine which connectors 421 use the specified language. 422 423 Args: 424 repo_path: Path to the Airbyte monorepo 425 language: Language to filter by 426 427 Returns: 428 Set of connector technical names using the specified language 429 430 Example: 431 >>> python_connectors = get_connectors_by_language( 432 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 433 ... ) 434 >>> "source-faker" in python_connectors 435 True 436 """ 437 repo_path = Path(repo_path) 438 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 439 440 if not connectors_dir.exists(): 441 raise ValueError(f"Connectors directory not found: {connectors_dir}") 442 443 language_value = language.value 444 connectors_by_language = set() 445 446 # Iterate through all connector directories 447 for connector_dir in connectors_dir.iterdir(): 448 if not connector_dir.is_dir(): 449 continue 450 451 # Determine language based on file structure 452 connector_name = connector_dir.name 453 detected_language = _detect_connector_language(connector_dir, connector_name) 454 455 if detected_language == language_value: 456 connectors_by_language.add(connector_dir.name) 457 458 return connectors_by_language 459 460 461def get_connectors_by_type( 462 repo_path: str | Path, 463 connector_type: ConnectorType, 464) -> set[str]: 465 """Get set of all connector IDs for a specific type (source or destination). 466 467 This function reads connector directories to determine which connectors 468 match the specified type based on their metadata.yaml or name prefix. 469 470 Args: 471 repo_path: Path to the Airbyte monorepo 472 connector_type: Type to filter by (source or destination) 473 474 Returns: 475 Set of connector technical names matching the specified type 476 477 Example: 478 >>> source_connectors = get_connectors_by_type( 479 ... "/path/to/airbyte", ConnectorType.SOURCE 480 ... ) 481 >>> "source-postgres" in source_connectors 482 True 483 """ 484 repo_path = Path(repo_path) 485 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 486 487 if not connectors_dir.exists(): 488 raise ValueError(f"Connectors directory not found: {connectors_dir}") 489 490 type_value = connector_type.value 491 connectors_by_type = set() 492 493 for connector_dir in connectors_dir.iterdir(): 494 if not connector_dir.is_dir(): 495 continue 496 497 connector_name = connector_dir.name 498 499 # First try to get type from metadata.yaml 500 metadata = get_connector_metadata(repo_path, connector_name) 501 if metadata: 502 metadata_type = metadata.get("connectorType") 503 if metadata_type == type_value: 504 connectors_by_type.add(connector_name) 505 continue 506 507 # Fallback to name prefix detection 508 if connector_name.startswith(f"{type_value}-"): 509 connectors_by_type.add(connector_name) 510 511 return connectors_by_type 512 513 514def get_connectors_by_subtype( 515 repo_path: str | Path, 516 connector_subtype: ConnectorSubtype, 517) -> set[str]: 518 """Get set of all connector IDs for a specific subtype (api, database, file, etc.). 519 520 This function reads connector metadata.yaml files to determine which connectors 521 match the specified subtype. 522 523 Args: 524 repo_path: Path to the Airbyte monorepo 525 connector_subtype: Subtype to filter by (api, database, file, custom) 526 527 Returns: 528 Set of connector technical names matching the specified subtype 529 530 Example: 531 >>> database_connectors = get_connectors_by_subtype( 532 ... "/path/to/airbyte", ConnectorSubtype.DATABASE 533 ... ) 534 >>> "source-postgres" in database_connectors 535 True 536 """ 537 repo_path = Path(repo_path) 538 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 539 540 if not connectors_dir.exists(): 541 raise ValueError(f"Connectors directory not found: {connectors_dir}") 542 543 subtype_value = connector_subtype.value 544 connectors_by_subtype = set() 545 546 for connector_dir in connectors_dir.iterdir(): 547 if not connector_dir.is_dir(): 548 continue 549 550 connector_name = connector_dir.name 551 metadata = get_connector_metadata(repo_path, connector_name) 552 553 if metadata: 554 metadata_subtype = metadata.get("connectorSubtype") 555 if metadata_subtype == subtype_value: 556 connectors_by_subtype.add(connector_name) 557 558 return connectors_by_subtype 559 560 561@lru_cache(maxsize=1024) 562def _detect_connector_language(connector_dir: Path, connector_name: str) -> str | None: 563 """Detect the language of a connector based on its file structure. 564 565 Args: 566 connector_dir: Path to the connector directory 567 connector_name: Technical name of the connector 568 569 Returns: 570 Language string (python, java, low-code, manifest-only) or None 571 """ 572 # Check for manifest-only (manifest.yaml at root) 573 if (connector_dir / "manifest.yaml").is_file(): 574 return "manifest-only" 575 576 # Check for low-code (manifest.yaml in source directory) 577 source_dir = connector_dir / connector_name.replace("-", "_") 578 if (source_dir / "manifest.yaml").is_file(): 579 return "low-code" 580 581 # Check for Python (setup.py or pyproject.toml) 582 if (connector_dir / "setup.py").is_file() or ( 583 connector_dir / "pyproject.toml" 584 ).is_file(): 585 return "python" 586 587 # Check for Java/Kotlin (src/main/java or src/main/kotlin) 588 if (connector_dir / "src" / "main" / "java").exists() or ( 589 connector_dir / "src" / "main" / "kotlin" 590 ).exists(): 591 return "java" 592 593 return None 594 595 596@lru_cache(maxsize=128) 597def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 598 """Get set of connectors using local CDK reference instead of published version. 599 600 This function detects connectors that are configured to use a local CDK 601 checkout rather than a published CDK version. The detection method differs 602 by connector language: 603 - Python: Checks for path-based dependency in pyproject.toml 604 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 605 606 Args: 607 repo_path: Path to the Airbyte monorepo 608 609 Returns: 610 Set of connector technical names using local CDK reference 611 612 Example: 613 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 614 >>> "destination-motherduck" in local_cdk_connectors 615 True 616 """ 617 repo_path = Path(repo_path) 618 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 619 620 if not connectors_dir.exists(): 621 raise ValueError(f"Connectors directory not found: {connectors_dir}") 622 623 local_cdk_connectors = set() 624 625 # Iterate through all connector directories 626 for connector_dir in connectors_dir.iterdir(): 627 if not connector_dir.is_dir(): 628 continue 629 630 connector_name = connector_dir.name 631 detected_language = _detect_connector_language(connector_dir, connector_name) 632 633 # Check for local CDK based on language 634 if detected_language in ("python", "low-code") and _has_local_cdk_python( 635 connector_dir 636 ): 637 # Python connectors: check pyproject.toml for path-based CDK dependency 638 local_cdk_connectors.add(connector_name) 639 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 640 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 641 local_cdk_connectors.add(connector_name) 642 643 return local_cdk_connectors 644 645 646def _has_local_cdk_python(connector_dir: Path) -> bool: 647 """Check if a Python connector uses local CDK reference. 648 649 Args: 650 connector_dir: Path to the connector directory 651 652 Returns: 653 True if connector uses local CDK reference 654 """ 655 pyproject_file = connector_dir / "pyproject.toml" 656 if not pyproject_file.exists(): 657 return False 658 659 try: 660 content = pyproject_file.read_text() 661 # Look for path-based airbyte-cdk dependency 662 # Pattern: airbyte-cdk = { path = "..." } or airbyte-cdk = {path = "..."} 663 return bool(re.search(r"airbyte-cdk\s*=\s*\{[^}]*path\s*=", content)) 664 except Exception: 665 return False 666 667 668def _has_local_cdk_java(connector_dir: Path) -> bool: 669 """Check if a Java/Kotlin connector uses local CDK reference. 670 671 Mirrors the implementation from airbyte-ci/connectors/connector_ops/connector_ops/utils.py 672 673 Args: 674 connector_dir: Path to the connector directory 675 676 Returns: 677 True if connector uses local CDK reference 678 """ 679 # Check both build.gradle and build.gradle.kts 680 for build_file_name in ("build.gradle", "build.gradle.kts"): 681 build_file = connector_dir / build_file_name 682 if not build_file.exists(): 683 continue 684 685 try: 686 # Read file and strip inline comments 687 contents = "\n".join( 688 [ 689 line.split("//")[0] # Remove inline comments 690 for line in build_file.read_text().split("\n") 691 ] 692 ) 693 # Remove spaces and check for useLocalCdk=true 694 contents = contents.replace(" ", "") 695 if "useLocalCdk=true" in contents: 696 return True 697 except Exception: 698 continue 699 700 return False 701 702 703@dataclass 704class ConnectorListResult: 705 """Result of listing connectors with filters.""" 706 707 connectors: list[str] 708 count: int 709 710 711def list_connectors( 712 repo_path: str | Path, 713 certified: bool | None = None, 714 modified: bool | None = None, 715 language_filter: set[str] | None = None, 716 language_exclude: set[str] | None = None, 717 connector_type: str | None = None, 718 connector_subtype: str | None = None, 719 base_ref: str | None = None, 720 head_ref: str | None = None, 721 pr_number: int | None = None, 722 pr_owner: str | None = None, 723 pr_repo: str | None = None, 724 gh_token: str | None = None, 725) -> ConnectorListResult: 726 """List connectors in the Airbyte monorepo with flexible filtering. 727 728 This is the core capability function that encapsulates all filtering logic. 729 Both CLI and MCP layers should use this function. 730 731 When `modified` filtering is requested and both `gh_token` and full PR 732 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 733 GitHub API is used to determine changed files — this is the most reliable 734 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 735 when the token is not provided or PR context is incomplete. 736 737 Args: 738 repo_path: Path to the Airbyte monorepo 739 certified: Filter by certification status (True=certified only, 740 False=non-certified only, None=all) 741 modified: Filter by modification status (True=modified only, 742 False=not-modified only, None=all) 743 language_filter: Set of languages to include (python, java, low-code, manifest-only) 744 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 745 connector_type: Filter by connector type (source, destination, or None for all) 746 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 747 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 748 head_ref: Head git reference for modification detection (default: "HEAD") 749 pr_number: Pull request number (enables GitHub API path for modification detection) 750 pr_owner: Repository owner for PR (e.g. "airbytehq") 751 pr_repo: Repository name for PR (e.g. "airbyte") 752 gh_token: GitHub API token. When provided together with PR context, 753 the GitHub API is used instead of local git diff. 754 755 Returns: 756 ConnectorListResult with sorted list of connector names and count 757 758 Raises: 759 ValueError: If both language_filter and language_exclude are provided 760 """ 761 # Validate mutual exclusivity of language filters 762 if language_filter is not None and language_exclude is not None: 763 raise ValueError( 764 "Cannot specify both language_filter and language_exclude. " 765 "Only one language filter parameter is accepted." 766 ) 767 768 # Start with all connectors 769 result = get_all_connectors(repo_path) 770 771 # Apply certified filter 772 if certified is not None: 773 certified_set = get_certified_connectors(repo_path) 774 if certified: 775 # Include only certified 776 result &= certified_set 777 else: 778 # Include only non-certified 779 result -= certified_set 780 781 # Apply modified filter 782 if modified is not None: 783 changed_set: set[str] | None = None 784 785 # Use GitHub API when token and full PR context are explicitly provided 786 if ( 787 gh_token is not None 788 and pr_number is not None 789 and pr_owner is not None 790 and pr_repo is not None 791 ): 792 try: 793 changed_set = set( 794 get_modified_connectors_from_github( 795 pr_number, pr_owner, pr_repo, gh_token=gh_token 796 ) 797 ) 798 except (GitHubAPIError, requests.RequestException): 799 logger.warning( 800 "GitHub API call failed for PR %s/%s#%d; " 801 "falling back to local git diff", 802 pr_owner, 803 pr_repo, 804 pr_number, 805 exc_info=True, 806 ) 807 808 # Fallback to local git diff 809 if changed_set is None: 810 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 811 head = head_ref if head_ref is not None else "HEAD" 812 changed_set = set(get_modified_connectors(repo_path, base, head)) 813 814 if modified: 815 # Include only modified 816 result &= changed_set 817 else: 818 # Include only not-modified 819 result -= changed_set 820 821 # Apply language include filter 822 if language_filter: 823 # Get connectors for all specified languages and union them 824 lang_result: set[str] = set() 825 for lang in language_filter: 826 lang_result |= get_connectors_by_language( 827 repo_path, ConnectorLanguage(lang) 828 ) 829 result &= lang_result 830 831 # Apply language exclude filter 832 if language_exclude: 833 # Get connectors for all specified languages and exclude them 834 for lang in language_exclude: 835 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 836 result -= excluded 837 838 # Apply connector type filter 839 if connector_type is not None: 840 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 841 result &= type_set 842 843 # Apply connector subtype filter 844 if connector_subtype is not None: 845 subtype_set = get_connectors_by_subtype( 846 repo_path, ConnectorSubtype(connector_subtype) 847 ) 848 result &= subtype_set 849 850 return ConnectorListResult( 851 connectors=sorted(result), 852 count=len(result), 853 )
37class ConnectorSubtype(StrEnum): 38 """Connector subtypes based on data source category.""" 39 40 API = "api" 41 DATABASE = "database" 42 FILE = "file" 43 CUSTOM = "custom"
Connector subtypes based on data source category.
86def get_modified_connectors( 87 repo_path: str | Path, 88 base_ref: str = GIT_DEFAULT_BRANCH, 89 head_ref: str = "HEAD", 90) -> list[str]: 91 """Get list of connector IDs modified according to a local git diff. 92 93 This is the *local-only* fallback used when the GitHub API path is not 94 available (e.g. no PR number or no token). It uses `git merge-base` 95 when possible so that only changes introduced on the head branch are 96 reported, falling back to a plain two-ref diff when the merge-base 97 cannot be determined (shallow clones without enough history). 98 99 Prefer `get_modified_connectors_from_github` whenever a PR number 100 is known, as it avoids shallow-clone pitfalls entirely. 101 102 Args: 103 repo_path: Path to the Airbyte monorepo. 104 base_ref: Base git reference to compare against. 105 head_ref: Head git reference to compare. 106 107 Returns: 108 Sorted list of connector technical names. 109 """ 110 repo_path = Path(repo_path) 111 112 # Try to find the merge-base so the diff only contains changes introduced 113 # on the head branch, avoiding false positives from unrelated base-branch 114 # commits. Falls back to a plain two-ref diff when unavailable. 115 effective_base = base_ref 116 try: 117 mb_result = subprocess.run( 118 ["git", "merge-base", base_ref, head_ref], 119 cwd=repo_path, 120 capture_output=True, 121 text=True, 122 check=True, 123 timeout=30, 124 ) 125 effective_base = mb_result.stdout.strip() 126 logger.debug("Using merge-base %s for diff", effective_base) 127 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): 128 logger.debug( 129 "Could not determine merge-base for %s..%s; falling back to plain diff", 130 base_ref, 131 head_ref, 132 ) 133 134 try: 135 result = subprocess.run( 136 ["git", "diff", "--name-only", effective_base, head_ref], 137 cwd=repo_path, 138 capture_output=True, 139 text=True, 140 check=True, 141 timeout=30, 142 ) 143 changed_files = result.stdout.strip().split("\n") 144 except subprocess.CalledProcessError as e: 145 raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e 146 except subprocess.TimeoutExpired as e: 147 raise RuntimeError("Git diff command timed out after 30 seconds") from e 148 149 return _extract_connector_names(changed_files)
Get list of connector IDs modified according to a local git diff.
This is the local-only fallback used when the GitHub API path is not
available (e.g. no PR number or no token). It uses git merge-base
when possible so that only changes introduced on the head branch are
reported, falling back to a plain two-ref diff when the merge-base
cannot be determined (shallow clones without enough history).
Prefer get_modified_connectors_from_github whenever a PR number
is known, as it avoids shallow-clone pitfalls entirely.
Arguments:
- repo_path: Path to the Airbyte monorepo.
- base_ref: Base git reference to compare against.
- head_ref: Head git reference to compare.
Returns:
Sorted list of connector technical names.
152def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str: 153 """Find the merge base of a base git reference and `HEAD`. 154 155 Resolve it once and pass it as `base_ref` to `get_changed_files` and 156 `get_file_at_merge_base` so both compare against the same commit. 157 158 Args: 159 path: A directory inside the git checkout. 160 base_ref: Base git reference, e.g. `origin/master`. 161 162 Returns: 163 The merge base commit's SHA. 164 165 Raises: 166 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 167 git checkout, or git times out. 168 OSError: If git cannot be run or `path` does not exist. 169 """ 170 return _git(path, "merge-base", base_ref, "HEAD").strip()
Find the merge base of a base git reference and HEAD.
Resolve it once and pass it as base_ref to get_changed_files and
get_file_at_merge_base so both compare against the same commit.
Arguments:
- path: A directory inside the git checkout.
- base_ref: Base git reference, e.g.
origin/master.
Returns:
The merge base commit's SHA.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
173def get_changed_files( 174 path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH 175) -> list[str]: 176 """List the files under a directory that changed since a base git reference. 177 178 Compares the working tree, untracked files included, with the merge base of 179 `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all 180 count. Unlike `get_modified_connectors`, it never falls back to a plain diff, 181 so a caller that skips work based on the result learns when the changes are 182 unknown. 183 184 Args: 185 path: A directory inside the git checkout. Only changes under it are listed. 186 base_ref: Base git reference to compare against, e.g. `origin/master`, 187 or a merge base from `get_merge_base`. 188 189 Returns: 190 Sorted paths relative to `path`, exactly as stored (git does not quote 191 them). A move lists both the old and the new path. 192 193 Raises: 194 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 195 git checkout, or git times out. 196 OSError: If git cannot be run or `path` does not exist. 197 """ 198 merge_base = get_merge_base(path, base_ref) 199 # -z prints paths verbatim, NUL-separated; without it git quotes paths with 200 # non-ASCII characters. --no-renames lists both sides of a move, so moving a 201 # file in or out of a directory shows up as a change on both sides. 202 changed = _git( 203 path, 204 "diff", 205 "--name-only", 206 "-z", 207 "--no-renames", 208 "--relative", 209 merge_base, 210 "--", 211 ).split("\0") 212 untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split( 213 "\0" 214 ) 215 return sorted({file_path for file_path in changed + untracked if file_path})
List the files under a directory that changed since a base git reference.
Compares the working tree, untracked files included, with the merge base of
base_ref and HEAD, so committed, uncommitted and untracked changes all
count. Unlike get_modified_connectors, it never falls back to a plain diff,
so a caller that skips work based on the result learns when the changes are
unknown.
Arguments:
- path: A directory inside the git checkout. Only changes under it are listed.
- base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
Sorted paths relative to
path, exactly as stored (git does not quote them). A move lists both the old and the new path.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
218def get_file_at_merge_base( 219 path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH 220) -> str | None: 221 """Read a file as it was at the merge base of a base git reference and `HEAD`. 222 223 This is the base side of the diff that `get_changed_files` lists, so the two 224 can be used together to tell what changed inside a file. 225 226 Args: 227 path: A directory inside the git checkout. 228 file_name: The file's path relative to `path`. 229 base_ref: Base git reference to compare against, e.g. `origin/master`, 230 or a merge base from `get_merge_base`. 231 232 Returns: 233 The file's content, or None if it did not exist at the merge base. 234 235 Raises: 236 subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a 237 git checkout, or git times out. 238 OSError: If git cannot be run or `path` does not exist. 239 """ 240 merge_base = get_merge_base(path, base_ref) 241 try: 242 # `<rev>:./<file>` resolves the file relative to `path`, not the repo root. 243 return _git(path, "show", f"{merge_base}:./{file_name}") 244 except subprocess.CalledProcessError: 245 return None
Read a file as it was at the merge base of a base git reference and HEAD.
This is the base side of the diff that get_changed_files lists, so the two
can be used together to tell what changed inside a file.
Arguments:
- path: A directory inside the git checkout.
- file_name: The file's path relative to
path. - base_ref: Base git reference to compare against, e.g.
origin/master, or a merge base fromget_merge_base.
Returns:
The file's content, or None if it did not exist at the merge base.
Raises:
- subprocess.SubprocessError: If
base_refis unknown,pathis not in a git checkout, or git times out. - OSError: If git cannot be run or
pathdoes not exist.
261def format_github_actions_connector_matrix( 262 connectors: list[str], 263) -> dict[str, list[str]]: 264 """Format connector names as a GitHub Actions matrix. 265 266 GitHub Actions matrix generation needs at least one value. Return an empty 267 connector placeholder when no connectors changed so callers can pair this 268 with an `if` guard on the matrix job. 269 """ 270 return {"connector": connectors or [""]}
Format connector names as a GitHub Actions matrix.
GitHub Actions matrix generation needs at least one value. Return an empty
connector placeholder when no connectors changed so callers can pair this
with an if guard on the matrix job.
273def get_modified_connectors_from_github( 274 pr_number: int, 275 pr_owner: str, 276 pr_repo: str, 277 gh_token: str, 278) -> list[str]: 279 """Get list of connector IDs modified in a PR via the GitHub API. 280 281 This is the preferred path when a PR number is known because it is 282 immune to shallow-clone depth issues and always returns an accurate 283 diff (only the files the PR actually changes). 284 285 Args: 286 pr_number: Pull request number. 287 pr_owner: Repository owner (e.g. `"airbytehq"`). 288 pr_repo: Repository name (e.g. `"airbyte"`). 289 gh_token: GitHub API token (must be provided explicitly). 290 291 Returns: 292 Sorted list of connector technical names. 293 294 Raises: 295 GitHubAPIError: If the API request fails. 296 """ 297 logger.info( 298 "Fetching changed files for PR %s/%s#%d via GitHub API", 299 pr_owner, 300 pr_repo, 301 pr_number, 302 ) 303 changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token) 304 return _extract_connector_names(changed_files)
Get list of connector IDs modified in a PR via the GitHub API.
This is the preferred path when a PR number is known because it is immune to shallow-clone depth issues and always returns an accurate diff (only the files the PR actually changes).
Arguments:
- pr_number: Pull request number.
- pr_owner: Repository owner (e.g.
"airbytehq"). - pr_repo: Repository name (e.g.
"airbyte"). - gh_token: GitHub API token (must be provided explicitly).
Returns:
Sorted list of connector technical names.
Raises:
- GitHubAPIError: If the API request fails.
307@lru_cache(maxsize=128) 308def get_certified_connectors(repo_path: str | Path) -> set[str]: 309 """Get set of all certified connector IDs from metadata. 310 311 This function reads the metadata.yaml file for each connector to determine 312 which connectors have "certified" support level. 313 314 Args: 315 repo_path: Path to the Airbyte monorepo 316 317 Returns: 318 Set of certified connector technical names 319 320 Example: 321 >>> certified = get_certified_connectors("/path/to/airbyte") 322 >>> "source-postgres" in certified 323 True 324 """ 325 repo_path = Path(repo_path) 326 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 327 328 if not connectors_dir.exists(): 329 raise ValueError(f"Connectors directory not found: {connectors_dir}") 330 331 certified_connectors = set() 332 333 # Iterate through all connector directories 334 for connector_dir in connectors_dir.iterdir(): 335 if not connector_dir.is_dir(): 336 continue 337 338 metadata_file = connector_dir / METADATA_FILE_NAME 339 if not metadata_file.exists(): 340 continue 341 342 # Read metadata to check support level 343 try: 344 import yaml 345 346 with open(metadata_file) as f: 347 metadata = yaml.safe_load(f) 348 349 support_level = metadata.get("data", {}).get("supportLevel") 350 if support_level == "certified": 351 certified_connectors.add(connector_dir.name) 352 except Exception: 353 # Skip connectors with invalid metadata 354 continue 355 356 return certified_connectors
Get set of all certified connector IDs from metadata.
This function reads the metadata.yaml file for each connector to determine which connectors have "certified" support level.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of certified connector technical names
Example:
>>> certified = get_certified_connectors("/path/to/airbyte") >>> "source-postgres" in certified True
359def get_all_connectors(repo_path: str | Path) -> set[str]: 360 """Get set of all connector IDs in the repository. 361 362 Args: 363 repo_path: Path to the Airbyte monorepo 364 365 Returns: 366 Set of all connector technical names 367 368 Example: 369 >>> all_connectors = get_all_connectors("/path/to/airbyte") 370 >>> "source-faker" in all_connectors 371 True 372 """ 373 repo_path = Path(repo_path) 374 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 375 376 if not connectors_dir.exists(): 377 raise ValueError(f"Connectors directory not found: {connectors_dir}") 378 379 return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
Get set of all connector IDs in the repository.
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of all connector technical names
Example:
>>> all_connectors = get_all_connectors("/path/to/airbyte") >>> "source-faker" in all_connectors True
382def get_connector_metadata( 383 repo_path: str | Path, 384 connector_name: str, 385) -> dict[str, Any] | None: 386 """Get metadata for a specific connector. 387 388 Args: 389 repo_path: Path to the Airbyte monorepo 390 connector_name: Technical name of the connector (e.g., "source-faker") 391 392 Returns: 393 The connector's metadata dict (the 'data' section), or None if not found 394 395 Example: 396 >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") 397 >>> metadata.get("supportLevel") 398 'certified' 399 """ 400 repo_path = Path(repo_path) 401 connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name 402 metadata_file = connector_dir / METADATA_FILE_NAME 403 404 if not metadata_file.exists(): 405 return None 406 407 try: 408 with open(metadata_file) as f: 409 metadata = yaml.safe_load(f) 410 return metadata.get("data", {}) 411 except Exception: 412 return None
Get metadata for a specific connector.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_name: Technical name of the connector (e.g., "source-faker")
Returns:
The connector's metadata dict (the 'data' section), or None if not found
Example:
>>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker") >>> metadata.get("supportLevel") 'certified'
415def get_connectors_by_language( 416 repo_path: str | Path, 417 language: ConnectorLanguage, 418) -> set[str]: 419 """Get set of all connector IDs for a specific language. 420 421 This function reads connector directories to determine which connectors 422 use the specified language. 423 424 Args: 425 repo_path: Path to the Airbyte monorepo 426 language: Language to filter by 427 428 Returns: 429 Set of connector technical names using the specified language 430 431 Example: 432 >>> python_connectors = get_connectors_by_language( 433 ... "/path/to/airbyte", ConnectorLanguage.PYTHON 434 ... ) 435 >>> "source-faker" in python_connectors 436 True 437 """ 438 repo_path = Path(repo_path) 439 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 440 441 if not connectors_dir.exists(): 442 raise ValueError(f"Connectors directory not found: {connectors_dir}") 443 444 language_value = language.value 445 connectors_by_language = set() 446 447 # Iterate through all connector directories 448 for connector_dir in connectors_dir.iterdir(): 449 if not connector_dir.is_dir(): 450 continue 451 452 # Determine language based on file structure 453 connector_name = connector_dir.name 454 detected_language = _detect_connector_language(connector_dir, connector_name) 455 456 if detected_language == language_value: 457 connectors_by_language.add(connector_dir.name) 458 459 return connectors_by_language
Get set of all connector IDs for a specific language.
This function reads connector directories to determine which connectors use the specified language.
Arguments:
- repo_path: Path to the Airbyte monorepo
- language: Language to filter by
Returns:
Set of connector technical names using the specified language
Example:
>>> python_connectors = get_connectors_by_language( ... "/path/to/airbyte", ConnectorLanguage.PYTHON ... ) >>> "source-faker" in python_connectors True
462def get_connectors_by_type( 463 repo_path: str | Path, 464 connector_type: ConnectorType, 465) -> set[str]: 466 """Get set of all connector IDs for a specific type (source or destination). 467 468 This function reads connector directories to determine which connectors 469 match the specified type based on their metadata.yaml or name prefix. 470 471 Args: 472 repo_path: Path to the Airbyte monorepo 473 connector_type: Type to filter by (source or destination) 474 475 Returns: 476 Set of connector technical names matching the specified type 477 478 Example: 479 >>> source_connectors = get_connectors_by_type( 480 ... "/path/to/airbyte", ConnectorType.SOURCE 481 ... ) 482 >>> "source-postgres" in source_connectors 483 True 484 """ 485 repo_path = Path(repo_path) 486 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 487 488 if not connectors_dir.exists(): 489 raise ValueError(f"Connectors directory not found: {connectors_dir}") 490 491 type_value = connector_type.value 492 connectors_by_type = set() 493 494 for connector_dir in connectors_dir.iterdir(): 495 if not connector_dir.is_dir(): 496 continue 497 498 connector_name = connector_dir.name 499 500 # First try to get type from metadata.yaml 501 metadata = get_connector_metadata(repo_path, connector_name) 502 if metadata: 503 metadata_type = metadata.get("connectorType") 504 if metadata_type == type_value: 505 connectors_by_type.add(connector_name) 506 continue 507 508 # Fallback to name prefix detection 509 if connector_name.startswith(f"{type_value}-"): 510 connectors_by_type.add(connector_name) 511 512 return connectors_by_type
Get set of all connector IDs for a specific type (source or destination).
This function reads connector directories to determine which connectors match the specified type based on their metadata.yaml or name prefix.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_type: Type to filter by (source or destination)
Returns:
Set of connector technical names matching the specified type
Example:
>>> source_connectors = get_connectors_by_type( ... "/path/to/airbyte", ConnectorType.SOURCE ... ) >>> "source-postgres" in source_connectors True
515def get_connectors_by_subtype( 516 repo_path: str | Path, 517 connector_subtype: ConnectorSubtype, 518) -> set[str]: 519 """Get set of all connector IDs for a specific subtype (api, database, file, etc.). 520 521 This function reads connector metadata.yaml files to determine which connectors 522 match the specified subtype. 523 524 Args: 525 repo_path: Path to the Airbyte monorepo 526 connector_subtype: Subtype to filter by (api, database, file, custom) 527 528 Returns: 529 Set of connector technical names matching the specified subtype 530 531 Example: 532 >>> database_connectors = get_connectors_by_subtype( 533 ... "/path/to/airbyte", ConnectorSubtype.DATABASE 534 ... ) 535 >>> "source-postgres" in database_connectors 536 True 537 """ 538 repo_path = Path(repo_path) 539 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 540 541 if not connectors_dir.exists(): 542 raise ValueError(f"Connectors directory not found: {connectors_dir}") 543 544 subtype_value = connector_subtype.value 545 connectors_by_subtype = set() 546 547 for connector_dir in connectors_dir.iterdir(): 548 if not connector_dir.is_dir(): 549 continue 550 551 connector_name = connector_dir.name 552 metadata = get_connector_metadata(repo_path, connector_name) 553 554 if metadata: 555 metadata_subtype = metadata.get("connectorSubtype") 556 if metadata_subtype == subtype_value: 557 connectors_by_subtype.add(connector_name) 558 559 return connectors_by_subtype
Get set of all connector IDs for a specific subtype (api, database, file, etc.).
This function reads connector metadata.yaml files to determine which connectors match the specified subtype.
Arguments:
- repo_path: Path to the Airbyte monorepo
- connector_subtype: Subtype to filter by (api, database, file, custom)
Returns:
Set of connector technical names matching the specified subtype
Example:
>>> database_connectors = get_connectors_by_subtype( ... "/path/to/airbyte", ConnectorSubtype.DATABASE ... ) >>> "source-postgres" in database_connectors True
597@lru_cache(maxsize=128) 598def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]: 599 """Get set of connectors using local CDK reference instead of published version. 600 601 This function detects connectors that are configured to use a local CDK 602 checkout rather than a published CDK version. The detection method differs 603 by connector language: 604 - Python: Checks for path-based dependency in pyproject.toml 605 - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files 606 607 Args: 608 repo_path: Path to the Airbyte monorepo 609 610 Returns: 611 Set of connector technical names using local CDK reference 612 613 Example: 614 >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") 615 >>> "destination-motherduck" in local_cdk_connectors 616 True 617 """ 618 repo_path = Path(repo_path) 619 connectors_dir = repo_path / CONNECTOR_PATH_PREFIX 620 621 if not connectors_dir.exists(): 622 raise ValueError(f"Connectors directory not found: {connectors_dir}") 623 624 local_cdk_connectors = set() 625 626 # Iterate through all connector directories 627 for connector_dir in connectors_dir.iterdir(): 628 if not connector_dir.is_dir(): 629 continue 630 631 connector_name = connector_dir.name 632 detected_language = _detect_connector_language(connector_dir, connector_name) 633 634 # Check for local CDK based on language 635 if detected_language in ("python", "low-code") and _has_local_cdk_python( 636 connector_dir 637 ): 638 # Python connectors: check pyproject.toml for path-based CDK dependency 639 local_cdk_connectors.add(connector_name) 640 elif detected_language == "java" and _has_local_cdk_java(connector_dir): 641 # Java/Kotlin connectors: check build.gradle for useLocalCdk=true 642 local_cdk_connectors.add(connector_name) 643 644 return local_cdk_connectors
Get set of connectors using local CDK reference instead of published version.
This function detects connectors that are configured to use a local CDK checkout rather than a published CDK version. The detection method differs by connector language:
- Python: Checks for path-based dependency in pyproject.toml
- Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
Arguments:
- repo_path: Path to the Airbyte monorepo
Returns:
Set of connector technical names using local CDK reference
Example:
>>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte") >>> "destination-motherduck" in local_cdk_connectors True
704@dataclass 705class ConnectorListResult: 706 """Result of listing connectors with filters.""" 707 708 connectors: list[str] 709 count: int
Result of listing connectors with filters.
712def list_connectors( 713 repo_path: str | Path, 714 certified: bool | None = None, 715 modified: bool | None = None, 716 language_filter: set[str] | None = None, 717 language_exclude: set[str] | None = None, 718 connector_type: str | None = None, 719 connector_subtype: str | None = None, 720 base_ref: str | None = None, 721 head_ref: str | None = None, 722 pr_number: int | None = None, 723 pr_owner: str | None = None, 724 pr_repo: str | None = None, 725 gh_token: str | None = None, 726) -> ConnectorListResult: 727 """List connectors in the Airbyte monorepo with flexible filtering. 728 729 This is the core capability function that encapsulates all filtering logic. 730 Both CLI and MCP layers should use this function. 731 732 When `modified` filtering is requested and both `gh_token` and full PR 733 context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the 734 GitHub API is used to determine changed files — this is the most reliable 735 method and avoids shallow-clone pitfalls. Falls back to local `git diff` 736 when the token is not provided or PR context is incomplete. 737 738 Args: 739 repo_path: Path to the Airbyte monorepo 740 certified: Filter by certification status (True=certified only, 741 False=non-certified only, None=all) 742 modified: Filter by modification status (True=modified only, 743 False=not-modified only, None=all) 744 language_filter: Set of languages to include (python, java, low-code, manifest-only) 745 language_exclude: Set of languages to exclude (mutually exclusive with language_filter) 746 connector_type: Filter by connector type (source, destination, or None for all) 747 connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all) 748 base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master") 749 head_ref: Head git reference for modification detection (default: "HEAD") 750 pr_number: Pull request number (enables GitHub API path for modification detection) 751 pr_owner: Repository owner for PR (e.g. "airbytehq") 752 pr_repo: Repository name for PR (e.g. "airbyte") 753 gh_token: GitHub API token. When provided together with PR context, 754 the GitHub API is used instead of local git diff. 755 756 Returns: 757 ConnectorListResult with sorted list of connector names and count 758 759 Raises: 760 ValueError: If both language_filter and language_exclude are provided 761 """ 762 # Validate mutual exclusivity of language filters 763 if language_filter is not None and language_exclude is not None: 764 raise ValueError( 765 "Cannot specify both language_filter and language_exclude. " 766 "Only one language filter parameter is accepted." 767 ) 768 769 # Start with all connectors 770 result = get_all_connectors(repo_path) 771 772 # Apply certified filter 773 if certified is not None: 774 certified_set = get_certified_connectors(repo_path) 775 if certified: 776 # Include only certified 777 result &= certified_set 778 else: 779 # Include only non-certified 780 result -= certified_set 781 782 # Apply modified filter 783 if modified is not None: 784 changed_set: set[str] | None = None 785 786 # Use GitHub API when token and full PR context are explicitly provided 787 if ( 788 gh_token is not None 789 and pr_number is not None 790 and pr_owner is not None 791 and pr_repo is not None 792 ): 793 try: 794 changed_set = set( 795 get_modified_connectors_from_github( 796 pr_number, pr_owner, pr_repo, gh_token=gh_token 797 ) 798 ) 799 except (GitHubAPIError, requests.RequestException): 800 logger.warning( 801 "GitHub API call failed for PR %s/%s#%d; " 802 "falling back to local git diff", 803 pr_owner, 804 pr_repo, 805 pr_number, 806 exc_info=True, 807 ) 808 809 # Fallback to local git diff 810 if changed_set is None: 811 base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH 812 head = head_ref if head_ref is not None else "HEAD" 813 changed_set = set(get_modified_connectors(repo_path, base, head)) 814 815 if modified: 816 # Include only modified 817 result &= changed_set 818 else: 819 # Include only not-modified 820 result -= changed_set 821 822 # Apply language include filter 823 if language_filter: 824 # Get connectors for all specified languages and union them 825 lang_result: set[str] = set() 826 for lang in language_filter: 827 lang_result |= get_connectors_by_language( 828 repo_path, ConnectorLanguage(lang) 829 ) 830 result &= lang_result 831 832 # Apply language exclude filter 833 if language_exclude: 834 # Get connectors for all specified languages and exclude them 835 for lang in language_exclude: 836 excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang)) 837 result -= excluded 838 839 # Apply connector type filter 840 if connector_type is not None: 841 type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type)) 842 result &= type_set 843 844 # Apply connector subtype filter 845 if connector_subtype is not None: 846 subtype_set = get_connectors_by_subtype( 847 repo_path, ConnectorSubtype(connector_subtype) 848 ) 849 result &= subtype_set 850 851 return ConnectorListResult( 852 connectors=sorted(result), 853 count=len(result), 854 )
List connectors in the Airbyte monorepo with flexible filtering.
This is the core capability function that encapsulates all filtering logic. Both CLI and MCP layers should use this function.
When modified filtering is requested and both gh_token and full PR
context (pr_number, pr_owner, pr_repo) are provided, the
GitHub API is used to determine changed files — this is the most reliable
method and avoids shallow-clone pitfalls. Falls back to local git diff
when the token is not provided or PR context is incomplete.
Arguments:
- repo_path: Path to the Airbyte monorepo
- certified: Filter by certification status (True=certified only, False=non-certified only, None=all)
- modified: Filter by modification status (True=modified only, False=not-modified only, None=all)
- language_filter: Set of languages to include (python, java, low-code, manifest-only)
- language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
- connector_type: Filter by connector type (source, destination, or None for all)
- connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
- base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
- head_ref: Head git reference for modification detection (default: "HEAD")
- pr_number: Pull request number (enables GitHub API path for modification detection)
- pr_owner: Repository owner for PR (e.g. "airbytehq")
- pr_repo: Repository name for PR (e.g. "airbyte")
- gh_token: GitHub API token. When provided together with PR context, the GitHub API is used instead of local git diff.
Returns:
ConnectorListResult with sorted list of connector names and count
Raises:
- ValueError: If both language_filter and language_exclude are provided