airbyte_ops_mcp.airbyte_repo.list_connectors

Detect changed connectors in the Airbyte monorepo.

This module provides functionality to detect which connectors have been modified by comparing git diffs between branches.

  1# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
  2"""Detect changed connectors in the Airbyte monorepo.
  3
  4This module provides functionality to detect which connectors have been modified
  5by comparing git diffs between branches.
  6"""
  7
  8from __future__ import annotations
  9
 10import logging
 11import re
 12import subprocess
 13from dataclasses import dataclass
 14from enum import StrEnum
 15from functools import lru_cache
 16from pathlib import Path
 17from typing import Any
 18
 19import requests
 20import yaml
 21
 22from airbyte_ops_mcp.github_api import GitHubAPIError, get_pr_changed_files
 23from airbyte_ops_mcp.registry._enums import ConnectorLanguage, ConnectorType
 24
 25CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
 26METADATA_FILE_NAME = "metadata.yaml"
 27GIT_DEFAULT_BRANCH = "origin/master"
 28
 29logger = logging.getLogger(__name__)
 30
 31
 32class ConnectorSubtype(StrEnum):
 33    """Connector subtypes based on data source category."""
 34
 35    API = "api"
 36    DATABASE = "database"
 37    FILE = "file"
 38    CUSTOM = "custom"
 39
 40
 41def _extract_connector_names(changed_files: list[str]) -> list[str]:
 42    """Extract unique connector names from a list of changed file paths.
 43
 44    Args:
 45        changed_files: File paths relative to the repo root.
 46
 47    Returns:
 48        Sorted list of connector technical names.
 49    """
 50    changed_connectors: set[str] = set()
 51    for file_path in changed_files:
 52        if file_path.startswith(CONNECTOR_PATH_PREFIX + "/"):
 53            # e.g. "airbyte-integrations/connectors/source-faker/..." -> "source-faker"
 54            path_parts = file_path.split("/")
 55            if len(path_parts) >= 3:
 56                changed_connectors.add(path_parts[2])
 57    return sorted(changed_connectors)
 58
 59
 60def get_modified_connectors(
 61    repo_path: str | Path,
 62    base_ref: str = GIT_DEFAULT_BRANCH,
 63    head_ref: str = "HEAD",
 64) -> list[str]:
 65    """Get list of connector IDs modified according to a local git diff.
 66
 67    This is the *local-only* fallback used when the GitHub API path is not
 68    available (e.g. no PR number or no token).  It uses `git merge-base`
 69    when possible so that only changes introduced on the head branch are
 70    reported, falling back to a plain two-ref diff when the merge-base
 71    cannot be determined (shallow clones without enough history).
 72
 73    Prefer `get_modified_connectors_from_github` whenever a PR number
 74    is known, as it avoids shallow-clone pitfalls entirely.
 75
 76    Args:
 77        repo_path: Path to the Airbyte monorepo.
 78        base_ref: Base git reference to compare against.
 79        head_ref: Head git reference to compare.
 80
 81    Returns:
 82        Sorted list of connector technical names.
 83    """
 84    repo_path = Path(repo_path)
 85
 86    # Try to find the merge-base so the diff only contains changes introduced
 87    # on the head branch, avoiding false positives from unrelated base-branch
 88    # commits.  Falls back to a plain two-ref diff when unavailable.
 89    effective_base = base_ref
 90    try:
 91        mb_result = subprocess.run(
 92            ["git", "merge-base", base_ref, head_ref],
 93            cwd=repo_path,
 94            capture_output=True,
 95            text=True,
 96            check=True,
 97            timeout=30,
 98        )
 99        effective_base = mb_result.stdout.strip()
100        logger.debug("Using merge-base %s for diff", effective_base)
101    except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
102        logger.debug(
103            "Could not determine merge-base for %s..%s; falling back to plain diff",
104            base_ref,
105            head_ref,
106        )
107
108    try:
109        result = subprocess.run(
110            ["git", "diff", "--name-only", effective_base, head_ref],
111            cwd=repo_path,
112            capture_output=True,
113            text=True,
114            check=True,
115            timeout=30,
116        )
117        changed_files = result.stdout.strip().split("\n")
118    except subprocess.CalledProcessError as e:
119        raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e
120    except subprocess.TimeoutExpired as e:
121        raise RuntimeError("Git diff command timed out after 30 seconds") from e
122
123    return _extract_connector_names(changed_files)
124
125
126def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str:
127    """Find the merge base of a base git reference and `HEAD`.
128
129    Resolve it once and pass it as `base_ref` to `get_changed_files` and
130    `get_file_at_merge_base` so both compare against the same commit.
131
132    Args:
133        path: A directory inside the git checkout.
134        base_ref: Base git reference, e.g. `origin/master`.
135
136    Returns:
137        The merge base commit's SHA.
138
139    Raises:
140        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
141            git checkout, or git times out.
142        OSError: If git cannot be run or `path` does not exist.
143    """
144    return _git(path, "merge-base", base_ref, "HEAD").strip()
145
146
147def get_changed_files(
148    path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH
149) -> list[str]:
150    """List the files under a directory that changed since a base git reference.
151
152    Compares the working tree, untracked files included, with the merge base of
153    `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all
154    count. Unlike `get_modified_connectors`, it never falls back to a plain diff,
155    so a caller that skips work based on the result learns when the changes are
156    unknown.
157
158    Args:
159        path: A directory inside the git checkout. Only changes under it are listed.
160        base_ref: Base git reference to compare against, e.g. `origin/master`,
161            or a merge base from `get_merge_base`.
162
163    Returns:
164        Sorted paths relative to `path`, exactly as stored (git does not quote
165        them). A move lists both the old and the new path.
166
167    Raises:
168        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
169            git checkout, or git times out.
170        OSError: If git cannot be run or `path` does not exist.
171    """
172    merge_base = get_merge_base(path, base_ref)
173    # -z prints paths verbatim, NUL-separated; without it git quotes paths with
174    # non-ASCII characters. --no-renames lists both sides of a move, so moving a
175    # file in or out of a directory shows up as a change on both sides.
176    changed = _git(
177        path,
178        "diff",
179        "--name-only",
180        "-z",
181        "--no-renames",
182        "--relative",
183        merge_base,
184        "--",
185    ).split("\0")
186    untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split(
187        "\0"
188    )
189    return sorted({file_path for file_path in changed + untracked if file_path})
190
191
192def get_file_at_merge_base(
193    path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH
194) -> str | None:
195    """Read a file as it was at the merge base of a base git reference and `HEAD`.
196
197    This is the base side of the diff that `get_changed_files` lists, so the two
198    can be used together to tell what changed inside a file.
199
200    Args:
201        path: A directory inside the git checkout.
202        file_name: The file's path relative to `path`.
203        base_ref: Base git reference to compare against, e.g. `origin/master`,
204            or a merge base from `get_merge_base`.
205
206    Returns:
207        The file's content, or None if it did not exist at the merge base.
208
209    Raises:
210        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
211            git checkout, or git times out.
212        OSError: If git cannot be run or `path` does not exist.
213    """
214    merge_base = get_merge_base(path, base_ref)
215    try:
216        # `<rev>:./<file>` resolves the file relative to `path`, not the repo root.
217        return _git(path, "show", f"{merge_base}:./{file_name}")
218    except subprocess.CalledProcessError:
219        return None
220
221
222def _git(cwd: str | Path, *args: str) -> str:
223    """Run a git command in `cwd` and return its output, raising on failure."""
224    return subprocess.run(
225        ["git", *args],
226        cwd=cwd,
227        capture_output=True,
228        encoding="utf-8",
229        errors="surrogateescape",
230        check=True,
231        timeout=30,
232    ).stdout
233
234
235def format_github_actions_connector_matrix(
236    connectors: list[str],
237) -> dict[str, list[str]]:
238    """Format connector names as a GitHub Actions matrix.
239
240    GitHub Actions matrix generation needs at least one value. Return an empty
241    connector placeholder when no connectors changed so callers can pair this
242    with an `if` guard on the matrix job.
243    """
244    return {"connector": connectors or [""]}
245
246
247def get_modified_connectors_from_github(
248    pr_number: int,
249    pr_owner: str,
250    pr_repo: str,
251    gh_token: str,
252) -> list[str]:
253    """Get list of connector IDs modified in a PR via the GitHub API.
254
255    This is the preferred path when a PR number is known because it is
256    immune to shallow-clone depth issues and always returns an accurate
257    diff (only the files the PR actually changes).
258
259    Args:
260        pr_number: Pull request number.
261        pr_owner: Repository owner (e.g. `"airbytehq"`).
262        pr_repo: Repository name (e.g. `"airbyte"`).
263        gh_token: GitHub API token (must be provided explicitly).
264
265    Returns:
266        Sorted list of connector technical names.
267
268    Raises:
269        GitHubAPIError: If the API request fails.
270    """
271    logger.info(
272        "Fetching changed files for PR %s/%s#%d via GitHub API",
273        pr_owner,
274        pr_repo,
275        pr_number,
276    )
277    changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token)
278    return _extract_connector_names(changed_files)
279
280
281@lru_cache(maxsize=128)
282def get_certified_connectors(repo_path: str | Path) -> set[str]:
283    """Get set of all certified connector IDs from metadata.
284
285    This function reads the metadata.yaml file for each connector to determine
286    which connectors have "certified" support level.
287
288    Args:
289        repo_path: Path to the Airbyte monorepo
290
291    Returns:
292        Set of certified connector technical names
293
294    Example:
295        >>> certified = get_certified_connectors("/path/to/airbyte")
296        >>> "source-postgres" in certified
297        True
298    """
299    repo_path = Path(repo_path)
300    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
301
302    if not connectors_dir.exists():
303        raise ValueError(f"Connectors directory not found: {connectors_dir}")
304
305    certified_connectors = set()
306
307    # Iterate through all connector directories
308    for connector_dir in connectors_dir.iterdir():
309        if not connector_dir.is_dir():
310            continue
311
312        metadata_file = connector_dir / METADATA_FILE_NAME
313        if not metadata_file.exists():
314            continue
315
316        # Read metadata to check support level
317        try:
318            import yaml
319
320            with open(metadata_file) as f:
321                metadata = yaml.safe_load(f)
322
323            support_level = metadata.get("data", {}).get("supportLevel")
324            if support_level == "certified":
325                certified_connectors.add(connector_dir.name)
326        except Exception:
327            # Skip connectors with invalid metadata
328            continue
329
330    return certified_connectors
331
332
333def get_all_connectors(repo_path: str | Path) -> set[str]:
334    """Get set of all connector IDs in the repository.
335
336    Args:
337        repo_path: Path to the Airbyte monorepo
338
339    Returns:
340        Set of all connector technical names
341
342    Example:
343        >>> all_connectors = get_all_connectors("/path/to/airbyte")
344        >>> "source-faker" in all_connectors
345        True
346    """
347    repo_path = Path(repo_path)
348    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
349
350    if not connectors_dir.exists():
351        raise ValueError(f"Connectors directory not found: {connectors_dir}")
352
353    return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
354
355
356def get_connector_metadata(
357    repo_path: str | Path,
358    connector_name: str,
359) -> dict[str, Any] | None:
360    """Get metadata for a specific connector.
361
362    Args:
363        repo_path: Path to the Airbyte monorepo
364        connector_name: Technical name of the connector (e.g., "source-faker")
365
366    Returns:
367        The connector's metadata dict (the 'data' section), or None if not found
368
369    Example:
370        >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker")
371        >>> metadata.get("supportLevel")
372        'certified'
373    """
374    repo_path = Path(repo_path)
375    connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name
376    metadata_file = connector_dir / METADATA_FILE_NAME
377
378    if not metadata_file.exists():
379        return None
380
381    try:
382        with open(metadata_file) as f:
383            metadata = yaml.safe_load(f)
384        return metadata.get("data", {})
385    except Exception:
386        return None
387
388
389def get_connectors_by_language(
390    repo_path: str | Path,
391    language: ConnectorLanguage,
392) -> set[str]:
393    """Get set of all connector IDs for a specific language.
394
395    This function reads connector directories to determine which connectors
396    use the specified language.
397
398    Args:
399        repo_path: Path to the Airbyte monorepo
400        language: Language to filter by
401
402    Returns:
403        Set of connector technical names using the specified language
404
405    Example:
406        >>> python_connectors = get_connectors_by_language(
407        ...     "/path/to/airbyte", ConnectorLanguage.PYTHON
408        ... )
409        >>> "source-faker" in python_connectors
410        True
411    """
412    repo_path = Path(repo_path)
413    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
414
415    if not connectors_dir.exists():
416        raise ValueError(f"Connectors directory not found: {connectors_dir}")
417
418    language_value = language.value
419    connectors_by_language = set()
420
421    # Iterate through all connector directories
422    for connector_dir in connectors_dir.iterdir():
423        if not connector_dir.is_dir():
424            continue
425
426        # Determine language based on file structure
427        connector_name = connector_dir.name
428        detected_language = _detect_connector_language(connector_dir, connector_name)
429
430        if detected_language == language_value:
431            connectors_by_language.add(connector_dir.name)
432
433    return connectors_by_language
434
435
436def get_connectors_by_type(
437    repo_path: str | Path,
438    connector_type: ConnectorType,
439) -> set[str]:
440    """Get set of all connector IDs for a specific type (source or destination).
441
442    This function reads connector directories to determine which connectors
443    match the specified type based on their metadata.yaml or name prefix.
444
445    Args:
446        repo_path: Path to the Airbyte monorepo
447        connector_type: Type to filter by (source or destination)
448
449    Returns:
450        Set of connector technical names matching the specified type
451
452    Example:
453        >>> source_connectors = get_connectors_by_type(
454        ...     "/path/to/airbyte", ConnectorType.SOURCE
455        ... )
456        >>> "source-postgres" in source_connectors
457        True
458    """
459    repo_path = Path(repo_path)
460    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
461
462    if not connectors_dir.exists():
463        raise ValueError(f"Connectors directory not found: {connectors_dir}")
464
465    type_value = connector_type.value
466    connectors_by_type = set()
467
468    for connector_dir in connectors_dir.iterdir():
469        if not connector_dir.is_dir():
470            continue
471
472        connector_name = connector_dir.name
473
474        # First try to get type from metadata.yaml
475        metadata = get_connector_metadata(repo_path, connector_name)
476        if metadata:
477            metadata_type = metadata.get("connectorType")
478            if metadata_type == type_value:
479                connectors_by_type.add(connector_name)
480                continue
481
482        # Fallback to name prefix detection
483        if connector_name.startswith(f"{type_value}-"):
484            connectors_by_type.add(connector_name)
485
486    return connectors_by_type
487
488
489def get_connectors_by_subtype(
490    repo_path: str | Path,
491    connector_subtype: ConnectorSubtype,
492) -> set[str]:
493    """Get set of all connector IDs for a specific subtype (api, database, file, etc.).
494
495    This function reads connector metadata.yaml files to determine which connectors
496    match the specified subtype.
497
498    Args:
499        repo_path: Path to the Airbyte monorepo
500        connector_subtype: Subtype to filter by (api, database, file, custom)
501
502    Returns:
503        Set of connector technical names matching the specified subtype
504
505    Example:
506        >>> database_connectors = get_connectors_by_subtype(
507        ...     "/path/to/airbyte", ConnectorSubtype.DATABASE
508        ... )
509        >>> "source-postgres" in database_connectors
510        True
511    """
512    repo_path = Path(repo_path)
513    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
514
515    if not connectors_dir.exists():
516        raise ValueError(f"Connectors directory not found: {connectors_dir}")
517
518    subtype_value = connector_subtype.value
519    connectors_by_subtype = set()
520
521    for connector_dir in connectors_dir.iterdir():
522        if not connector_dir.is_dir():
523            continue
524
525        connector_name = connector_dir.name
526        metadata = get_connector_metadata(repo_path, connector_name)
527
528        if metadata:
529            metadata_subtype = metadata.get("connectorSubtype")
530            if metadata_subtype == subtype_value:
531                connectors_by_subtype.add(connector_name)
532
533    return connectors_by_subtype
534
535
536@lru_cache(maxsize=1024)
537def _detect_connector_language(connector_dir: Path, connector_name: str) -> str | None:
538    """Detect the language of a connector based on its file structure.
539
540    Args:
541        connector_dir: Path to the connector directory
542        connector_name: Technical name of the connector
543
544    Returns:
545        Language string (python, java, low-code, manifest-only) or None
546    """
547    # Check for manifest-only (manifest.yaml at root)
548    if (connector_dir / "manifest.yaml").is_file():
549        return "manifest-only"
550
551    # Check for low-code (manifest.yaml in source directory)
552    source_dir = connector_dir / connector_name.replace("-", "_")
553    if (source_dir / "manifest.yaml").is_file():
554        return "low-code"
555
556    # Check for Python (setup.py or pyproject.toml)
557    if (connector_dir / "setup.py").is_file() or (
558        connector_dir / "pyproject.toml"
559    ).is_file():
560        return "python"
561
562    # Check for Java/Kotlin (src/main/java or src/main/kotlin)
563    if (connector_dir / "src" / "main" / "java").exists() or (
564        connector_dir / "src" / "main" / "kotlin"
565    ).exists():
566        return "java"
567
568    return None
569
570
571@lru_cache(maxsize=128)
572def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]:
573    """Get set of connectors using local CDK reference instead of published version.
574
575    This function detects connectors that are configured to use a local CDK
576    checkout rather than a published CDK version. The detection method differs
577    by connector language:
578    - Python: Checks for path-based dependency in pyproject.toml
579    - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
580
581    Args:
582        repo_path: Path to the Airbyte monorepo
583
584    Returns:
585        Set of connector technical names using local CDK reference
586
587    Example:
588        >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte")
589        >>> "destination-motherduck" in local_cdk_connectors
590        True
591    """
592    repo_path = Path(repo_path)
593    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
594
595    if not connectors_dir.exists():
596        raise ValueError(f"Connectors directory not found: {connectors_dir}")
597
598    local_cdk_connectors = set()
599
600    # Iterate through all connector directories
601    for connector_dir in connectors_dir.iterdir():
602        if not connector_dir.is_dir():
603            continue
604
605        connector_name = connector_dir.name
606        detected_language = _detect_connector_language(connector_dir, connector_name)
607
608        # Check for local CDK based on language
609        if detected_language in ("python", "low-code") and _has_local_cdk_python(
610            connector_dir
611        ):
612            # Python connectors: check pyproject.toml for path-based CDK dependency
613            local_cdk_connectors.add(connector_name)
614        elif detected_language == "java" and _has_local_cdk_java(connector_dir):
615            # Java/Kotlin connectors: check build.gradle for useLocalCdk=true
616            local_cdk_connectors.add(connector_name)
617
618    return local_cdk_connectors
619
620
621def _has_local_cdk_python(connector_dir: Path) -> bool:
622    """Check if a Python connector uses local CDK reference.
623
624    Args:
625        connector_dir: Path to the connector directory
626
627    Returns:
628        True if connector uses local CDK reference
629    """
630    pyproject_file = connector_dir / "pyproject.toml"
631    if not pyproject_file.exists():
632        return False
633
634    try:
635        content = pyproject_file.read_text()
636        # Look for path-based airbyte-cdk dependency
637        # Pattern: airbyte-cdk = { path = "..." } or airbyte-cdk = {path = "..."}
638        return bool(re.search(r"airbyte-cdk\s*=\s*\{[^}]*path\s*=", content))
639    except Exception:
640        return False
641
642
643def _has_local_cdk_java(connector_dir: Path) -> bool:
644    """Check if a Java/Kotlin connector uses local CDK reference.
645
646    Mirrors the implementation from airbyte-ci/connectors/connector_ops/connector_ops/utils.py
647
648    Args:
649        connector_dir: Path to the connector directory
650
651    Returns:
652        True if connector uses local CDK reference
653    """
654    # Check both build.gradle and build.gradle.kts
655    for build_file_name in ("build.gradle", "build.gradle.kts"):
656        build_file = connector_dir / build_file_name
657        if not build_file.exists():
658            continue
659
660        try:
661            # Read file and strip inline comments
662            contents = "\n".join(
663                [
664                    line.split("//")[0]  # Remove inline comments
665                    for line in build_file.read_text().split("\n")
666                ]
667            )
668            # Remove spaces and check for useLocalCdk=true
669            contents = contents.replace(" ", "")
670            if "useLocalCdk=true" in contents:
671                return True
672        except Exception:
673            continue
674
675    return False
676
677
678@dataclass
679class ConnectorListResult:
680    """Result of listing connectors with filters."""
681
682    connectors: list[str]
683    count: int
684
685
686def list_connectors(
687    repo_path: str | Path,
688    certified: bool | None = None,
689    modified: bool | None = None,
690    language_filter: set[str] | None = None,
691    language_exclude: set[str] | None = None,
692    connector_type: str | None = None,
693    connector_subtype: str | None = None,
694    base_ref: str | None = None,
695    head_ref: str | None = None,
696    pr_number: int | None = None,
697    pr_owner: str | None = None,
698    pr_repo: str | None = None,
699    gh_token: str | None = None,
700) -> ConnectorListResult:
701    """List connectors in the Airbyte monorepo with flexible filtering.
702
703    This is the core capability function that encapsulates all filtering logic.
704    Both CLI and MCP layers should use this function.
705
706    When `modified` filtering is requested and both `gh_token` and full PR
707    context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the
708    GitHub API is used to determine changed files — this is the most reliable
709    method and avoids shallow-clone pitfalls.  Falls back to local `git diff`
710    when the token is not provided or PR context is incomplete.
711
712    Args:
713        repo_path: Path to the Airbyte monorepo
714        certified: Filter by certification status (True=certified only,
715            False=non-certified only, None=all)
716        modified: Filter by modification status (True=modified only,
717            False=not-modified only, None=all)
718        language_filter: Set of languages to include (python, java, low-code, manifest-only)
719        language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
720        connector_type: Filter by connector type (source, destination, or None for all)
721        connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
722        base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
723        head_ref: Head git reference for modification detection (default: "HEAD")
724        pr_number: Pull request number (enables GitHub API path for modification detection)
725        pr_owner: Repository owner for PR (e.g. "airbytehq")
726        pr_repo: Repository name for PR (e.g. "airbyte")
727        gh_token: GitHub API token. When provided together with PR context,
728            the GitHub API is used instead of local git diff.
729
730    Returns:
731        ConnectorListResult with sorted list of connector names and count
732
733    Raises:
734        ValueError: If both language_filter and language_exclude are provided
735    """
736    # Validate mutual exclusivity of language filters
737    if language_filter is not None and language_exclude is not None:
738        raise ValueError(
739            "Cannot specify both language_filter and language_exclude. "
740            "Only one language filter parameter is accepted."
741        )
742
743    # Start with all connectors
744    result = get_all_connectors(repo_path)
745
746    # Apply certified filter
747    if certified is not None:
748        certified_set = get_certified_connectors(repo_path)
749        if certified:
750            # Include only certified
751            result &= certified_set
752        else:
753            # Include only non-certified
754            result -= certified_set
755
756    # Apply modified filter
757    if modified is not None:
758        changed_set: set[str] | None = None
759
760        # Use GitHub API when token and full PR context are explicitly provided
761        if (
762            gh_token is not None
763            and pr_number is not None
764            and pr_owner is not None
765            and pr_repo is not None
766        ):
767            try:
768                changed_set = set(
769                    get_modified_connectors_from_github(
770                        pr_number, pr_owner, pr_repo, gh_token=gh_token
771                    )
772                )
773            except (GitHubAPIError, requests.RequestException):
774                logger.warning(
775                    "GitHub API call failed for PR %s/%s#%d; "
776                    "falling back to local git diff",
777                    pr_owner,
778                    pr_repo,
779                    pr_number,
780                    exc_info=True,
781                )
782
783        # Fallback to local git diff
784        if changed_set is None:
785            base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH
786            head = head_ref if head_ref is not None else "HEAD"
787            changed_set = set(get_modified_connectors(repo_path, base, head))
788
789        if modified:
790            # Include only modified
791            result &= changed_set
792        else:
793            # Include only not-modified
794            result -= changed_set
795
796    # Apply language include filter
797    if language_filter:
798        # Get connectors for all specified languages and union them
799        lang_result: set[str] = set()
800        for lang in language_filter:
801            lang_result |= get_connectors_by_language(
802                repo_path, ConnectorLanguage(lang)
803            )
804        result &= lang_result
805
806    # Apply language exclude filter
807    if language_exclude:
808        # Get connectors for all specified languages and exclude them
809        for lang in language_exclude:
810            excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang))
811            result -= excluded
812
813    # Apply connector type filter
814    if connector_type is not None:
815        type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type))
816        result &= type_set
817
818    # Apply connector subtype filter
819    if connector_subtype is not None:
820        subtype_set = get_connectors_by_subtype(
821            repo_path, ConnectorSubtype(connector_subtype)
822        )
823        result &= subtype_set
824
825    return ConnectorListResult(
826        connectors=sorted(result),
827        count=len(result),
828    )
CONNECTOR_PATH_PREFIX = 'airbyte-integrations/connectors'
METADATA_FILE_NAME = 'metadata.yaml'
GIT_DEFAULT_BRANCH = 'origin/master'
logger = <Logger airbyte_ops_mcp.airbyte_repo.list_connectors (WARNING)>
class ConnectorSubtype(enum.StrEnum):
33class ConnectorSubtype(StrEnum):
34    """Connector subtypes based on data source category."""
35
36    API = "api"
37    DATABASE = "database"
38    FILE = "file"
39    CUSTOM = "custom"

Connector subtypes based on data source category.

API = <ConnectorSubtype.API: 'api'>
DATABASE = <ConnectorSubtype.DATABASE: 'database'>
FILE = <ConnectorSubtype.FILE: 'file'>
CUSTOM = <ConnectorSubtype.CUSTOM: 'custom'>
def get_modified_connectors( repo_path: str | pathlib.Path, base_ref: str = 'origin/master', head_ref: str = 'HEAD') -> list[str]:
 61def get_modified_connectors(
 62    repo_path: str | Path,
 63    base_ref: str = GIT_DEFAULT_BRANCH,
 64    head_ref: str = "HEAD",
 65) -> list[str]:
 66    """Get list of connector IDs modified according to a local git diff.
 67
 68    This is the *local-only* fallback used when the GitHub API path is not
 69    available (e.g. no PR number or no token).  It uses `git merge-base`
 70    when possible so that only changes introduced on the head branch are
 71    reported, falling back to a plain two-ref diff when the merge-base
 72    cannot be determined (shallow clones without enough history).
 73
 74    Prefer `get_modified_connectors_from_github` whenever a PR number
 75    is known, as it avoids shallow-clone pitfalls entirely.
 76
 77    Args:
 78        repo_path: Path to the Airbyte monorepo.
 79        base_ref: Base git reference to compare against.
 80        head_ref: Head git reference to compare.
 81
 82    Returns:
 83        Sorted list of connector technical names.
 84    """
 85    repo_path = Path(repo_path)
 86
 87    # Try to find the merge-base so the diff only contains changes introduced
 88    # on the head branch, avoiding false positives from unrelated base-branch
 89    # commits.  Falls back to a plain two-ref diff when unavailable.
 90    effective_base = base_ref
 91    try:
 92        mb_result = subprocess.run(
 93            ["git", "merge-base", base_ref, head_ref],
 94            cwd=repo_path,
 95            capture_output=True,
 96            text=True,
 97            check=True,
 98            timeout=30,
 99        )
100        effective_base = mb_result.stdout.strip()
101        logger.debug("Using merge-base %s for diff", effective_base)
102    except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
103        logger.debug(
104            "Could not determine merge-base for %s..%s; falling back to plain diff",
105            base_ref,
106            head_ref,
107        )
108
109    try:
110        result = subprocess.run(
111            ["git", "diff", "--name-only", effective_base, head_ref],
112            cwd=repo_path,
113            capture_output=True,
114            text=True,
115            check=True,
116            timeout=30,
117        )
118        changed_files = result.stdout.strip().split("\n")
119    except subprocess.CalledProcessError as e:
120        raise RuntimeError(f"Failed to get git diff: {e.stderr}") from e
121    except subprocess.TimeoutExpired as e:
122        raise RuntimeError("Git diff command timed out after 30 seconds") from e
123
124    return _extract_connector_names(changed_files)

Get list of connector IDs modified according to a local git diff.

This is the local-only fallback used when the GitHub API path is not available (e.g. no PR number or no token). It uses git merge-base when possible so that only changes introduced on the head branch are reported, falling back to a plain two-ref diff when the merge-base cannot be determined (shallow clones without enough history).

Prefer get_modified_connectors_from_github whenever a PR number is known, as it avoids shallow-clone pitfalls entirely.

Arguments:
  • repo_path: Path to the Airbyte monorepo.
  • base_ref: Base git reference to compare against.
  • head_ref: Head git reference to compare.
Returns:

Sorted list of connector technical names.

def get_merge_base(path: str | pathlib.Path, base_ref: str = 'origin/master') -> str:
127def get_merge_base(path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH) -> str:
128    """Find the merge base of a base git reference and `HEAD`.
129
130    Resolve it once and pass it as `base_ref` to `get_changed_files` and
131    `get_file_at_merge_base` so both compare against the same commit.
132
133    Args:
134        path: A directory inside the git checkout.
135        base_ref: Base git reference, e.g. `origin/master`.
136
137    Returns:
138        The merge base commit's SHA.
139
140    Raises:
141        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
142            git checkout, or git times out.
143        OSError: If git cannot be run or `path` does not exist.
144    """
145    return _git(path, "merge-base", base_ref, "HEAD").strip()

Find the merge base of a base git reference and HEAD.

Resolve it once and pass it as base_ref to get_changed_files and get_file_at_merge_base so both compare against the same commit.

Arguments:
  • path: A directory inside the git checkout.
  • base_ref: Base git reference, e.g. origin/master.
Returns:

The merge base commit's SHA.

Raises:
  • subprocess.SubprocessError: If base_ref is unknown, path is not in a git checkout, or git times out.
  • OSError: If git cannot be run or path does not exist.
def get_changed_files(path: str | pathlib.Path, base_ref: str = 'origin/master') -> list[str]:
148def get_changed_files(
149    path: str | Path, base_ref: str = GIT_DEFAULT_BRANCH
150) -> list[str]:
151    """List the files under a directory that changed since a base git reference.
152
153    Compares the working tree, untracked files included, with the merge base of
154    `base_ref` and `HEAD`, so committed, uncommitted and untracked changes all
155    count. Unlike `get_modified_connectors`, it never falls back to a plain diff,
156    so a caller that skips work based on the result learns when the changes are
157    unknown.
158
159    Args:
160        path: A directory inside the git checkout. Only changes under it are listed.
161        base_ref: Base git reference to compare against, e.g. `origin/master`,
162            or a merge base from `get_merge_base`.
163
164    Returns:
165        Sorted paths relative to `path`, exactly as stored (git does not quote
166        them). A move lists both the old and the new path.
167
168    Raises:
169        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
170            git checkout, or git times out.
171        OSError: If git cannot be run or `path` does not exist.
172    """
173    merge_base = get_merge_base(path, base_ref)
174    # -z prints paths verbatim, NUL-separated; without it git quotes paths with
175    # non-ASCII characters. --no-renames lists both sides of a move, so moving a
176    # file in or out of a directory shows up as a change on both sides.
177    changed = _git(
178        path,
179        "diff",
180        "--name-only",
181        "-z",
182        "--no-renames",
183        "--relative",
184        merge_base,
185        "--",
186    ).split("\0")
187    untracked = _git(path, "ls-files", "-z", "--others", "--exclude-standard").split(
188        "\0"
189    )
190    return sorted({file_path for file_path in changed + untracked if file_path})

List the files under a directory that changed since a base git reference.

Compares the working tree, untracked files included, with the merge base of base_ref and HEAD, so committed, uncommitted and untracked changes all count. Unlike get_modified_connectors, it never falls back to a plain diff, so a caller that skips work based on the result learns when the changes are unknown.

Arguments:
  • path: A directory inside the git checkout. Only changes under it are listed.
  • base_ref: Base git reference to compare against, e.g. origin/master, or a merge base from get_merge_base.
Returns:

Sorted paths relative to path, exactly as stored (git does not quote them). A move lists both the old and the new path.

Raises:
  • subprocess.SubprocessError: If base_ref is unknown, path is not in a git checkout, or git times out.
  • OSError: If git cannot be run or path does not exist.
def get_file_at_merge_base( path: str | pathlib.Path, file_name: str, base_ref: str = 'origin/master') -> str | None:
193def get_file_at_merge_base(
194    path: str | Path, file_name: str, base_ref: str = GIT_DEFAULT_BRANCH
195) -> str | None:
196    """Read a file as it was at the merge base of a base git reference and `HEAD`.
197
198    This is the base side of the diff that `get_changed_files` lists, so the two
199    can be used together to tell what changed inside a file.
200
201    Args:
202        path: A directory inside the git checkout.
203        file_name: The file's path relative to `path`.
204        base_ref: Base git reference to compare against, e.g. `origin/master`,
205            or a merge base from `get_merge_base`.
206
207    Returns:
208        The file's content, or None if it did not exist at the merge base.
209
210    Raises:
211        subprocess.SubprocessError: If `base_ref` is unknown, `path` is not in a
212            git checkout, or git times out.
213        OSError: If git cannot be run or `path` does not exist.
214    """
215    merge_base = get_merge_base(path, base_ref)
216    try:
217        # `<rev>:./<file>` resolves the file relative to `path`, not the repo root.
218        return _git(path, "show", f"{merge_base}:./{file_name}")
219    except subprocess.CalledProcessError:
220        return None

Read a file as it was at the merge base of a base git reference and HEAD.

This is the base side of the diff that get_changed_files lists, so the two can be used together to tell what changed inside a file.

Arguments:
  • path: A directory inside the git checkout.
  • file_name: The file's path relative to path.
  • base_ref: Base git reference to compare against, e.g. origin/master, or a merge base from get_merge_base.
Returns:

The file's content, or None if it did not exist at the merge base.

Raises:
  • subprocess.SubprocessError: If base_ref is unknown, path is not in a git checkout, or git times out.
  • OSError: If git cannot be run or path does not exist.
def format_github_actions_connector_matrix(connectors: list[str]) -> dict[str, list[str]]:
236def format_github_actions_connector_matrix(
237    connectors: list[str],
238) -> dict[str, list[str]]:
239    """Format connector names as a GitHub Actions matrix.
240
241    GitHub Actions matrix generation needs at least one value. Return an empty
242    connector placeholder when no connectors changed so callers can pair this
243    with an `if` guard on the matrix job.
244    """
245    return {"connector": connectors or [""]}

Format connector names as a GitHub Actions matrix.

GitHub Actions matrix generation needs at least one value. Return an empty connector placeholder when no connectors changed so callers can pair this with an if guard on the matrix job.

def get_modified_connectors_from_github(pr_number: int, pr_owner: str, pr_repo: str, gh_token: str) -> list[str]:
248def get_modified_connectors_from_github(
249    pr_number: int,
250    pr_owner: str,
251    pr_repo: str,
252    gh_token: str,
253) -> list[str]:
254    """Get list of connector IDs modified in a PR via the GitHub API.
255
256    This is the preferred path when a PR number is known because it is
257    immune to shallow-clone depth issues and always returns an accurate
258    diff (only the files the PR actually changes).
259
260    Args:
261        pr_number: Pull request number.
262        pr_owner: Repository owner (e.g. `"airbytehq"`).
263        pr_repo: Repository name (e.g. `"airbyte"`).
264        gh_token: GitHub API token (must be provided explicitly).
265
266    Returns:
267        Sorted list of connector technical names.
268
269    Raises:
270        GitHubAPIError: If the API request fails.
271    """
272    logger.info(
273        "Fetching changed files for PR %s/%s#%d via GitHub API",
274        pr_owner,
275        pr_repo,
276        pr_number,
277    )
278    changed_files = get_pr_changed_files(pr_owner, pr_repo, pr_number, token=gh_token)
279    return _extract_connector_names(changed_files)

Get list of connector IDs modified in a PR via the GitHub API.

This is the preferred path when a PR number is known because it is immune to shallow-clone depth issues and always returns an accurate diff (only the files the PR actually changes).

Arguments:
  • pr_number: Pull request number.
  • pr_owner: Repository owner (e.g. "airbytehq").
  • pr_repo: Repository name (e.g. "airbyte").
  • gh_token: GitHub API token (must be provided explicitly).
Returns:

Sorted list of connector technical names.

Raises:
  • GitHubAPIError: If the API request fails.
@lru_cache(maxsize=128)
def get_certified_connectors(repo_path: str | pathlib.Path) -> set[str]:
282@lru_cache(maxsize=128)
283def get_certified_connectors(repo_path: str | Path) -> set[str]:
284    """Get set of all certified connector IDs from metadata.
285
286    This function reads the metadata.yaml file for each connector to determine
287    which connectors have "certified" support level.
288
289    Args:
290        repo_path: Path to the Airbyte monorepo
291
292    Returns:
293        Set of certified connector technical names
294
295    Example:
296        >>> certified = get_certified_connectors("/path/to/airbyte")
297        >>> "source-postgres" in certified
298        True
299    """
300    repo_path = Path(repo_path)
301    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
302
303    if not connectors_dir.exists():
304        raise ValueError(f"Connectors directory not found: {connectors_dir}")
305
306    certified_connectors = set()
307
308    # Iterate through all connector directories
309    for connector_dir in connectors_dir.iterdir():
310        if not connector_dir.is_dir():
311            continue
312
313        metadata_file = connector_dir / METADATA_FILE_NAME
314        if not metadata_file.exists():
315            continue
316
317        # Read metadata to check support level
318        try:
319            import yaml
320
321            with open(metadata_file) as f:
322                metadata = yaml.safe_load(f)
323
324            support_level = metadata.get("data", {}).get("supportLevel")
325            if support_level == "certified":
326                certified_connectors.add(connector_dir.name)
327        except Exception:
328            # Skip connectors with invalid metadata
329            continue
330
331    return certified_connectors

Get set of all certified connector IDs from metadata.

This function reads the metadata.yaml file for each connector to determine which connectors have "certified" support level.

Arguments:
  • repo_path: Path to the Airbyte monorepo
Returns:

Set of certified connector technical names

Example:
>>> certified = get_certified_connectors("/path/to/airbyte")
>>> "source-postgres" in certified
True
def get_all_connectors(repo_path: str | pathlib.Path) -> set[str]:
334def get_all_connectors(repo_path: str | Path) -> set[str]:
335    """Get set of all connector IDs in the repository.
336
337    Args:
338        repo_path: Path to the Airbyte monorepo
339
340    Returns:
341        Set of all connector technical names
342
343    Example:
344        >>> all_connectors = get_all_connectors("/path/to/airbyte")
345        >>> "source-faker" in all_connectors
346        True
347    """
348    repo_path = Path(repo_path)
349    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
350
351    if not connectors_dir.exists():
352        raise ValueError(f"Connectors directory not found: {connectors_dir}")
353
354    return {p.name for p in connectors_dir.iterdir() if p.is_dir()}

Get set of all connector IDs in the repository.

Arguments:
  • repo_path: Path to the Airbyte monorepo
Returns:

Set of all connector technical names

Example:
>>> all_connectors = get_all_connectors("/path/to/airbyte")
>>> "source-faker" in all_connectors
True
def get_connector_metadata( repo_path: str | pathlib.Path, connector_name: str) -> dict[str, typing.Any] | None:
357def get_connector_metadata(
358    repo_path: str | Path,
359    connector_name: str,
360) -> dict[str, Any] | None:
361    """Get metadata for a specific connector.
362
363    Args:
364        repo_path: Path to the Airbyte monorepo
365        connector_name: Technical name of the connector (e.g., "source-faker")
366
367    Returns:
368        The connector's metadata dict (the 'data' section), or None if not found
369
370    Example:
371        >>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker")
372        >>> metadata.get("supportLevel")
373        'certified'
374    """
375    repo_path = Path(repo_path)
376    connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name
377    metadata_file = connector_dir / METADATA_FILE_NAME
378
379    if not metadata_file.exists():
380        return None
381
382    try:
383        with open(metadata_file) as f:
384            metadata = yaml.safe_load(f)
385        return metadata.get("data", {})
386    except Exception:
387        return None

Get metadata for a specific connector.

Arguments:
  • repo_path: Path to the Airbyte monorepo
  • connector_name: Technical name of the connector (e.g., "source-faker")
Returns:

The connector's metadata dict (the 'data' section), or None if not found

Example:
>>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker")
>>> metadata.get("supportLevel")
'certified'
def get_connectors_by_language( repo_path: str | pathlib.Path, language: airbyte_ops_mcp.registry.ConnectorLanguage) -> set[str]:
390def get_connectors_by_language(
391    repo_path: str | Path,
392    language: ConnectorLanguage,
393) -> set[str]:
394    """Get set of all connector IDs for a specific language.
395
396    This function reads connector directories to determine which connectors
397    use the specified language.
398
399    Args:
400        repo_path: Path to the Airbyte monorepo
401        language: Language to filter by
402
403    Returns:
404        Set of connector technical names using the specified language
405
406    Example:
407        >>> python_connectors = get_connectors_by_language(
408        ...     "/path/to/airbyte", ConnectorLanguage.PYTHON
409        ... )
410        >>> "source-faker" in python_connectors
411        True
412    """
413    repo_path = Path(repo_path)
414    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
415
416    if not connectors_dir.exists():
417        raise ValueError(f"Connectors directory not found: {connectors_dir}")
418
419    language_value = language.value
420    connectors_by_language = set()
421
422    # Iterate through all connector directories
423    for connector_dir in connectors_dir.iterdir():
424        if not connector_dir.is_dir():
425            continue
426
427        # Determine language based on file structure
428        connector_name = connector_dir.name
429        detected_language = _detect_connector_language(connector_dir, connector_name)
430
431        if detected_language == language_value:
432            connectors_by_language.add(connector_dir.name)
433
434    return connectors_by_language

Get set of all connector IDs for a specific language.

This function reads connector directories to determine which connectors use the specified language.

Arguments:
  • repo_path: Path to the Airbyte monorepo
  • language: Language to filter by
Returns:

Set of connector technical names using the specified language

Example:
>>> python_connectors = get_connectors_by_language(
...     "/path/to/airbyte", ConnectorLanguage.PYTHON
... )
>>> "source-faker" in python_connectors
True
def get_connectors_by_type( repo_path: str | pathlib.Path, connector_type: airbyte_ops_mcp.registry.ConnectorType) -> set[str]:
437def get_connectors_by_type(
438    repo_path: str | Path,
439    connector_type: ConnectorType,
440) -> set[str]:
441    """Get set of all connector IDs for a specific type (source or destination).
442
443    This function reads connector directories to determine which connectors
444    match the specified type based on their metadata.yaml or name prefix.
445
446    Args:
447        repo_path: Path to the Airbyte monorepo
448        connector_type: Type to filter by (source or destination)
449
450    Returns:
451        Set of connector technical names matching the specified type
452
453    Example:
454        >>> source_connectors = get_connectors_by_type(
455        ...     "/path/to/airbyte", ConnectorType.SOURCE
456        ... )
457        >>> "source-postgres" in source_connectors
458        True
459    """
460    repo_path = Path(repo_path)
461    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
462
463    if not connectors_dir.exists():
464        raise ValueError(f"Connectors directory not found: {connectors_dir}")
465
466    type_value = connector_type.value
467    connectors_by_type = set()
468
469    for connector_dir in connectors_dir.iterdir():
470        if not connector_dir.is_dir():
471            continue
472
473        connector_name = connector_dir.name
474
475        # First try to get type from metadata.yaml
476        metadata = get_connector_metadata(repo_path, connector_name)
477        if metadata:
478            metadata_type = metadata.get("connectorType")
479            if metadata_type == type_value:
480                connectors_by_type.add(connector_name)
481                continue
482
483        # Fallback to name prefix detection
484        if connector_name.startswith(f"{type_value}-"):
485            connectors_by_type.add(connector_name)
486
487    return connectors_by_type

Get set of all connector IDs for a specific type (source or destination).

This function reads connector directories to determine which connectors match the specified type based on their metadata.yaml or name prefix.

Arguments:
  • repo_path: Path to the Airbyte monorepo
  • connector_type: Type to filter by (source or destination)
Returns:

Set of connector technical names matching the specified type

Example:
>>> source_connectors = get_connectors_by_type(
...     "/path/to/airbyte", ConnectorType.SOURCE
... )
>>> "source-postgres" in source_connectors
True
def get_connectors_by_subtype( repo_path: str | pathlib.Path, connector_subtype: ConnectorSubtype) -> set[str]:
490def get_connectors_by_subtype(
491    repo_path: str | Path,
492    connector_subtype: ConnectorSubtype,
493) -> set[str]:
494    """Get set of all connector IDs for a specific subtype (api, database, file, etc.).
495
496    This function reads connector metadata.yaml files to determine which connectors
497    match the specified subtype.
498
499    Args:
500        repo_path: Path to the Airbyte monorepo
501        connector_subtype: Subtype to filter by (api, database, file, custom)
502
503    Returns:
504        Set of connector technical names matching the specified subtype
505
506    Example:
507        >>> database_connectors = get_connectors_by_subtype(
508        ...     "/path/to/airbyte", ConnectorSubtype.DATABASE
509        ... )
510        >>> "source-postgres" in database_connectors
511        True
512    """
513    repo_path = Path(repo_path)
514    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
515
516    if not connectors_dir.exists():
517        raise ValueError(f"Connectors directory not found: {connectors_dir}")
518
519    subtype_value = connector_subtype.value
520    connectors_by_subtype = set()
521
522    for connector_dir in connectors_dir.iterdir():
523        if not connector_dir.is_dir():
524            continue
525
526        connector_name = connector_dir.name
527        metadata = get_connector_metadata(repo_path, connector_name)
528
529        if metadata:
530            metadata_subtype = metadata.get("connectorSubtype")
531            if metadata_subtype == subtype_value:
532                connectors_by_subtype.add(connector_name)
533
534    return connectors_by_subtype

Get set of all connector IDs for a specific subtype (api, database, file, etc.).

This function reads connector metadata.yaml files to determine which connectors match the specified subtype.

Arguments:
  • repo_path: Path to the Airbyte monorepo
  • connector_subtype: Subtype to filter by (api, database, file, custom)
Returns:

Set of connector technical names matching the specified subtype

Example:
>>> database_connectors = get_connectors_by_subtype(
...     "/path/to/airbyte", ConnectorSubtype.DATABASE
... )
>>> "source-postgres" in database_connectors
True
@lru_cache(maxsize=128)
def get_connectors_with_local_cdk(repo_path: str | pathlib.Path) -> set[str]:
572@lru_cache(maxsize=128)
573def get_connectors_with_local_cdk(repo_path: str | Path) -> set[str]:
574    """Get set of connectors using local CDK reference instead of published version.
575
576    This function detects connectors that are configured to use a local CDK
577    checkout rather than a published CDK version. The detection method differs
578    by connector language:
579    - Python: Checks for path-based dependency in pyproject.toml
580    - Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
581
582    Args:
583        repo_path: Path to the Airbyte monorepo
584
585    Returns:
586        Set of connector technical names using local CDK reference
587
588    Example:
589        >>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte")
590        >>> "destination-motherduck" in local_cdk_connectors
591        True
592    """
593    repo_path = Path(repo_path)
594    connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
595
596    if not connectors_dir.exists():
597        raise ValueError(f"Connectors directory not found: {connectors_dir}")
598
599    local_cdk_connectors = set()
600
601    # Iterate through all connector directories
602    for connector_dir in connectors_dir.iterdir():
603        if not connector_dir.is_dir():
604            continue
605
606        connector_name = connector_dir.name
607        detected_language = _detect_connector_language(connector_dir, connector_name)
608
609        # Check for local CDK based on language
610        if detected_language in ("python", "low-code") and _has_local_cdk_python(
611            connector_dir
612        ):
613            # Python connectors: check pyproject.toml for path-based CDK dependency
614            local_cdk_connectors.add(connector_name)
615        elif detected_language == "java" and _has_local_cdk_java(connector_dir):
616            # Java/Kotlin connectors: check build.gradle for useLocalCdk=true
617            local_cdk_connectors.add(connector_name)
618
619    return local_cdk_connectors

Get set of connectors using local CDK reference instead of published version.

This function detects connectors that are configured to use a local CDK checkout rather than a published CDK version. The detection method differs by connector language:

  • Python: Checks for path-based dependency in pyproject.toml
  • Java/Kotlin: Checks for useLocalCdk=true in build.gradle files
Arguments:
  • repo_path: Path to the Airbyte monorepo
Returns:

Set of connector technical names using local CDK reference

Example:
>>> local_cdk_connectors = get_connectors_with_local_cdk("/path/to/airbyte")
>>> "destination-motherduck" in local_cdk_connectors
True
@dataclass
class ConnectorListResult:
679@dataclass
680class ConnectorListResult:
681    """Result of listing connectors with filters."""
682
683    connectors: list[str]
684    count: int

Result of listing connectors with filters.

ConnectorListResult(connectors: list[str], count: int)
connectors: list[str]
count: int
def list_connectors( repo_path: str | pathlib.Path, certified: bool | None = None, modified: bool | None = None, language_filter: set[str] | None = None, language_exclude: set[str] | None = None, connector_type: str | None = None, connector_subtype: str | None = None, base_ref: str | None = None, head_ref: str | None = None, pr_number: int | None = None, pr_owner: str | None = None, pr_repo: str | None = None, gh_token: str | None = None) -> ConnectorListResult:
687def list_connectors(
688    repo_path: str | Path,
689    certified: bool | None = None,
690    modified: bool | None = None,
691    language_filter: set[str] | None = None,
692    language_exclude: set[str] | None = None,
693    connector_type: str | None = None,
694    connector_subtype: str | None = None,
695    base_ref: str | None = None,
696    head_ref: str | None = None,
697    pr_number: int | None = None,
698    pr_owner: str | None = None,
699    pr_repo: str | None = None,
700    gh_token: str | None = None,
701) -> ConnectorListResult:
702    """List connectors in the Airbyte monorepo with flexible filtering.
703
704    This is the core capability function that encapsulates all filtering logic.
705    Both CLI and MCP layers should use this function.
706
707    When `modified` filtering is requested and both `gh_token` and full PR
708    context (`pr_number`, `pr_owner`, `pr_repo`) are provided, the
709    GitHub API is used to determine changed files — this is the most reliable
710    method and avoids shallow-clone pitfalls.  Falls back to local `git diff`
711    when the token is not provided or PR context is incomplete.
712
713    Args:
714        repo_path: Path to the Airbyte monorepo
715        certified: Filter by certification status (True=certified only,
716            False=non-certified only, None=all)
717        modified: Filter by modification status (True=modified only,
718            False=not-modified only, None=all)
719        language_filter: Set of languages to include (python, java, low-code, manifest-only)
720        language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
721        connector_type: Filter by connector type (source, destination, or None for all)
722        connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
723        base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
724        head_ref: Head git reference for modification detection (default: "HEAD")
725        pr_number: Pull request number (enables GitHub API path for modification detection)
726        pr_owner: Repository owner for PR (e.g. "airbytehq")
727        pr_repo: Repository name for PR (e.g. "airbyte")
728        gh_token: GitHub API token. When provided together with PR context,
729            the GitHub API is used instead of local git diff.
730
731    Returns:
732        ConnectorListResult with sorted list of connector names and count
733
734    Raises:
735        ValueError: If both language_filter and language_exclude are provided
736    """
737    # Validate mutual exclusivity of language filters
738    if language_filter is not None and language_exclude is not None:
739        raise ValueError(
740            "Cannot specify both language_filter and language_exclude. "
741            "Only one language filter parameter is accepted."
742        )
743
744    # Start with all connectors
745    result = get_all_connectors(repo_path)
746
747    # Apply certified filter
748    if certified is not None:
749        certified_set = get_certified_connectors(repo_path)
750        if certified:
751            # Include only certified
752            result &= certified_set
753        else:
754            # Include only non-certified
755            result -= certified_set
756
757    # Apply modified filter
758    if modified is not None:
759        changed_set: set[str] | None = None
760
761        # Use GitHub API when token and full PR context are explicitly provided
762        if (
763            gh_token is not None
764            and pr_number is not None
765            and pr_owner is not None
766            and pr_repo is not None
767        ):
768            try:
769                changed_set = set(
770                    get_modified_connectors_from_github(
771                        pr_number, pr_owner, pr_repo, gh_token=gh_token
772                    )
773                )
774            except (GitHubAPIError, requests.RequestException):
775                logger.warning(
776                    "GitHub API call failed for PR %s/%s#%d; "
777                    "falling back to local git diff",
778                    pr_owner,
779                    pr_repo,
780                    pr_number,
781                    exc_info=True,
782                )
783
784        # Fallback to local git diff
785        if changed_set is None:
786            base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH
787            head = head_ref if head_ref is not None else "HEAD"
788            changed_set = set(get_modified_connectors(repo_path, base, head))
789
790        if modified:
791            # Include only modified
792            result &= changed_set
793        else:
794            # Include only not-modified
795            result -= changed_set
796
797    # Apply language include filter
798    if language_filter:
799        # Get connectors for all specified languages and union them
800        lang_result: set[str] = set()
801        for lang in language_filter:
802            lang_result |= get_connectors_by_language(
803                repo_path, ConnectorLanguage(lang)
804            )
805        result &= lang_result
806
807    # Apply language exclude filter
808    if language_exclude:
809        # Get connectors for all specified languages and exclude them
810        for lang in language_exclude:
811            excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang))
812            result -= excluded
813
814    # Apply connector type filter
815    if connector_type is not None:
816        type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type))
817        result &= type_set
818
819    # Apply connector subtype filter
820    if connector_subtype is not None:
821        subtype_set = get_connectors_by_subtype(
822            repo_path, ConnectorSubtype(connector_subtype)
823        )
824        result &= subtype_set
825
826    return ConnectorListResult(
827        connectors=sorted(result),
828        count=len(result),
829    )

List connectors in the Airbyte monorepo with flexible filtering.

This is the core capability function that encapsulates all filtering logic. Both CLI and MCP layers should use this function.

When modified filtering is requested and both gh_token and full PR context (pr_number, pr_owner, pr_repo) are provided, the GitHub API is used to determine changed files — this is the most reliable method and avoids shallow-clone pitfalls. Falls back to local git diff when the token is not provided or PR context is incomplete.

Arguments:
  • repo_path: Path to the Airbyte monorepo
  • certified: Filter by certification status (True=certified only, False=non-certified only, None=all)
  • modified: Filter by modification status (True=modified only, False=not-modified only, None=all)
  • language_filter: Set of languages to include (python, java, low-code, manifest-only)
  • language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
  • connector_type: Filter by connector type (source, destination, or None for all)
  • connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
  • base_ref: Base git reference for modification detection (default: GIT_DEFAULT_BRANCH, e.g. "origin/master")
  • head_ref: Head git reference for modification detection (default: "HEAD")
  • pr_number: Pull request number (enables GitHub API path for modification detection)
  • pr_owner: Repository owner for PR (e.g. "airbytehq")
  • pr_repo: Repository name for PR (e.g. "airbyte")
  • gh_token: GitHub API token. When provided together with PR context, the GitHub API is used instead of local git diff.
Returns:

ConnectorListResult with sorted list of connector names and count

Raises:
  • ValueError: If both language_filter and language_exclude are provided