airbyte.destinations.util

  1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
  2"""Destination utilities.
  3
  4For usage examples, see the `airbyte.destinations` module documentation.
  5"""
  6
  7from __future__ import annotations
  8
  9from typing import TYPE_CHECKING, Any
 10
 11from airbyte._executors.util import get_connector_executor
 12from airbyte.destinations.base import Destination
 13
 14
 15if TYPE_CHECKING:
 16    from pathlib import Path
 17
 18    from airbyte.callbacks import ConfigChangeCallback
 19
 20
 21def get_destination(  # noqa: PLR0913 # Too many arguments
 22    name: str,
 23    config: dict[str, Any] | None = None,
 24    *,
 25    config_change_callback: ConfigChangeCallback | None = None,
 26    version: str | None = None,
 27    use_python: bool | Path | str | None = None,
 28    pip_url: str | None = None,
 29    local_executable: Path | str | None = None,
 30    docker_image: str | bool | None = None,
 31    use_host_network: bool = False,
 32    install_if_missing: bool = True,
 33    install_root: Path | None = None,
 34    no_executor: bool = False,
 35) -> Destination:
 36    """Get a connector by name and version.
 37
 38    Args:
 39        name: connector name
 40        config: connector config - if not provided, you need to set it later via the set_config
 41            method.
 42        config_change_callback: callback function to be called when the connector config changes.
 43        streams: list of stream names to select for reading. If set to "*", all streams will be
 44            selected. If not provided, you can set it later via the `select_streams()` or
 45            `select_all_streams()` method.
 46        version: connector version - if not provided, the currently installed version will be used.
 47            If no version is installed, the latest available version will be used. The version can
 48            also be set to "latest" to force the use of the latest available version.
 49        use_python: (Optional.) Python interpreter specification:
 50            - True: Use current Python interpreter. (Inferred if `pip_url` is set.)
 51            - False: Use Docker instead.
 52            - Path: Use interpreter at this path.
 53            - str: Use specific Python version. E.g. "3.11" or "3.11.10". If the version is not yet
 54                installed, it will be installed by uv. (This generally adds less than 3 seconds
 55                to install times.)
 56        pip_url: connector pip URL - if not provided, the pip url will be inferred from the
 57            connector name.
 58        local_executable: If set, the connector will be assumed to already be installed and will be
 59            executed using this path or executable name. Otherwise, the connector will be installed
 60            automatically in a virtual environment.
 61        docker_image: If set, the connector will be executed using Docker. You can specify `True`
 62            to use the default image for the connector, or you can specify a custom image name.
 63            If `version` is specified and your image name does not already contain a tag
 64            (e.g. `my-image:latest`), the version will be appended as a tag (e.g. `my-image:0.1.0`).
 65        use_host_network: If set, along with docker_image, the connector will be executed using
 66            the host network. This is useful for connectors that need to access resources on
 67            the host machine, such as a local database. This parameter is ignored when
 68            `docker_image` is not set.
 69        install_if_missing: Whether to install the connector if it is not available locally. This
 70            parameter is ignored when local_executable is set.
 71        install_root: (Optional.) The root directory where the virtual environment will be
 72            created. If not provided, the current working directory will be used.
 73        no_executor: If True, use NoOpExecutor which fetches specs from the registry without
 74            local installation. This is useful for scenarios where you need to validate
 75            configurations but don't need to run the connector locally (e.g., deploying to Cloud).
 76    """
 77    executor = get_connector_executor(
 78        name=name,
 79        version=version,
 80        use_python=use_python,
 81        pip_url=pip_url,
 82        local_executable=local_executable,
 83        docker_image=docker_image,
 84        use_host_network=use_host_network,
 85        install_if_missing=install_if_missing,
 86        install_root=install_root,
 87        no_executor=no_executor,
 88    )
 89
 90    return Destination(
 91        name=name,
 92        config=config,
 93        config_change_callback=config_change_callback,
 94        executor=executor,
 95    )
 96
 97
 98def get_noop_destination(
 99    *,
100    install_if_missing: bool = True,
101) -> Destination:
102    """Get a devnull (no-op) destination.
103
104    This is useful for performance benchmarking of sources, without
105    adding the overhead of writing data to a real destination.
106    """
107    return get_destination(
108        "destination-dev-null",
109        config={
110            "test_destination": {
111                "test_destination_type": "SILENT",
112            }
113        },
114        docker_image=True,
115        install_if_missing=install_if_missing,
116    )
117
118
119__all__ = [
120    "get_destination",
121    "get_noop_destination",
122]