airbyte_cdk.sources.file_based.config.validate_config_transfer_modes

View Source

 1#
 2# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 3#
 4
 5from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
 6    AbstractFileBasedSpec,
 7    DeliverRawFiles,
 8)
 9from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
10
11DELIVERY_TYPE_KEY = "delivery_type"
12DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = "use_permissions_transfer"
13DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = "use_file_transfer"
14PRESERVE_DIRECTORY_STRUCTURE_KEY = "preserve_directory_structure"
15INCLUDE_IDENTITIES_STREAM_KEY = "include_identities_stream"
16
17
18def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
19    """Returns `True` if the configuration uses file transfer mode."""
20    return (
21        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
22        and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
23    )
24
25
26def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
27    """
28    Determines whether to preserve directory structure during file transfer.
29
30    When enabled, files maintain their subdirectory paths in the destination.
31    When disabled, files are flattened to the root of the destination.
32
33    Args:
34        parsed_config: The parsed configuration containing delivery method settings
35
36    Returns:
37        True if directory structure should be preserved (default), False otherwise
38    """
39    if (
40        use_file_transfer(parsed_config)
41        and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
42        and isinstance(parsed_config.delivery_method, DeliverRawFiles)
43    ):
44        return parsed_config.delivery_method.preserve_directory_structure
45    return True
46
47
48def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
49    """
50    Determines whether to use permissions transfer to sync ACLs and Identities
51
52    Args:
53        parsed_config: The parsed configuration containing delivery method settings
54
55    Returns:
56        True if permissions transfer should be enabled, False otherwise
57    """
58    return (
59        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
60        and parsed_config.delivery_method.delivery_type
61        == DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
62    )
63
64
65def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
66    """
67    There are scenarios where user may not have access to identities but still is valuable to get ACLs
68
69    Args:
70        parsed_config: The parsed configuration containing delivery method settings
71
72    Returns:
73        True if we should include Identities stream.
74    """
75    if (
76        use_permissions_transfer(parsed_config)
77        and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
78        and isinstance(parsed_config.delivery_method, DeliverPermissions)
79    ):
80        return parsed_config.delivery_method.include_identities_stream
81    return False

DELIVERY_TYPE_KEY = 'delivery_type'

DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = 'use_permissions_transfer'

DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = 'use_file_transfer'

PRESERVE_DIRECTORY_STRUCTURE_KEY = 'preserve_directory_structure'

INCLUDE_IDENTITIES_STREAM_KEY = 'include_identities_stream'

def use_file_transfer( parsed_config: airbyte_cdk.sources.file_based.AbstractFileBasedSpec) -> bool: View Source

19def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
20    """Returns `True` if the configuration uses file transfer mode."""
21    return (
22        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
23        and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
24    )

Returns True if the configuration uses file transfer mode.

def preserve_directory_structure( parsed_config: airbyte_cdk.sources.file_based.AbstractFileBasedSpec) -> bool: View Source

27def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
28    """
29    Determines whether to preserve directory structure during file transfer.
30
31    When enabled, files maintain their subdirectory paths in the destination.
32    When disabled, files are flattened to the root of the destination.
33
34    Args:
35        parsed_config: The parsed configuration containing delivery method settings
36
37    Returns:
38        True if directory structure should be preserved (default), False otherwise
39    """
40    if (
41        use_file_transfer(parsed_config)
42        and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
43        and isinstance(parsed_config.delivery_method, DeliverRawFiles)
44    ):
45        return parsed_config.delivery_method.preserve_directory_structure
46    return True

Determines whether to preserve directory structure during file transfer.

When enabled, files maintain their subdirectory paths in the destination. When disabled, files are flattened to the root of the destination.

Arguments:

parsed_config: The parsed configuration containing delivery method settings

Returns:

True if directory structure should be preserved (default), False otherwise

def use_permissions_transfer( parsed_config: airbyte_cdk.sources.file_based.AbstractFileBasedSpec) -> bool: View Source

49def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
50    """
51    Determines whether to use permissions transfer to sync ACLs and Identities
52
53    Args:
54        parsed_config: The parsed configuration containing delivery method settings
55
56    Returns:
57        True if permissions transfer should be enabled, False otherwise
58    """
59    return (
60        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
61        and parsed_config.delivery_method.delivery_type
62        == DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
63    )

Determines whether to use permissions transfer to sync ACLs and Identities

Arguments:

parsed_config: The parsed configuration containing delivery method settings

Returns:

True if permissions transfer should be enabled, False otherwise

def include_identities_stream( parsed_config: airbyte_cdk.sources.file_based.AbstractFileBasedSpec) -> bool: View Source

66def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
67    """
68    There are scenarios where user may not have access to identities but still is valuable to get ACLs
69
70    Args:
71        parsed_config: The parsed configuration containing delivery method settings
72
73    Returns:
74        True if we should include Identities stream.
75    """
76    if (
77        use_permissions_transfer(parsed_config)
78        and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
79        and isinstance(parsed_config.delivery_method, DeliverPermissions)
80    ):
81        return parsed_config.delivery_method.include_identities_stream
82    return False

There are scenarios where user may not have access to identities but still is valuable to get ACLs

Arguments:

parsed_config: The parsed configuration containing delivery method settings

Returns:

True if we should include Identities stream.