airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver

View Source

  1#
  2# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  3#
  4
  5import re
  6from copy import deepcopy
  7from typing import Any, Dict, Mapping, Set, Tuple, Union
  8
  9from airbyte_cdk.sources.declarative.parsers.custom_exceptions import (
 10    CircularReferenceException,
 11    UndefinedReferenceException,
 12)
 13
 14REF_TAG = "$ref"
 15
 16# Manifest fields whose values are connector config rather than components. Any string starting with
 17# `#/` is a reference, so without this a config value shaped like a pointer would be resolved - or, if
 18# it resolves to nothing, would raise during preprocessing and break every command.
 19_FIELDS_HOLDING_CONFIG_VALUES = frozenset({"config_overrides"})
 20
 21
 22class ManifestReferenceResolver:
 23    """
 24    An incoming manifest can contain references to values previously defined.
 25    This parser will dereference these values to produce a complete ConnectionDefinition.
 26
 27    References can be defined using a #/<arg> string.
 28    ```
 29    key: 1234
 30    reference: "#/key"
 31    ```
 32    will produce the following definition:
 33    ```
 34    key: 1234
 35    reference: 1234
 36    ```
 37    This also works with objects:
 38    ```
 39    key_value_pairs:
 40      k1: v1
 41      k2: v2
 42    same_key_value_pairs: "#/key_value_pairs"
 43    ```
 44    will produce the following definition:
 45    ```
 46    key_value_pairs:
 47      k1: v1
 48      k2: v2
 49    same_key_value_pairs:
 50      k1: v1
 51      k2: v2
 52    ```
 53
 54    The $ref keyword can be used to refer to an object and enhance it with addition key-value pairs
 55    ```
 56    key_value_pairs:
 57      k1: v1
 58      k2: v2
 59    same_key_value_pairs:
 60      $ref: "#/key_value_pairs"
 61      k3: v3
 62    ```
 63    will produce the following definition:
 64    ```
 65    key_value_pairs:
 66      k1: v1
 67      k2: v2
 68    same_key_value_pairs:
 69      k1: v1
 70      k2: v2
 71      k3: v3
 72    ```
 73
 74    References can also point to nested values.
 75    Nested references are ambiguous because one could define a key containing with `.`
 76    in this example, we want to refer to the limit key in the dict object:
 77    ```
 78    dict:
 79        limit: 50
 80    limit_ref: "#/dict/limit"
 81    ```
 82    will produce the following definition:
 83    ```
 84    dict
 85        limit: 50
 86    limit-ref: 50
 87    ```
 88
 89    whereas here we want to access the `nested/path` value.
 90    ```
 91    nested:
 92        path: "first one"
 93    nested/path: "uh oh"
 94    value: "#/nested/path
 95    ```
 96    will produce the following definition:
 97    ```
 98    nested:
 99        path: "first one"
100    nested/path: "uh oh"
101    value: "uh oh"
102    ```
103
104    to resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
105    until we find a key with the given path, or until there is nothing to traverse.
106    """
107
108    def preprocess_manifest(self, manifest: Mapping[str, Any]) -> Dict[str, Any]:
109        """
110        :param manifest: incoming manifest that could have references to previously defined components
111        :return:
112        """
113        return self._evaluate_node(manifest, manifest, set())  # type: ignore[no-any-return]
114
115    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:
116        if isinstance(node, dict):
117            evaluated_dict = {
118                k: deepcopy(v)
119                if k in _FIELDS_HOLDING_CONFIG_VALUES
120                else self._evaluate_node(v, manifest, visited)
121                for k, v in node.items()
122                if not self._is_ref_key(k)
123            }
124            if REF_TAG in node:
125                # The node includes a $ref key, so we splat the referenced value(s) into the evaluated dict
126                evaluated_ref = self._evaluate_node(node[REF_TAG], manifest, visited)
127                if not isinstance(evaluated_ref, dict):
128                    return evaluated_ref
129                else:
130                    # The values defined on the component take precedence over the reference values
131                    return evaluated_ref | evaluated_dict
132            else:
133                return evaluated_dict
134        elif isinstance(node, list):
135            return [self._evaluate_node(v, manifest, visited) for v in node]
136        elif self._is_ref(node):
137            if node in visited:
138                raise CircularReferenceException(node)
139            visited.add(node)
140            ret = self._evaluate_node(self._lookup_ref_value(node, manifest), manifest, visited)
141            visited.remove(node)
142            return ret
143        else:
144            return node
145
146    def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
147        ref_match = re.match(r"#/(.*)", ref)
148        if not ref_match:
149            raise ValueError(f"Invalid reference format {ref}")
150        try:
151            path = ref_match.groups()[0]
152            return self._read_ref_value(path, manifest)
153        except (AttributeError, KeyError, IndexError):
154            raise UndefinedReferenceException(path, ref)
155
156    @staticmethod
157    def _is_ref(node: Any) -> bool:
158        return isinstance(node, str) and node.startswith("#/")
159
160    @staticmethod
161    def _is_ref_key(key: str) -> bool:
162        return bool(key == REF_TAG)
163
164    @staticmethod
165    def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:
166        """
167        Read the value at the referenced location of the manifest.
168
169        References are ambiguous because one could define a key containing `/`
170        In this example, we want to refer to the `limit` key in the `dict` object:
171            dict:
172                limit: 50
173            limit_ref: "#/dict/limit"
174
175        Whereas here we want to access the `nested/path` value.
176          nested:
177            path: "first one"
178          nested/path: "uh oh"
179          value: "#/nested/path"
180
181        To resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
182        until we find a key with the given path, or until there is nothing to traverse.
183
184        Consider the path foo/bar/baz. To resolve the ambiguity, we first try 'foo/bar/baz' in its entirety as a top-level key. If this
185        fails, we try 'foo' as the top-level key, and if this succeeds, pass 'bar/baz' on as the key to be tried at the next level.
186        """
187        while ref:
188            try:
189                return manifest_node[ref]
190            except (KeyError, TypeError):
191                head, ref = _parse_path(ref)
192                manifest_node = manifest_node[head]  # type: ignore # Couldn't figure out how to fix this since manifest_node can get reassigned into other types like lists
193        return manifest_node
194
195
196def _parse_path(ref: str) -> Tuple[Union[str, int], str]:
197    """
198    Return the next path component, together with the rest of the path.
199
200    A path component may be a string key, or an int index.
201
202    >>> _parse_path("foo/bar")
203    "foo", "bar"
204    >>> _parse_path("foo/7/8/bar")
205    "foo", "7/8/bar"
206    >>> _parse_path("7/8/bar")
207    7, "8/bar"
208    >>> _parse_path("8/bar")
209    8, "bar"
210    >>> _parse_path("8foo/bar")
211    "8foo", "bar"
212    """
213    match = re.match(r"([^/]*)/?(.*)", ref)
214    if match:
215        first, rest = match.groups()
216        try:
217            return int(first), rest
218        except ValueError:
219            return first, rest
220    else:
221        raise ValueError(f"Invalid path {ref} specified")

REF_TAG = '$ref'

class ManifestReferenceResolver: View Source

 23class ManifestReferenceResolver:
 24    """
 25    An incoming manifest can contain references to values previously defined.
 26    This parser will dereference these values to produce a complete ConnectionDefinition.
 27
 28    References can be defined using a #/<arg> string.
 29    ```
 30    key: 1234
 31    reference: "#/key"
 32    ```
 33    will produce the following definition:
 34    ```
 35    key: 1234
 36    reference: 1234
 37    ```
 38    This also works with objects:
 39    ```
 40    key_value_pairs:
 41      k1: v1
 42      k2: v2
 43    same_key_value_pairs: "#/key_value_pairs"
 44    ```
 45    will produce the following definition:
 46    ```
 47    key_value_pairs:
 48      k1: v1
 49      k2: v2
 50    same_key_value_pairs:
 51      k1: v1
 52      k2: v2
 53    ```
 54
 55    The $ref keyword can be used to refer to an object and enhance it with addition key-value pairs
 56    ```
 57    key_value_pairs:
 58      k1: v1
 59      k2: v2
 60    same_key_value_pairs:
 61      $ref: "#/key_value_pairs"
 62      k3: v3
 63    ```
 64    will produce the following definition:
 65    ```
 66    key_value_pairs:
 67      k1: v1
 68      k2: v2
 69    same_key_value_pairs:
 70      k1: v1
 71      k2: v2
 72      k3: v3
 73    ```
 74
 75    References can also point to nested values.
 76    Nested references are ambiguous because one could define a key containing with `.`
 77    in this example, we want to refer to the limit key in the dict object:
 78    ```
 79    dict:
 80        limit: 50
 81    limit_ref: "#/dict/limit"
 82    ```
 83    will produce the following definition:
 84    ```
 85    dict
 86        limit: 50
 87    limit-ref: 50
 88    ```
 89
 90    whereas here we want to access the `nested/path` value.
 91    ```
 92    nested:
 93        path: "first one"
 94    nested/path: "uh oh"
 95    value: "#/nested/path
 96    ```
 97    will produce the following definition:
 98    ```
 99    nested:
100        path: "first one"
101    nested/path: "uh oh"
102    value: "uh oh"
103    ```
104
105    to resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
106    until we find a key with the given path, or until there is nothing to traverse.
107    """
108
109    def preprocess_manifest(self, manifest: Mapping[str, Any]) -> Dict[str, Any]:
110        """
111        :param manifest: incoming manifest that could have references to previously defined components
112        :return:
113        """
114        return self._evaluate_node(manifest, manifest, set())  # type: ignore[no-any-return]
115
116    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:
117        if isinstance(node, dict):
118            evaluated_dict = {
119                k: deepcopy(v)
120                if k in _FIELDS_HOLDING_CONFIG_VALUES
121                else self._evaluate_node(v, manifest, visited)
122                for k, v in node.items()
123                if not self._is_ref_key(k)
124            }
125            if REF_TAG in node:
126                # The node includes a $ref key, so we splat the referenced value(s) into the evaluated dict
127                evaluated_ref = self._evaluate_node(node[REF_TAG], manifest, visited)
128                if not isinstance(evaluated_ref, dict):
129                    return evaluated_ref
130                else:
131                    # The values defined on the component take precedence over the reference values
132                    return evaluated_ref | evaluated_dict
133            else:
134                return evaluated_dict
135        elif isinstance(node, list):
136            return [self._evaluate_node(v, manifest, visited) for v in node]
137        elif self._is_ref(node):
138            if node in visited:
139                raise CircularReferenceException(node)
140            visited.add(node)
141            ret = self._evaluate_node(self._lookup_ref_value(node, manifest), manifest, visited)
142            visited.remove(node)
143            return ret
144        else:
145            return node
146
147    def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
148        ref_match = re.match(r"#/(.*)", ref)
149        if not ref_match:
150            raise ValueError(f"Invalid reference format {ref}")
151        try:
152            path = ref_match.groups()[0]
153            return self._read_ref_value(path, manifest)
154        except (AttributeError, KeyError, IndexError):
155            raise UndefinedReferenceException(path, ref)
156
157    @staticmethod
158    def _is_ref(node: Any) -> bool:
159        return isinstance(node, str) and node.startswith("#/")
160
161    @staticmethod
162    def _is_ref_key(key: str) -> bool:
163        return bool(key == REF_TAG)
164
165    @staticmethod
166    def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:
167        """
168        Read the value at the referenced location of the manifest.
169
170        References are ambiguous because one could define a key containing `/`
171        In this example, we want to refer to the `limit` key in the `dict` object:
172            dict:
173                limit: 50
174            limit_ref: "#/dict/limit"
175
176        Whereas here we want to access the `nested/path` value.
177          nested:
178            path: "first one"
179          nested/path: "uh oh"
180          value: "#/nested/path"
181
182        To resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
183        until we find a key with the given path, or until there is nothing to traverse.
184
185        Consider the path foo/bar/baz. To resolve the ambiguity, we first try 'foo/bar/baz' in its entirety as a top-level key. If this
186        fails, we try 'foo' as the top-level key, and if this succeeds, pass 'bar/baz' on as the key to be tried at the next level.
187        """
188        while ref:
189            try:
190                return manifest_node[ref]
191            except (KeyError, TypeError):
192                head, ref = _parse_path(ref)
193                manifest_node = manifest_node[head]  # type: ignore # Couldn't figure out how to fix this since manifest_node can get reassigned into other types like lists
194        return manifest_node

An incoming manifest can contain references to values previously defined. This parser will dereference these values to produce a complete ConnectionDefinition.

References can be defined using a #/ string.

key: 1234
reference: "#/key"