From 64e67e577911d70eb2504f9f7f42be8ed4b5ec3d Mon Sep 17 00:00:00 2001 From: Volv G Date: Fri, 25 Sep 2026 18:56:01 -0700 Subject: [PATCH 1/2] Accept a None default on In[T] under Python 3.10 (v0.1.26) Python 3.10's get_type_hints still applies PEP 484 implicit Optional, which 3.11 removed, so `x: In[str] = None` resolved to Optional[In[str]] and was no longer recognised as a graph input. Strip an Optional wrapper when it is paired with a None default, in the @pipeline signature path and the cfg predicate. Written as a behavioural rule rather than a version check, so all supported Pythons take the same code path. A wider union is left alone. --- .../tangle-cli/src/tangle_cli/__init__.py | 2 +- .../src/tangle_cli/pipeline_compiler.py | 3 +- .../src/tangle_cli/python_pipeline/trace.py | 20 ++- pyproject.toml | 2 +- tests/test_optional_annotations.py | 134 ++++++++++++++++++ tests/test_packaging.py | 2 +- uv.lock | 2 +- 7 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 tests/test_optional_annotations.py diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index 6aa85f8..c6e0a11 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.25" + __version__ = "0.1.26" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py index dd4cb6b..1c4826d 100644 --- a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py +++ b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py @@ -77,7 +77,7 @@ from .python_pipeline.ref import CallableRef from .python_pipeline.registered import _REGISTERED_URL_PLACEHOLDER from .python_pipeline.subpipeline import _SUBPIPELINE_URL_PLACEHOLDER, SubpipelineRef -from .python_pipeline.trace import trace_pipeline +from .python_pipeline.trace import _strip_optional_none, trace_pipeline from .python_pipeline.types import In from .schema_validation import ( CALLER_ANNOTATION_POLICY, @@ -2589,6 +2589,7 @@ def _pipeline_accepts_cfg(pipeline_fn: PipelineFn) -> bool: annotation = resolved_hints.get("cfg", annotation) except Exception: pass + annotation = _strip_optional_none(annotation, param.default) return getattr(annotation, "__origin__", None) is not In diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py index e8d8570..01529df 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py @@ -92,6 +92,22 @@ def _is_in_annotation(annotation: Any) -> bool: return getattr(annotation, "__origin__", None) is In +def _strip_optional_none(annotation: Any, default: Any) -> Any: + """Drop an ``Optional`` wrapper paired with a ``None`` default. + + Python 3.10's ``get_type_hints`` still applies PEP 484 implicit + ``Optional``, which 3.11 removed. A wider union is left alone. + """ + if default is not None: + return annotation + if typing.get_origin(annotation) is not typing.Union: + return annotation + args = [a for a in typing.get_args(annotation) if a is not type(None)] + if len(args) != 1: + return annotation + return args[0] + + def _is_out_annotation(annotation: Any) -> bool: return getattr(annotation, "__origin__", None) is Out @@ -246,7 +262,9 @@ def trace_pipeline( call_kwargs: dict[str, Any] = {} for param_name, param in sig.parameters.items(): - annotation = resolved_hints.get(param_name, param.annotation) + annotation = _strip_optional_none( + resolved_hints.get(param_name, param.annotation), param.default + ) if param_name == "cfg" and not _is_in_annotation(annotation): call_kwargs[param_name] = cfg diff --git a/pyproject.toml b/pyproject.toml index 77f768d..ac4d1c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.25" +version = "0.1.26" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/test_optional_annotations.py b/tests/test_optional_annotations.py new file mode 100644 index 0000000..688eb6d --- /dev/null +++ b/tests/test_optional_annotations.py @@ -0,0 +1,134 @@ +"""A ``None``-defaulted ``In[T]`` parameter resolves the same on every Python. + +Python 3.10's ``typing.get_type_hints`` still applies PEP 484 implicit +``Optional``, which 3.11 removed. ``x: In[str] = None`` therefore resolved to +``In[str]`` on 3.11+ and ``Optional[In[str]]`` on 3.10, where the tracer then +rejected it as "not annotated In[T]". CI runs 3.12 and 3.13 while +``requires-python`` is ``>=3.10``, so nothing caught it. + +These tests exercise the unwrapping directly, so they fail on 3.10 without the +fix and still pin the rule on the versions CI runs. +""" + +from __future__ import annotations + +import inspect +import textwrap +from pathlib import Path +from typing import Optional, Union + +import yaml + +from tangle_cli.pipeline_compiler import compile_pipeline +from tangle_cli.python_pipeline import In +from tangle_cli.python_pipeline.trace import _strip_optional_none + +_PIPELINE = ''' +from typing import Optional + +from tangle_cli.python_pipeline import In, Out, pipeline, task + + +@task(image="python:3.12") +def greet(greeting: str = "hi"): + """Write a greeting. + + Metadata: + Name: Greet + """ + print(greeting) + + +@pipeline("Optionals") +def optionals(__PARAMS__) -> Out[str]: + run_greet = greet(greeting=b) + return run_greet +''' + + +def _compile(tmp_path: Path, params: str, case: str, *, config: str | None = None) -> dict: + case_dir = tmp_path / case + case_dir.mkdir(parents=True, exist_ok=True) + if config is not None: + (case_dir / "config.yaml").write_text(config, encoding="utf-8") + script = case_dir / "pipeline.py" + script.write_text( + textwrap.dedent(_PIPELINE).replace("__PARAMS__", params), encoding="utf-8" + ) + out = case_dir / "compiled.yaml" + compile_pipeline(script, out) + return yaml.safe_load(out.read_text(encoding="utf-8")) + + +# ============================================================================ +# The unwrapping rule +# ============================================================================ + + +def test_an_optional_paired_with_a_none_default_is_unwrapped(): + """This is the shape 3.10 invents for ``x: In[str] = None``.""" + assert _strip_optional_none(Optional[In[str]], None) is In[str] + + +def test_an_optional_without_a_none_default_is_left_alone(): + """Only the implicit-Optional pairing is rewritten; an explicit Optional + on a parameter with a real default, or none at all, keeps what the author + wrote.""" + annotation = Optional[In[str]] + + assert _strip_optional_none(annotation, "x") is annotation + assert _strip_optional_none(annotation, inspect.Parameter.empty) is annotation + + +def test_a_bare_annotation_is_left_alone(): + assert _strip_optional_none(In[str], None) is In[str] + assert _strip_optional_none(str, None) is str + + +def test_a_wider_union_is_left_alone(): + """``Union[A, B, None]`` is a genuine union; unwrapping it would invent a + type the author never wrote.""" + annotation = Union[In[str], In[int], None] + + assert _strip_optional_none(annotation, None) is annotation + + +# ============================================================================ +# End to end +# ============================================================================ + + +def test_a_none_defaulted_input_compiles(tmp_path): + doc = _compile(tmp_path, 'a: In[str] = None, b: In[str] = "x"', "none_default") + + assert doc["inputs"] == [ + {"name": "a", "type": "String", "default": None, "optional": True}, + {"name": "b", "type": "String", "default": "x", "optional": True}, + ] + + +def test_an_explicit_optional_input_compiles_the_same_way(tmp_path): + """3.10 and 3.11+ disagree about which of these two spellings you wrote, + so both must land on the same document.""" + doc = _compile( + tmp_path, + 'a: Optional[In[str]] = None, b: In[str] = "x"', + "explicit_optional", + ) + + assert doc["inputs"][0] == { + "name": "a", + "type": "String", + "default": None, + "optional": True, + } + + +def test_a_cfg_parameter_defaulting_to_none_is_still_the_config(tmp_path): + """``cfg`` is detected by name plus 'not an In[T]'. On 3.10 the implicit + Optional made that check read differently, so pin it here too.""" + doc = _compile( + tmp_path, 'cfg=None, b: In[str] = "x"', "cfg_none", config="key: value\n" + ) + + assert [i["name"] for i in doc["inputs"]] == ["b"] diff --git a/tests/test_packaging.py b/tests/test_packaging.py index e208db5..47f6124 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.25" in metadata + assert "Version: 0.1.26" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/uv.lock b/uv.lock index c93d043..b0f9507 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.25" +version = "0.1.26" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" }, From 4f9cf8ea6545de8ae35042745bf6760046c1852b Mon Sep 17 00:00:00 2001 From: Volv G Date: Fri, 25 Sep 2026 19:01:29 -0700 Subject: [PATCH 2/2] Emit every componentRef locator ref() is given ref() rejected url= with name=, so five corpus pipelines under relevance/experiments and relevance-tools could not be expressed in Python at all. The dehydrated schema allows url, name and digest together, and the hydrator resolves each present locator independently and keeps the highest component version. Allow any combination of the three and emit each one in the canonical key order url, name, digest. Multiple locators are an upgrade path, not a narrowing filter: the name can win over the url. tag= and a ref with no locator stay rejected. --- README.md | 20 ++ .../src/tangle_cli/python_pipeline/emit.py | 32 +- .../src/tangle_cli/python_pipeline/ref.py | 30 +- tests/test_ref_locator_combinations.py | 305 ++++++++++++++++++ 4 files changed, 353 insertions(+), 34 deletions(-) create mode 100644 tests/test_ref_locator_combinations.py diff --git a/README.md b/README.md index 4f0c0a9..888e84d 100644 --- a/README.md +++ b/README.md @@ -905,6 +905,26 @@ def parent(seed: In[str]) -> Out[str]: Use `ref(url=...)`, `ref(name=...)`, or `ref(digest=...)` to call an existing component YAML or published component instead of authoring a local `@task`. Use `@registered(fragment=..., gen_config=...)` for operation wrappers that are already present in an existing `gen_config.yaml`; the compiler rewrites those calls to `resolve://...#fragment` without generating a new sidecar. +##### Component locators + +`ref()` takes any combination of `url`, `name` and `digest`, and emits every one it is given, always in the key order `url, name, digest`: + +| Call | `componentRef` | +| --- | --- | +| `ref(url="file://./c.yaml")` | `{url}` | +| `ref(name="My Comp")` | `{name}` | +| `ref(digest="<64hex>")` | `{digest}` | +| `ref(url="gs://b/c.yaml", digest="<64hex>")` | `{url, digest}` | +| `ref(name="My Comp", digest="<64hex>")` | `{name, digest}` | +| `ref(url="file://./c.yaml", name="My Comp")` | `{url, name}` | +| `ref(url="file://./c.yaml", name="My Comp", digest="<64hex>")` | `{url, name, digest}` | + +Multiple locators are **not** a narrowing filter. The hydrator resolves each one independently and keeps whichever resolves to the **highest component version**, tie-breaking `digest` > `name` > `url`; a locator that fails to resolve is warned about and skipped, and the resolved `componentRef` is replaced wholesale by the winner's `{name, digest, spec}`. So `ref(url=..., name=...)` means "use this file, but prefer the published component if it is newer" — the name can win. Pass a single locator when you want exactly one component, and add `digest=` to pin a version. + +`tag=` is accepted in the signature but rejected at compile time: the hydrator has no tag fetcher and `tag` is not a property of the dehydrated schema. + +A ref with no locator at all is rejected. Diagnostics name the offending keyword, never the value. + ##### Dynamic arguments and runtime placeholders Task argument values can be literals, graph inputs, task outputs, or supported dynamic data. Use `dynamic_secret("NAME")` to emit a runtime secret reference: diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py index a255f2d..0d7f6ea 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py @@ -182,17 +182,14 @@ def _emit_task( def _emit_component_ref(node: TaskNode) -> dict[str, Any]: - """Render ``componentRef`` as a PURE ref — ``{url[, digest]}``, - ``{name[, digest]}`` or ``{digest}``. Never emits ``spec`` or ``text``. + """Render ``componentRef`` as a PURE ref over whichever locators the + ref carries. Never emits ``spec`` or ``text``. - Locator dispatch (first matching branch wins): - - * ``ref_url`` set -> ``{"url": …}``; a ``ref_digest`` pins it - (``{"url": …, "digest": …}``). Subpipeline refs always take this - branch via their ``subpipeline://pending`` sentinel URL. - * ``ref_name`` set -> ``{"name": …}``; a ``ref_digest`` pins it - (``{"name": …, "digest": …}``). - * ``ref_digest`` alone -> ``{"digest": …}``. + Key order is canonical — ``url``, ``name``, ``digest`` — and every + locator present is emitted; the schema allows them together and the + hydrator resolves each one, keeping the highest component version. + Subpipeline refs arrive here with their ``subpipeline://pending`` + sentinel URL and no other locator. A node with ``ref_url=None`` AND no name/digest is only reachable via ``@task`` refs (the ``ref()`` factory rejects a no-locator call). Such @@ -203,18 +200,15 @@ def _emit_component_ref(node: TaskNode) -> dict[str, Any]: placeholder never reaches the written output. It is still a pure ref — no ``spec``/``text``. """ + cref: dict[str, Any] = {} if node.ref_url: - cref: dict[str, Any] = {"url": node.ref_url} - if node.ref_digest: - cref["digest"] = node.ref_digest - return cref + cref["url"] = node.ref_url if node.ref_name: - cref = {"name": node.ref_name} - if node.ref_digest: - cref["digest"] = node.ref_digest - return cref + cref["name"] = node.ref_name if node.ref_digest: - return {"digest": node.ref_digest} + cref["digest"] = node.ref_digest + if cref: + return cref # Transient placeholder for @task refs. The compile driver computes # the real ``resolve://./.components.yaml#`` URL after # tracing (it depends on the output path) and rewrites this in place diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py index 0276d92..99aa9de 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py @@ -476,11 +476,11 @@ def ref( ) -> CallableRef: """Build a CallableRef pointing at a Tangle component. - A ref carries a *locator* — either a ``url`` or a published ``name`` — - and may optionally pin a ``digest`` alongside either (or stand on its - own). All values are stored verbatim (no normalization). The emitter - turns the locator into the matching ``componentRef`` form, and the - hydrator resolves it via ``_fetch_component_by_{url,name,digest}``. + A ref carries at least one *locator* — a ``url``, a published ``name``, + or a ``digest`` — and may carry several. All values are stored verbatim + (no normalization). The emitter turns the locators into the matching + ``componentRef`` form, and the hydrator resolves it via + ``_fetch_component_by_{url,name,digest}``. Supported (WORKING) locator combinations: @@ -495,6 +495,11 @@ def ref( ``{"digest": …}``. - ``ref(url="gs://b/x.yaml", digest="<64hex>")`` — a URL pinned to a digest — emits ``{"url": …, "digest": …}``. + - ``ref(url="file://./c.yaml", name="My Comp")`` — emits + ``{"url": …, "name": …}``, optionally with ``digest``. Note the + hydrator resolves EVERY locator present and keeps the highest + component version, so the name can win over the url; see the README. + This is an upgrade path, not a narrowing one. ``tag`` is accepted in the signature for forward compatibility but is **NOT supported yet**: the hydrator has no tag fetcher, so a ``tag`` @@ -503,9 +508,8 @@ def ref( ``name=`` and/or ``digest=`` instead. Raises: - CompileError: if ``tag`` is passed (deferred); if both ``url`` and - ``name`` are passed (conflicting primary locators); or if no - locator at all is given. + CompileError: if ``tag`` is passed (deferred), or if no locator at + all is given. """ # 1. tag is not resolvable end-to-end yet (hydrator has no tag fetcher). if tag is not None: @@ -513,13 +517,9 @@ def ref( "ref(tag=...) is not supported yet: the hydrator resolves " "digest/name/url only. Pin by name=... and/or digest=... instead." ) - # 2. exactly one PRIMARY locator family: url XOR name (digest is optional - # and may accompany either, or stand alone). - if url is not None and name is not None: - raise CompileError( - "ref() takes EITHER url=... OR name=..., not both (conflicting " - "locators). Use digest=... to pin a version alongside either." - ) + # 2. at least one locator. Any combination of url/name/digest is legal: + # the dehydrated schema allows them together, and the hydrator treats + # each as an independent candidate. if url is None and name is None and digest is None: raise CompileError( "ref() requires a locator: pass url=..., or name=...[, digest=...], " diff --git a/tests/test_ref_locator_combinations.py b/tests/test_ref_locator_combinations.py new file mode 100644 index 0000000..abf79fd --- /dev/null +++ b/tests/test_ref_locator_combinations.py @@ -0,0 +1,305 @@ +"""``ref()`` emits every locator it is given, in canonical key order. + +The dehydrated schema allows ``url``, ``name`` and ``digest`` together, and +the hydrator resolves each one independently and keeps the highest component +version (``_resolve_best_ref`` / ``_pick_best_candidate``). ``ref()`` used to +reject ``url`` with ``name``, so five corpus pipelines under +``relevance/experiments/shop_app`` and ``relevance-tools`` could not be +expressed in Python at all. + +A name beside a url is an upgrade path, not a filter: it can win. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import yaml + +from tangle_cli.pipeline_compiler import compile_pipeline +from tangle_cli.pipeline_hydrator import PipelineHydrator +from tangle_cli.python_pipeline import ref +from tangle_cli.python_pipeline.errors import CompileError + +_DIGEST = "a" * 64 + +def _component(name: str, version: str) -> dict: + return { + "name": name, + "metadata": {"annotations": {"version": version}}, + "implementation": {"container": {"image": "python:3.12"}}, + } + + +_HEADER = ''' +from tangle_cli.python_pipeline import Out, pipeline, ref + +COMPONENT = ref(__REF_ARGS__) + +''' + +_BODY = ''' +@pipeline("Locators") +def locators() -> Out[str]: + run_it = COMPONENT.named("Run It")() + return run_it +''' + + +def _compile(tmp_path: Path, ref_args: str, case: str, *, component: str = "c.yaml") -> dict: + """Compile a one-task pipeline and return its ``componentRef``. + + ``component`` is created relative to the output directory because the + compiler refuses a relative ``file://`` ref whose target is missing. + """ + case_dir = tmp_path / case + case_dir.mkdir(parents=True, exist_ok=True) + target = (case_dir / component).resolve() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(yaml.safe_dump(_component("My Comp", "1.0.0")), encoding="utf-8") + script = case_dir / "pipeline.py" + script.write_text( + textwrap.dedent(_HEADER).replace("__REF_ARGS__", ref_args) + + textwrap.dedent(_BODY), + encoding="utf-8", + ) + out = case_dir / "compiled.yaml" + compile_pipeline(script, out) + doc = yaml.safe_load(out.read_text(encoding="utf-8")) + return doc["implementation"]["graph"]["tasks"]["Run It"]["componentRef"] + + +# ============================================================================ +# Emitted shape +# ============================================================================ + + +@pytest.mark.parametrize( + "ref_args, expected, case", + [ + ('url="file://./c.yaml"', {"url": "file://./c.yaml"}, "url"), + ('name="My Comp"', {"name": "My Comp"}, "name"), + (f'digest="{_DIGEST}"', {"digest": _DIGEST}, "digest"), + ( + f'url="file://./c.yaml", digest="{_DIGEST}"', + {"url": "file://./c.yaml", "digest": _DIGEST}, + "url_digest", + ), + ( + f'name="My Comp", digest="{_DIGEST}"', + {"name": "My Comp", "digest": _DIGEST}, + "name_digest", + ), + ( + 'url="file://./c.yaml", name="My Comp"', + {"url": "file://./c.yaml", "name": "My Comp"}, + "url_name", + ), + ( + f'url="file://./c.yaml", name="My Comp", digest="{_DIGEST}"', + {"url": "file://./c.yaml", "name": "My Comp", "digest": _DIGEST}, + "url_name_digest", + ), + ], +) +def test_every_locator_combination_emits_its_keys(tmp_path, ref_args, expected, case): + assert _compile(tmp_path, ref_args, case) == expected + + +def test_locator_keys_are_emitted_in_canonical_order(tmp_path): + """Key order is the emitter's, not the caller's, so two pipelines writing + the same locators produce the same bytes.""" + forward = _compile( + tmp_path, + f'url="file://./c.yaml", name="My Comp", digest="{_DIGEST}"', + "order_forward", + ) + reversed_kwargs = _compile( + tmp_path, + f'digest="{_DIGEST}", name="My Comp", url="file://./c.yaml"', + "order_reversed", + ) + + assert list(forward) == ["url", "name", "digest"] + assert list(reversed_kwargs) == ["url", "name", "digest"] + + +def test_the_corpus_shape_round_trips(tmp_path): + """The shape five corpus pipelines use — a relative component url beside + the component's published name — which could not be authored in Python + before.""" + corpus = { + "name": "Build Vantage Features", + "url": "file://../../components/build-vantage-features.yaml", + } + + emitted = _compile( + tmp_path, + 'url="file://../../components/build-vantage-features.yaml",' + ' name="Build Vantage Features"', + "corpus/tangle/pipeline", + component="../../components/build-vantage-features.yaml", + ) + + assert emitted == corpus + assert set(emitted) == set(corpus) + + +# ============================================================================ +# Chaining is unaffected +# ============================================================================ + + +def test_chaining_preserves_every_locator(tmp_path): + """``.named`` / ``.bind`` / ``.with_annotations`` rebuild the ref, so the + locators have to survive the copy.""" + case_dir = tmp_path / "chained" + case_dir.mkdir() + (case_dir / "c.yaml").write_text( + yaml.safe_dump(_component("My Comp", "1.0.0")), encoding="utf-8" + ) + script = case_dir / "pipeline.py" + script.write_text( + textwrap.dedent( + f''' + from tangle_cli.python_pipeline import Out, pipeline, ref + + COMPONENT = ( + ref(url="file://./c.yaml", name="My Comp", digest="{_DIGEST}") + .bind(fixed="1") + .with_annotations({{"team": "search"}}) + ) + + + @pipeline("Chained") + def chained() -> Out[str]: + run_it = COMPONENT.named("Run It")(other="2") + return run_it + ''' + ), + encoding="utf-8", + ) + out = case_dir / "compiled.yaml" + compile_pipeline(script, out) + + task = yaml.safe_load(out.read_text(encoding="utf-8"))["implementation"]["graph"][ + "tasks" + ]["Run It"] + assert task["componentRef"] == { + "url": "file://./c.yaml", + "name": "My Comp", + "digest": _DIGEST, + } + assert task["arguments"] == {"fixed": "1", "other": "2"} + assert task["annotations"] == {"team": "search"} + + +# ============================================================================ +# Hydration +# ============================================================================ + + +def test_the_hydrator_resolves_both_locators_and_keeps_the_newer(tmp_path): + """This is the behaviour the corpus relies on, and the reason the name is + worth emitting beside the url: whichever locator resolves to the higher + version wins.""" + hydrator = PipelineHydrator(client=MagicMock()) + hydrator._resolve_registered_component = MagicMock( # type: ignore[method-assign] + side_effect=lambda ref_type, ref_value, path, base_dir: { + "url": ("digest-from-url", _component("From URL", "1.0.0")), + "name": ("digest-from-name", _component("From Name", "2.0.0")), + }[ref_type] + ) + + task = hydrator._resolve_task( + "Run It", + {"componentRef": {"url": "file://./c.yaml", "name": "My Comp"}}, + "p.tasks.Run It", + ) + + assert task["componentRef"]["spec"]["name"] == "From Name" + assert task["componentRef"]["digest"] == "digest-from-name" + + +def test_the_hydrator_keeps_the_url_when_it_is_the_newer(tmp_path): + """The same mechanism in the other direction, so the test above is not + just pinning the tie-break order.""" + hydrator = PipelineHydrator(client=MagicMock()) + hydrator._resolve_registered_component = MagicMock( # type: ignore[method-assign] + side_effect=lambda ref_type, ref_value, path, base_dir: { + "url": ("digest-from-url", _component("From URL", "3.0.0")), + "name": ("digest-from-name", _component("From Name", "2.0.0")), + }[ref_type] + ) + + task = hydrator._resolve_task( + "Run It", + {"componentRef": {"url": "file://./c.yaml", "name": "My Comp"}}, + "p.tasks.Run It", + ) + + assert task["componentRef"]["spec"]["name"] == "From URL" + + +def test_a_compiled_url_name_ref_hydrates(tmp_path): + """End to end: compile the Python, then hydrate what it wrote.""" + component = tmp_path / "c.yaml" + component.write_text(yaml.safe_dump(_component("My Comp", "1.0.0")), encoding="utf-8") + case_dir = tmp_path / "e2e" + case_dir.mkdir() + script = case_dir / "pipeline.py" + script.write_text( + textwrap.dedent( + ''' + from tangle_cli.python_pipeline import Out, pipeline, ref + + COMPONENT = ref(url="file://../c.yaml", name="My Comp") + + + @pipeline("Hydrated") + def hydrated() -> Out[str]: + run_it = COMPONENT.named("Run It")() + return run_it + ''' + ), + encoding="utf-8", + ) + out = case_dir / "compiled.yaml" + compile_pipeline(script, out) + + hydrator = PipelineHydrator(client=MagicMock()) + result = hydrator.hydrate_file(out) + task = result.data["implementation"]["graph"]["tasks"]["Run It"] + + assert task["componentRef"]["spec"]["name"] == "My Comp" + + +# ============================================================================ +# Still rejected +# ============================================================================ + + +def test_a_ref_with_no_locator_is_rejected(): + with pytest.raises(CompileError) as excinfo: + ref() + + assert "requires a locator" in str(excinfo.value) + + +def test_a_tag_ref_is_still_rejected(): + """The hydrator has no tag fetcher, and ``tag`` is not even a property of + the dehydrated schema.""" + with pytest.raises(CompileError) as excinfo: + ref(url="file://./c.yaml", name="My Comp", tag="v1") + + assert "tag" in str(excinfo.value) + + +def test_rejection_messages_do_not_echo_values(): + with pytest.raises(CompileError) as excinfo: + ref(tag="super-secret-tag") + + assert "super-secret-tag" not in str(excinfo.value)