Python 3.10 implicit-Optional fix; emit every componentRef locator (v0.1.26)

README.md

@@ -905,6 +905,26 @@ def parent(seed: In[str]) -> Out[str]:
 
 Use `ref(url=...)`, `ref(name=...)`, or `ref(digest=...)` to call an existing component YAML or published component instead of authoring a local `@task`. Use `@registered(fragment=..., gen_config=...)` for operation wrappers that are already present in an existing `gen_config.yaml`; the compiler rewrites those calls to `resolve://...#fragment` without generating a new sidecar.
 
+##### Component locators
+
+`ref()` takes any combination of `url`, `name` and `digest`, and emits every one it is given, always in the key order `url, name, digest`:
+
+| Call | `componentRef` |
+| --- | --- |
+| `ref(url="file://./c.yaml")` | `{url}` |
+| `ref(name="My Comp")` | `{name}` |
+| `ref(digest="<64hex>")` | `{digest}` |
+| `ref(url="gs://b/c.yaml", digest="<64hex>")` | `{url, digest}` |
+| `ref(name="My Comp", digest="<64hex>")` | `{name, digest}` |
+| `ref(url="file://./c.yaml", name="My Comp")` | `{url, name}` |
+| `ref(url="file://./c.yaml", name="My Comp", digest="<64hex>")` | `{url, name, digest}` |
+
+Multiple locators are **not** a narrowing filter. The hydrator resolves each one independently and keeps whichever resolves to the **highest component version**, tie-breaking `digest` > `name` > `url`; a locator that fails to resolve is warned about and skipped, and the resolved `componentRef` is replaced wholesale by the winner's `{name, digest, spec}`. So `ref(url=..., name=...)` means "use this file, but prefer the published component if it is newer" — the name can win. Pass a single locator when you want exactly one component, and add `digest=` to pin a version.
+
+`tag=` is accepted in the signature but rejected at compile time: the hydrator has no tag fetcher and `tag` is not a property of the dehydrated schema.
+
+A ref with no locator at all is rejected. Diagnostics name the offending keyword, never the value.
+
 ##### Dynamic arguments and runtime placeholders
 
 Task argument values can be literals, graph inputs, task outputs, or supported dynamic data. Use `dynamic_secret("NAME")` to emit a runtime secret reference:

packages/tangle-cli/src/tangle_cli/__init__.py

@@ -14,6 +14,6 @@
 try:
     __version__ = metadata_version("tangle-cli")
 except PackageNotFoundError:
-    __version__ = "0.1.25"
+    __version__ = "0.1.26"
 
 __all__ = ["TangleDynamicDiscoveryClient", "__version__"]

packages/tangle-cli/src/tangle_cli/pipeline_compiler.py

@@ -77,7 +77,7 @@
 from .python_pipeline.ref import CallableRef
 from .python_pipeline.registered import _REGISTERED_URL_PLACEHOLDER
 from .python_pipeline.subpipeline import _SUBPIPELINE_URL_PLACEHOLDER, SubpipelineRef
-from .python_pipeline.trace import trace_pipeline
+from .python_pipeline.trace import _strip_optional_none, trace_pipeline
 from .python_pipeline.types import In
 from .schema_validation import (
     CALLER_ANNOTATION_POLICY,
@@ -2589,6 +2589,7 @@ def _pipeline_accepts_cfg(pipeline_fn: PipelineFn) -> bool:
         annotation = resolved_hints.get("cfg", annotation)
     except Exception:
         pass
+    annotation = _strip_optional_none(annotation, param.default)
     return getattr(annotation, "__origin__", None) is not In
 
 

packages/tangle-cli/src/tangle_cli/python_pipeline/emit.py

@@ -182,17 +182,14 @@ def _emit_task(
 
 
 def _emit_component_ref(node: TaskNode) -> dict[str, Any]:
-    """Render ``componentRef`` as a PURE ref — ``{url[, digest]}``,
-    ``{name[, digest]}`` or ``{digest}``. Never emits ``spec`` or ``text``.
+    """Render ``componentRef`` as a PURE ref over whichever locators the
+    ref carries. Never emits ``spec`` or ``text``.
 
-    Locator dispatch (first matching branch wins):
-
-    * ``ref_url`` set        -> ``{"url": …}``; a ``ref_digest`` pins it
-      (``{"url": …, "digest": …}``). Subpipeline refs always take this
-      branch via their ``subpipeline://pending`` sentinel URL.
-    * ``ref_name`` set       -> ``{"name": …}``; a ``ref_digest`` pins it
-      (``{"name": …, "digest": …}``).
-    * ``ref_digest`` alone   -> ``{"digest": …}``.
+    Key order is canonical — ``url``, ``name``, ``digest`` — and every
+    locator present is emitted; the schema allows them together and the
+    hydrator resolves each one, keeping the highest component version.
+    Subpipeline refs arrive here with their ``subpipeline://pending``
+    sentinel URL and no other locator.
 
     A node with ``ref_url=None`` AND no name/digest is only reachable via
     ``@task`` refs (the ``ref()`` factory rejects a no-locator call). Such
@@ -203,18 +200,15 @@ def _emit_component_ref(node: TaskNode) -> dict[str, Any]:
     placeholder never reaches the written output. It is still a pure ref
     — no ``spec``/``text``.
     """
+    cref: dict[str, Any] = {}
     if node.ref_url:
-        cref: dict[str, Any] = {"url": node.ref_url}
-        if node.ref_digest:
-            cref["digest"] = node.ref_digest
-        return cref
+        cref["url"] = node.ref_url
     if node.ref_name:
-        cref = {"name": node.ref_name}
-        if node.ref_digest:
-            cref["digest"] = node.ref_digest
-        return cref
+        cref["name"] = node.ref_name
     if node.ref_digest:
-        return {"digest": node.ref_digest}
+        cref["digest"] = node.ref_digest
+    if cref:
+        return cref
     # Transient placeholder for @task refs. The compile driver computes
     # the real ``resolve://./<stem>.components.yaml#<fragment>`` URL after
     # tracing (it depends on the output path) and rewrites this in place

packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py

@@ -476,11 +476,11 @@ def ref(
 ) -> CallableRef:
     """Build a CallableRef pointing at a Tangle component.
 
-    A ref carries a *locator* — either a ``url`` or a published ``name`` —
-    and may optionally pin a ``digest`` alongside either (or stand on its
-    own). All values are stored verbatim (no normalization). The emitter
-    turns the locator into the matching ``componentRef`` form, and the
-    hydrator resolves it via ``_fetch_component_by_{url,name,digest}``.
+    A ref carries at least one *locator* — a ``url``, a published ``name``,
+    or a ``digest`` — and may carry several. All values are stored verbatim
+    (no normalization). The emitter turns the locators into the matching
+    ``componentRef`` form, and the hydrator resolves it via
+    ``_fetch_component_by_{url,name,digest}``.
 
     Supported (WORKING) locator combinations:
 
@@ -495,6 +495,11 @@ def ref(
       ``{"digest": …}``.
     - ``ref(url="gs://b/x.yaml", digest="<64hex>")`` — a URL pinned to a
       digest — emits ``{"url": …, "digest": …}``.
+    - ``ref(url="file://./c.yaml", name="My Comp")`` — emits
+      ``{"url": …, "name": …}``, optionally with ``digest``. Note the
+      hydrator resolves EVERY locator present and keeps the highest
+      component version, so the name can win over the url; see the README.
+      This is an upgrade path, not a narrowing one.
 
     ``tag`` is accepted in the signature for forward compatibility but is
     **NOT supported yet**: the hydrator has no tag fetcher, so a ``tag``
@@ -503,23 +508,18 @@ def ref(
     ``name=`` and/or ``digest=`` instead.
 
     Raises:
-        CompileError: if ``tag`` is passed (deferred); if both ``url`` and
-            ``name`` are passed (conflicting primary locators); or if no
-            locator at all is given.
+        CompileError: if ``tag`` is passed (deferred), or if no locator at
+            all is given.
     """
     # 1. tag is not resolvable end-to-end yet (hydrator has no tag fetcher).
     if tag is not None:
         raise CompileError(
             "ref(tag=...) is not supported yet: the hydrator resolves "
             "digest/name/url only. Pin by name=... and/or digest=... instead."
         )
-    # 2. exactly one PRIMARY locator family: url XOR name (digest is optional
-    #    and may accompany either, or stand alone).
-    if url is not None and name is not None:
-        raise CompileError(
-            "ref() takes EITHER url=... OR name=..., not both (conflicting "
-            "locators). Use digest=... to pin a version alongside either."
-        )
+    # 2. at least one locator. Any combination of url/name/digest is legal:
+    #    the dehydrated schema allows them together, and the hydrator treats
+    #    each as an independent candidate.
     if url is None and name is None and digest is None:
         raise CompileError(
             "ref() requires a locator: pass url=..., or name=...[, digest=...], "

packages/tangle-cli/src/tangle_cli/python_pipeline/trace.py

@@ -92,6 +92,22 @@ def _is_in_annotation(annotation: Any) -> bool:
     return getattr(annotation, "__origin__", None) is In
 
 
+def _strip_optional_none(annotation: Any, default: Any) -> Any:
+    """Drop an ``Optional`` wrapper paired with a ``None`` default.
+
+    Python 3.10's ``get_type_hints`` still applies PEP 484 implicit
+    ``Optional``, which 3.11 removed. A wider union is left alone.
+    """
+    if default is not None:
+        return annotation
+    if typing.get_origin(annotation) is not typing.Union:
+        return annotation
+    args = [a for a in typing.get_args(annotation) if a is not type(None)]
+    if len(args) != 1:
+        return annotation
+    return args[0]
+
+
 def _is_out_annotation(annotation: Any) -> bool:
     return getattr(annotation, "__origin__", None) is Out
 
@@ -246,7 +262,9 @@ def trace_pipeline(
     call_kwargs: dict[str, Any] = {}
 
     for param_name, param in sig.parameters.items():
-        annotation = resolved_hints.get(param_name, param.annotation)
+        annotation = _strip_optional_none(
+            resolved_hints.get(param_name, param.annotation), param.default
+        )
 
         if param_name == "cfg" and not _is_in_annotation(annotation):
             call_kwargs[param_name] = cfg

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tangle-cli"
-version = "0.1.25"
+version = "0.1.26"
 description = "CLI for Tangle, the open-source ML pipeline orchestration platform"
 readme = "README.md"
 authors = [

tests/test_optional_annotations.py

@@ -0,0 +1,134 @@
+"""A ``None``-defaulted ``In[T]`` parameter resolves the same on every Python.
+
+Python 3.10's ``typing.get_type_hints`` still applies PEP 484 implicit
+``Optional``, which 3.11 removed. ``x: In[str] = None`` therefore resolved to
+``In[str]`` on 3.11+ and ``Optional[In[str]]`` on 3.10, where the tracer then
+rejected it as "not annotated In[T]". CI runs 3.12 and 3.13 while
+``requires-python`` is ``>=3.10``, so nothing caught it.
+
+These tests exercise the unwrapping directly, so they fail on 3.10 without the
+fix and still pin the rule on the versions CI runs.
+"""
+
+from __future__ import annotations
+
+import inspect
+import textwrap
+from pathlib import Path
+from typing import Optional, Union
+
+import yaml
+
+from tangle_cli.pipeline_compiler import compile_pipeline
+from tangle_cli.python_pipeline import In
+from tangle_cli.python_pipeline.trace import _strip_optional_none
+
+_PIPELINE = '''
+from typing import Optional
+
+from tangle_cli.python_pipeline import In, Out, pipeline, task
+
+
+@task(image="python:3.12")
+def greet(greeting: str = "hi"):
+    """Write a greeting.
+
+    Metadata:
+        Name: Greet
+    """
+    print(greeting)
+
+
+@pipeline("Optionals")
+def optionals(__PARAMS__) -> Out[str]:
+    run_greet = greet(greeting=b)
+    return run_greet
+'''
+
+
+def _compile(tmp_path: Path, params: str, case: str, *, config: str | None = None) -> dict:
+    case_dir = tmp_path / case
+    case_dir.mkdir(parents=True, exist_ok=True)
+    if config is not None:
+        (case_dir / "config.yaml").write_text(config, encoding="utf-8")
+    script = case_dir / "pipeline.py"
+    script.write_text(
+        textwrap.dedent(_PIPELINE).replace("__PARAMS__", params), encoding="utf-8"
+    )
+    out = case_dir / "compiled.yaml"
+    compile_pipeline(script, out)
+    return yaml.safe_load(out.read_text(encoding="utf-8"))
+
+
+# ============================================================================
+# The unwrapping rule
+# ============================================================================
+
+
+def test_an_optional_paired_with_a_none_default_is_unwrapped():
+    """This is the shape 3.10 invents for ``x: In[str] = None``."""
+    assert _strip_optional_none(Optional[In[str]], None) is In[str]
+
+
+def test_an_optional_without_a_none_default_is_left_alone():
+    """Only the implicit-Optional pairing is rewritten; an explicit Optional
+    on a parameter with a real default, or none at all, keeps what the author
+    wrote."""
+    annotation = Optional[In[str]]
+
+    assert _strip_optional_none(annotation, "x") is annotation
+    assert _strip_optional_none(annotation, inspect.Parameter.empty) is annotation
+
+
+def test_a_bare_annotation_is_left_alone():
+    assert _strip_optional_none(In[str], None) is In[str]
+    assert _strip_optional_none(str, None) is str
+
+
+def test_a_wider_union_is_left_alone():
+    """``Union[A, B, None]`` is a genuine union; unwrapping it would invent a
+    type the author never wrote."""
+    annotation = Union[In[str], In[int], None]
+
+    assert _strip_optional_none(annotation, None) is annotation
+
+
+# ============================================================================
+# End to end
+# ============================================================================
+
+
+def test_a_none_defaulted_input_compiles(tmp_path):
+    doc = _compile(tmp_path, 'a: In[str] = None, b: In[str] = "x"', "none_default")
+
+    assert doc["inputs"] == [
+        {"name": "a", "type": "String", "default": None, "optional": True},
+        {"name": "b", "type": "String", "default": "x", "optional": True},
+    ]
+
+
+def test_an_explicit_optional_input_compiles_the_same_way(tmp_path):
+    """3.10 and 3.11+ disagree about which of these two spellings you wrote,
+    so both must land on the same document."""
+    doc = _compile(
+        tmp_path,
+        'a: Optional[In[str]] = None, b: In[str] = "x"',
+        "explicit_optional",
+    )
+
+    assert doc["inputs"][0] == {
+        "name": "a",
+        "type": "String",
+        "default": None,
+        "optional": True,
+    }
+
+
+def test_a_cfg_parameter_defaulting_to_none_is_still_the_config(tmp_path):
+    """``cfg`` is detected by name plus 'not an In[T]'. On 3.10 the implicit
+    Optional made that check read differently, so pin it here too."""
+    doc = _compile(
+        tmp_path, 'cfg=None, b: In[str] = "x"', "cfg_none", config="key: value\n"
+    )
+
+    assert [i["name"] for i in doc["inputs"]] == ["b"]

tests/test_packaging.py

@@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api
     requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")]
     assert not any(name.startswith("tangle_api/") for name in names)
     assert "tangle_cli/openapi/openapi.json" not in names
-    assert "Version: 0.1.25" in metadata
+    assert "Version: 0.1.26" in metadata
     assert "Requires-Dist: tangle-api==0.1.1" in requires_dist
     assert not any("extra == 'native'" in line for line in requires_dist)
     assert "Provides-Extra: native" in metadata