From 848d7680a7d8b5ab60f34c5a9564e8adaae116f5 Mon Sep 17 00:00:00 2001 From: Volv G Date: Fri, 25 Sep 2026 14:18:16 -0700 Subject: [PATCH] Per-command TANGLE_ROOT_CONFIG layered beneath --config; remove ArgsContainer.origin() (v0.1.24) *(AI-assisted)* --- README.md | 40 +- .../tangle-cli/src/tangle_cli/__init__.py | 2 +- packages/tangle-cli/src/tangle_cli/api_cli.py | 27 +- .../src/tangle_cli/args_container.py | 356 +++++++++- packages/tangle-cli/src/tangle_cli/cli.py | 31 +- .../tangle-cli/src/tangle_cli/cli_helpers.py | 45 +- pyproject.toml | 2 +- tests/conftest.py | 7 + tests/test_args_container_env.py | 43 +- tests/test_args_container_typed.py | 5 +- tests/test_packaging.py | 2 +- tests/test_pipeline_runs_cli.py | 35 + tests/test_root_config.py | 657 ++++++++++++++++++ uv.lock | 2 +- 14 files changed, 1193 insertions(+), 61 deletions(-) create mode 100644 tests/test_root_config.py diff --git a/README.md b/README.md index aa61685..635b5b1 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ API-backed commands commonly accept these options. Explicit CLI options win over | `--token`, `TANGLE_API_TOKEN` | Bearer token shorthand. | | `--auth-header`, `TANGLE_API_AUTH_HEADER`, `TANGLE_AUTH_HEADER` | Full `Authorization` value such as `Bearer ...` or `Basic ...`. | | `-H`, `--header`, `TANGLE_API_HEADERS` | Extra headers. Repeatable as CLI flags; env accepts a JSON object or newline-separated `Name: value` entries. | +| `TANGLE_ROOT_CONFIG` | Path to a per-command YAML/JSON root config layered beneath `--config` (see [Per-command root config](#per-command-root-config-tangle_root_config)). | | `--config` | YAML/JSON defaults. Many commands accept a single object, a list of objects, or `_defaults` + `configs`, optionally wrapped in a top-level `_select` environment selector. Values may be read from the environment with `{_env: NAME}`. | | `--log-type` | SDK progress logs: `console`, `none`, or `file`. Logs go to stderr or a temp log file so structured stdout stays parseable. | | `TANGLE_VERBOSE=1` | Redacted HTTP request/response diagnostics only. This is separate from normal progress logging. | @@ -230,6 +231,41 @@ uv run tangle sdk pipeline-runs submit --config submit.yaml For generated `tangle api` commands, config keys use generated CLI parameter names such as `base_url`, `schema_source`, `body`, and endpoint parameters like `limit`, `filter`, or `id`. +### Per-command root config (`TANGLE_ROOT_CONFIG`) + +Set `TANGLE_ROOT_CONFIG` to a YAML/JSON file that gives individual commands a base config beneath their `--config`. The file is keyed by command, because the same values rarely suit every command: + +```yaml +# $TANGLE_ROOT_CONFIG +_shared: &search {annotations: {team: search}} # underscore keys: YAML anchor helpers + +commands: + "tangle sdk pipeline-runs submit": + <<: *search + base_url: https://api.example + "tangle sdk published-components publish": *search + "tangle-deploy pipeline-run submit": *search # another CLI's commands can share the file +``` + +- **Shape.** The top level must be the `commands` selector. Besides it, only underscore-prefixed helper keys (for YAML anchors) may appear at the top level; any other key is an error. Each `commands` key is a command's full invocation path, program first. Each value is a single config object. +- **Command identity.** `ArgsContainer.load(..., command=...)` and `load_config(..., command=...)` take the running command's full path explicitly, program first (e.g. `"tangle sdk secrets delete"`). Without `command=` (a plain library call), `TANGLE_ROOT_CONFIG` is ignored and not read. `tangle` builds the path as Cyclopts resolves the command line: the program, then each group, then the leaf, with options and arguments excluded and each level canonicalized to its first registered name. Its CLI helpers (`load_args_or_exit`, `load_config_or_exit`) then pass it as `command=`. Another CLI builds its own path the same way, e.g. from click contexts as groups resolve. +- **Normalization.** `normalize_command(path, aliases)` collapses whitespace and rewrites alias or deprecated prefixes to the canonical path (`COMMAND_ALIASES` maps `tangle-cli` to `tangle`; a downstream CLI passes its own table, such as legacy entry-point names). Root keys and the running command are compared after normalization, exact and case-sensitive. +- **Selection.** Only the running command's entry applies. A command with no entry gets no root config; that is a no-op, not an error. Entries for unknown commands, including other CLIs' commands, are ignored. A key that closely matches the running command logs a warning naming both keys, never a value. +- **Layering.** Without `--config`, the entry is the config. With `--config`, each command config entry is deep-merged over the root entry, with `--config` winning at each leaf. Lists and scalars from `--config` replace root values wholesale. For `_defaults` + `configs`, the file's own `_defaults` are applied first (still a shallow merge), so the order is root < `_defaults` < entry. Precedence per field is **CLI > `--config` > `TANGLE_ROOT_CONFIG` > environment (`EnvField`) > default**, and a CLI value still replaces the whole field. +- **`null`.** + - `null` is rejected anywhere in `TANGLE_ROOT_CONFIG`; the error names the key path. + - In a `--config` layered over an active root entry, a `null` mapping value means *absent*, at any depth: + - at a key the root entry provides, it unsets the inherited value, so `annotations: {team: null}` removes just `team`; + - at a key the root does not set, it is ignored rather than setting `None`. + + Either way, resolution falls through to the environment tier, then the default. `null` items inside lists are values and are kept. + - Without an active root entry — `TANGLE_ROOT_CONFIG` unset, no entry for the command, or no command identity — `--config` `null` keeps its existing meaning: the config supplies `None`. +- **`_select` / `_env`.** A document-level `_select` may choose between whole `commands` documents. After the command's entry is picked, the entry itself may be a `_select`, and its `_env` values are looked up then — only for the running command. Every directive in the file, in any command, is structure-checked in every environment. Scalar typing applies to the merged value. +- **Relative paths** resolve against the file a value came from. `args.config_source(name)` names that file, at top-level-key granularity. A value inherited from `TANGLE_ROOT_CONFIG` is never resolved against the `--config` directory. +- **`base_url`** is allowed in an entry. It counts as config for credential isolation: ambient environment credentials (such as `TANGLE_API_TOKEN`) are not sent to a URL that came from config. +- **Errors fail closed.** An unset or empty `TANGLE_ROOT_CONFIG` means no root config. A set path that is missing, not a file, unreadable, malformed, or contains `null` is an error naming `TANGLE_ROOT_CONFIG` and the path, never a value. A relative path is relative to the current directory. +- **Scope.** `TANGLE_ROOT_CONFIG` applies to `--config` loading only; pipeline `config.yaml` files and `TaskEnv.from_config` are never layered. For `tangle api`, the pre-dispatch schema bootstrap takes the identity from the leading command tokens (`tangle api `). + ### Environment-selected configs (`_select`) Any command that accepts `--config`, and any Python pipeline `config.yaml` (see [Environment-selected pipeline config](#environment-selected-pipeline-config-_select-_env)), can pick one of several config documents from an environment variable by making `_select` the top-level node: @@ -1094,7 +1130,9 @@ Use these for generic downstream behavior such as alternate storage, extra annot `cli_options.py` centralizes shared Cyclopts annotations such as `BaseUrlOption`, `TokenOption`, `AuthHeaderOption`, `HeaderOption`, `ConfigOption`, and `LogTypeOption`. `cli_helpers.py` centralizes config loading, JSON printing, credential-isolation helpers, and the native-safe `LazyTangleApiClient` proxy. `logger.py` provides `ConsoleLogger`, `NullLogger`, `CaptureLogger`, `logger_for_log_type(...)`, and `run_with_logging(...)`. -`ArgsContainer.load(...)` field specs are tuples (see `ArgsContainer._resolve`). To give one field an environment tier, wrap its unchanged spec: `token=EnvField("TANGLE_PROD_TOKEN", (token, None))`. The field then resolves CLI > config > `os.environ["TANGLE_PROD_TOKEN"]` > default; the raw string (empty counts as set) goes through the spec's converter or the [scalar typing](#scalar-field-typing), and a conversion error names the variable without echoing its value. Nothing is mapped automatically: fields that are not wrapped never read the environment. `args.origin(name)` reports where each field came from — `cli`, `config`, `env:NAME`, or `default` — without the value. +`ArgsContainer.load(...)` field specs are tuples (see `ArgsContainer._resolve`). To give one field an environment tier, wrap its unchanged spec: `token=EnvField("TANGLE_PROD_TOKEN", (token, None))`. The field then resolves CLI > config > `os.environ["TANGLE_PROD_TOKEN"]` > default; the raw string (empty counts as set) goes through the spec's converter or the [scalar typing](#scalar-field-typing), and a conversion error names the variable without echoing its value. Nothing is mapped automatically: fields that are not wrapped never read the environment. + +Code that reads config values without `ArgsContainer.load` should call `load_config(config_path, command=...)` (in `tangle_cli.args_container`) so it sees the same [`TANGLE_ROOT_CONFIG`](#per-command-root-config-tangle_root_config) layering; `ArgsContainer._load_config_file` reads exactly one file. A CLI other than `tangle` passes its own resolved command path, e.g. `ArgsContainer.load(config, command="tangle-deploy pipeline-run submit", **specs)`. To resolve a relative path from config against the file it was written in, use `args.config_source(name)` (or `LoadedConfig.sources`) rather than the `--config` path: a value inherited from `TANGLE_ROOT_CONFIG` names the root file. Use these helpers for new SDK commands so top-level imports remain native-free, `--config` behavior stays consistent, credentials from config do not accidentally mix with ambient environment auth, and progress logs stay off structured stdout. diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index 540ae8a..c4fa030 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.23" + __version__ = "0.1.24" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/api_cli.py b/packages/tangle-cli/src/tangle_cli/api_cli.py index db98df8..fc2dd79 100644 --- a/packages/tangle-cli/src/tangle_cli/api_cli.py +++ b/packages/tangle-cli/src/tangle_cli/api_cli.py @@ -23,7 +23,7 @@ import platformdirs from cyclopts import App, Parameter -from .args_container import ArgsContainer, ConfigFileError +from .args_container import ArgsContainer, ConfigFileError, load_config from .api_schema import ( SUPPORTED_METHODS, CliParameter, @@ -755,17 +755,34 @@ def _auth_header_from_argv(argv: list[str], *, include_env_credentials: bool = T return auth_header +def _api_command_identity(api_tail: list[str]) -> str: + """``tangle api `` from the tokens after ``api``. + + Best effort for the pre-dispatch schema bootstrap: the leading non-option + tokens, at most two (a group and its operation). Dispatch itself uses the + identity Cyclopts resolves. + """ + + names: list[str] = [] + for token in api_tail: + if token.startswith("-") or len(names) == 2: + break + names.append(token) + return " ".join(("tangle", "api", *names)) + + def _config_value_from_argv(argv: list[str], key: str) -> Any: config_path = _option_from_argv(argv, "--config") - if config_path is None: - return None try: - configs = ArgsContainer._load_config_file(config_path) + # Layered over this command's TANGLE_ROOT_CONFIG entry, so the pre-parse sees + # what the command will. The dynamic command tree does not exist yet, + # so its identity comes from the leading command tokens. + configs = load_config(config_path, command=_api_command_identity(argv)) except ConfigFileError as exc: raise SystemExit(f"Config error: {exc}") from exc if not configs: return None - return configs[0].get(key) + return configs[0].values.get(key) def _optional_str(value: Any) -> str | None: diff --git a/packages/tangle-cli/src/tangle_cli/args_container.py b/packages/tangle-cli/src/tangle_cli/args_container.py index 67f9f6e..2a72874 100644 --- a/packages/tangle-cli/src/tangle_cli/args_container.py +++ b/packages/tangle-cli/src/tangle_cli/args_container.py @@ -8,6 +8,11 @@ in :class:`EnvField`) > default. Config values may themselves be read from the environment with the ``{_env: NAME}`` value directive. +``TANGLE_ROOT_CONFIG`` may name a per-command root config: the entry for the +command passed as ``command=`` deep-merges beneath that command's ``--config``, +so precedence is CLI > ``--config`` > ``TANGLE_ROOT_CONFIG`` > environment > +default (see :func:`load_config`). + Scalar fields are typed strictly: a field whose default is a bool/int/float (or whose spec names :func:`strict_bool` / :func:`strict_int` / :func:`strict_float`) parses a config or environment string into that type @@ -21,8 +26,9 @@ import json import os import re -from collections.abc import Callable -from dataclasses import dataclass +import difflib +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import Any, cast @@ -43,6 +49,15 @@ _ENV_DIRECTIVE_KEYS = (ENV_KEY, "default") # Bounds recursion while rewriting a document that actually uses ``_env``. _MAX_ENV_VALUE_DEPTH = 256 + +#: Environment variable naming the per-command root config file. +ROOT_CONFIG_ENV = "TANGLE_ROOT_CONFIG" +#: The one required top-level key of a TANGLE_ROOT_CONFIG document. +ROOT_COMMANDS_KEY = "commands" +#: Alternate command-path prefixes normalized to their canonical form. +COMMAND_ALIASES: Mapping[str, str] = {"tangle-cli": "tangle"} +# Bounds recursion while deep-merging TANGLE_ROOT_CONFIG beneath a command config. +_MAX_MERGE_DEPTH = 256 _MAX_RENDERED_PATH_LENGTH = 160 # Guards against recursive YAML aliases producing an endless selector chain. @@ -349,14 +364,19 @@ def _direct_env_sources(entry: dict[str, Any]) -> dict[str, str]: class ArgsContainer: """Container for resolved CLI arguments with config-file defaults.""" + #: Instance attributes that are bookkeeping, not resolved fields. + _PRIVATE_ATTRS = ("_config", "_config_sources", "_field_config_keys") + def __init__( self, resolved: dict[str, Any], raw_config: dict[str, Any], - origins: dict[str, str] | None = None, + config_sources: dict[Any, Path] | None = None, + field_config_keys: dict[str, Any] | None = None, ): self._config = raw_config - self._origins = dict(origins or {}) + self._config_sources = dict(config_sources or {}) + self._field_config_keys = dict(field_config_keys or {}) for key, value in resolved.items(): setattr(self, key, value) @@ -378,18 +398,22 @@ def to_dict(self) -> dict[str, Any]: return { key: value for key, value in vars(self).items() - if key not in ("_config", "_origins") + if key not in ArgsContainer._PRIVATE_ATTRS } - def origin(self, name: str) -> str | None: - """Return where field *name* was resolved from, never its value. + def config_source(self, name: str) -> Path | None: + """Return the config file that supplied field *name*, else ``None``. - One of ``"cli"``, ``"config"``, ``"env:NAME"`` (the :class:`EnvField` - tier), or ``"default"``; ``None`` for an unknown field. A config value - read through an ``_env`` directive reports ``"config"``. + For resolving a relative path value against the directory of the file + it was written in: a value inherited from ``TANGLE_ROOT_CONFIG`` names the + root file, a value from ``--config`` names that file. ``None`` when + the value came from the CLI, the environment tier, or the default. + Granularity is the top-level config key. """ - return self._origins.get(name) + if name not in self._field_config_keys: + return None + return self._config_sources.get(self._field_config_keys[name]) @staticmethod def _validate_branch_document( @@ -742,6 +766,18 @@ def _load_config_file( return [entry for entry, _ in ArgsContainer._load_config_entries(config_path, logger)] + @staticmethod + def _parse_config_file(path: Path) -> Any: + """Parse one YAML (``.yaml``/``.yml``) or JSON file; ``None`` for empty YAML.""" + + try: + with path.open(encoding="utf-8") as f: + if path.suffix in (".yaml", ".yml"): + return yaml.safe_load(f) + return json.load(f) + except (OSError, json.JSONDecodeError, yaml.YAMLError) as exc: + raise ConfigFileError(f"Error loading config file: {exc}") from exc + @staticmethod def _load_config_entries( config_path: str | Path | None, @@ -764,6 +800,9 @@ def _load_config_entries( then replaced by the variable's string value. Every directive is shape checked first, but only those in the selected document's config entries (and ``_defaults``) are looked up. + + This loader reads exactly one file and never consults ``TANGLE_ROOT_CONFIG``; + see :func:`load_config` for that. """ log = logger or get_default_logger() @@ -774,16 +813,9 @@ def _load_config_entries( if not path.exists(): raise ConfigFileError(f"Config file not found: {config_path}") - try: - with path.open(encoding="utf-8") as f: - if path.suffix in (".yaml", ".yml"): - parsed = yaml.safe_load(f) - if parsed is None: - return [({}, {})] - else: - parsed = json.load(f) - except (OSError, json.JSONDecodeError, yaml.YAMLError) as exc: - raise ConfigFileError(f"Error loading config file: {exc}") from exc + parsed = ArgsContainer._parse_config_file(path) + if parsed is None: + return [({}, {})] resolution = resolve_config_document(parsed, path) parsed = resolution.document @@ -901,6 +933,7 @@ def convert(value: Any) -> Any: def _resolve( config: dict[str, Any], env_sources: dict[str, str] | None = None, + loaded: LoadedConfig | None = None, /, **kwargs: Any, ) -> ArgsContainer: @@ -928,7 +961,7 @@ def _resolve( """ resolved: dict[str, Any] = {} - origins: dict[str, str] = {} + field_config_keys: dict[str, Any] = {} required_fields: list[str] = [] env_names: dict[str, str] = {} @@ -1004,7 +1037,8 @@ def _resolve( f"variable {env_name}" ) from None resolved[param_name] = value - origins[param_name] = origin + if origin == "config": + field_config_keys[param_name] = config_key for field_name in required_fields: if resolved.get(field_name) is None: @@ -1017,7 +1051,14 @@ def _resolve( f"{field_name} is required (via CLI argument or config file)" ) - return ArgsContainer(resolved, config, origins) + if loaded is None: + return ArgsContainer(resolved, config, field_config_keys=field_config_keys) + return ArgsContainer( + resolved, + config, + config_sources=loaded.sources, + field_config_keys=field_config_keys, + ) @staticmethod def _describe_source( @@ -1041,12 +1082,267 @@ def _describe_source( def load( config_path: str | Path | None, logger: Logger | None = None, + *, + command: str | None = None, **kwargs: Any, ) -> list[ArgsContainer]: - """Load a config file and resolve CLI args against each config entry.""" + """Load a config file and resolve CLI args against each config entry. + + *command* is the running command's full path (e.g. ``"tangle sdk + secrets delete"``); its ``TANGLE_ROOT_CONFIG`` entry, if any, is layered + beneath the config (see :func:`load_config`). Without it the root + config is ignored. + """ + + return [ + ArgsContainer._resolve(entry.values, entry.env_sources, entry, **kwargs) + for entry in load_config(config_path, logger, command=command) + ] + + +@dataclass(frozen=True) +class LoadedConfig: + """One resolved config entry, layered over ``TANGLE_ROOT_CONFIG``. + + ``values`` is the merged mapping a command sees. ``sources`` maps each + top-level key to the file that supplied it (``--config`` for any key it + defines, else the ``TANGLE_ROOT_CONFIG`` file), for resolving relative paths + against the right directory. + """ + + values: dict[str, Any] + sources: dict[Any, Path] = field(default_factory=lambda: {}) + env_sources: dict[str, str] = field(default_factory=lambda: {}, repr=False) + + +def _deep_merge( + base: dict[Any, Any], override: dict[Any, Any], path: _ConfigPath = () +) -> dict[Any, Any]: + """Merge *override* over *base*: mappings merge key by key; lists and + scalars replace wholesale. A ``null`` mapping value in *override* means + "absent": it removes a key *base* provides and is dropped otherwise.""" + + if len(path) > _MAX_MERGE_DEPTH: + raise ConfigFileError( + f"config nesting exceeds {_MAX_MERGE_DEPTH} levels while merging over {ROOT_CONFIG_ENV}" + ) + merged = dict(base) + for key, value in override.items(): + if value is None: + merged.pop(key, None) + continue + current = merged.get(key) + if key in merged and isinstance(current, dict) and isinstance(value, dict): + merged[key] = _deep_merge( + cast(dict[Any, Any], current), cast(dict[Any, Any], value), (*path, key) + ) + elif isinstance(value, dict): + # A mapping the root does not have: still drop its null leaves. + merged[key] = _deep_merge({}, cast(dict[Any, Any], value), (*path, key)) + else: + merged[key] = value + return merged - entries = ArgsContainer._load_config_entries(config_path, logger=logger) - return [ArgsContainer._resolve(config, sources, **kwargs) for config, sources in entries] + +def _reject_nulls(document: Any) -> None: + """Reject any ``null`` value in a TANGLE_ROOT_CONFIG document, naming its key path.""" + + seen: set[int] = set() + stack: list[tuple[Any, _ConfigPath]] = [(document, ())] + while stack: + node, path = stack.pop() + if isinstance(node, dict): + if id(node) in seen: + continue + seen.add(id(node)) + items: list[tuple[Any, Any]] = list(cast(dict[Any, Any], node).items()) + elif isinstance(node, list): + if id(node) in seen: + continue + seen.add(id(node)) + items = list(enumerate(cast(list[Any], node))) + else: + continue + for key, value in reversed(items): + if value is None: + raise ConfigFileError( + f"null is not allowed in {ROOT_CONFIG_ENV} (at {_render_config_path((*path, key))})" + ) + stack.append((value, (*path, key))) + + +def normalize_command(command: str, aliases: Mapping[str, str] | None = None) -> str: + """Canonical command identity: the full invocation path, program first. + + Whitespace runs collapse to single spaces, then the longest whole-token + prefix found in *aliases* (default :data:`COMMAND_ALIASES`, e.g. + ``tangle-cli`` -> ``tangle``) is replaced by its canonical path, so an + alias or deprecated group/command name maps to the one real path. + Matching is otherwise exact and case-sensitive. + """ + + parts = command.split() if isinstance(command, str) else [] + if not parts: + raise ConfigFileError("a command identity must be a non-empty string") + table = { + " ".join(k.split()): " ".join(v.split()) + for k, v in (COMMAND_ALIASES if aliases is None else aliases).items() + } + # Rewrite to a fixed point (a renamed program and a renamed group can both + # apply); bounded so a cyclic alias table cannot loop. + for _ in range(len(table) + 1): + for length in range(len(parts), 0, -1): + prefix = " ".join(parts[:length]) + if prefix in table and table[prefix] != prefix: + parts = [*table[prefix].split(), *parts[length:]] + break + else: + break + return " ".join(parts) + + +def _root_config_entry(command: str, logger: Logger | None) -> LoadedConfig | None: + """Load the running *command*'s ``TANGLE_ROOT_CONFIG`` entry, or ``None``. + + ``None`` when ``TANGLE_ROOT_CONFIG`` is unset/empty or has no entry for + *command*. A set variable naming a missing, unreadable, or malformed file + fails closed; diagnostics name the variable, the path, and command keys, + never a config value. + """ + + raw = os.environ.get(ROOT_CONFIG_ENV) + if not raw: + return None + path = Path(raw).expanduser() + if not path.exists(): + raise ConfigFileError(f"{ROOT_CONFIG_ENV} names a config file that does not exist: {path}") + if not path.is_file(): + raise ConfigFileError(f"{ROOT_CONFIG_ENV} must name a config file: {path}") + try: + return _select_root_entry(path, command, logger or get_default_logger()) + except ConfigFileError as exc: + raise ConfigFileError(f"{ROOT_CONFIG_ENV} ({path}): {exc}") from exc + + +def _select_root_entry(path: Path, command: str, log: Logger) -> LoadedConfig | None: + parsed = ArgsContainer._parse_config_file(path) + _reject_nulls(parsed) + # A document-level _select picks among whole `commands` documents first; + # every _env in the file (dormant commands included) is shape-checked here + # but none is read. + document = resolve_config_document(parsed, path, mapping_only=True).document + if not isinstance(document, dict) or ROOT_COMMANDS_KEY not in document: + raise ConfigFileError( + f"must be a mapping with a top-level '{ROOT_COMMANDS_KEY}' selector " + "keyed by command, e.g. 'tangle sdk pipeline-runs submit'" + ) + document_dict = cast(dict[Any, Any], document) + extra = sorted( + _render_config_key(key) + for key in document_dict + if key != ROOT_COMMANDS_KEY and not (isinstance(key, str) and key.startswith("_")) + ) + if extra: + raise ConfigFileError( + f"only '{ROOT_COMMANDS_KEY}' and underscore-prefixed helper keys may appear at " + f"the top level, got: {', '.join(extra)}" + ) + commands = document_dict[ROOT_COMMANDS_KEY] + if not isinstance(commands, dict) or ENV_KEY in commands: + raise ConfigFileError(f"'{ROOT_COMMANDS_KEY}' must map command names to config objects") + + table: dict[str, tuple[Any, Any]] = {} + for key, entry in cast(dict[Any, Any], commands).items(): + rendered = _render_config_key(key) + if not isinstance(key, str) or not key.split(): + raise ConfigFileError( + f"'{ROOT_COMMANDS_KEY}' keys must be non-empty command names, got {rendered}" + ) + name = normalize_command(key) + if name in table: + raise ConfigFileError( + f"'{ROOT_COMMANDS_KEY}' names command {_render_config_key(name)} more than once" + ) + if not isinstance(entry, dict) or ENV_KEY in entry: + raise ConfigFileError( + f"'{ROOT_COMMANDS_KEY}' entry {rendered} must be a config object" + ) + table[name] = (key, entry) + + if command not in table: + close = difflib.get_close_matches(command, list(table), n=1, cutoff=0.85) + if close: + log.warn( + f"{ROOT_CONFIG_ENV} ({path}) has no entry for {_render_config_key(command)}; " + f"did you mean {_render_config_key(close[0])}? No root config applied." + ) + return None + + key, entry = table[command] + # Then the entry's own _select (if any) and its _env lookups -- only here. + resolution = resolve_config_document(entry, path, mapping_only=True) + selected = cast(dict[str, Any], resolution.document) + values = resolution.resolve_entry(selected, (ROOT_COMMANDS_KEY, key)) + source = path.resolve() + return LoadedConfig(values, {k: source for k in values}, _direct_env_sources(selected)) + + +def load_config( + config_path: str | Path | None, + logger: Logger | None = None, + *, + command: str | None = None, +) -> list[LoadedConfig]: + """Load ``--config`` entries layered over the command's ``TANGLE_ROOT_CONFIG`` entry. + + The one loader for commands that take ``--config``; ``ArgsContainer.load`` + uses it, and callers reading config values directly should too. + *command* is the running command's full path (see :func:`normalize_command`), + passed explicitly by the CLI layer. With no *command* (e.g. a library + call), or no ``TANGLE_ROOT_CONFIG`` entry for it, this returns exactly the + ``--config`` entries and never reads the root file. Otherwise: + + * no ``--config`` -> the root entry is the single entry; + * each ``--config`` entry (after that file's own ``_defaults``, which stay + a shallow merge) is deep-merged over the root entry: mappings merge key + by key and lists and scalars replace. ``null`` is rejected anywhere in + the root file. In a ``--config`` layered over a root entry, a ``null`` + mapping value means "absent" at any depth: it unsets a key the root + entry provides and is dropped where the root does not set the key, so + resolution falls through to the env tier and default. Without an active + root entry, ``--config`` ``null`` keeps its existing meaning. + + Each file resolves its own ``_select`` / ``_env`` before merging. + """ + + identity = normalize_command(command) if command is not None else None + root = _root_config_entry(identity, logger) if identity is not None else None + if root is not None and config_path is None: + return [root] + entries = ArgsContainer._load_config_entries(config_path, logger) + + command_source = Path(config_path).resolve() if config_path is not None else None + loaded: list[LoadedConfig] = [] + for values, env_sources in entries: + if root is None: + own = {key: command_source for key in values} if command_source is not None else {} + loaded.append(LoadedConfig(values, own, env_sources)) + continue + merged = _deep_merge(root.values, values) + loaded.append( + LoadedConfig( + merged, + { + key: root.sources[key] if key not in values else cast(Path, command_source) + for key in merged + }, + { + **{k: v for k, v in root.env_sources.items() if k not in values}, + **{k: v for k, v in env_sources.items() if k in merged}, + }, + ) + ) + return loaded @dataclass(frozen=True) @@ -1106,14 +1402,20 @@ def resolve_config_document( __all__ = [ + "COMMAND_ALIASES", "ENV_KEY", + "ROOT_COMMANDS_KEY", + "ROOT_CONFIG_ENV", "SELECT_KEY", "ArgsContainer", "ConfigFileError", "EnvField", + "LoadedConfig", "ResolvedConfigDocument", "resolve_config_document", "strict_bool", "strict_float", "strict_int", + "load_config", + "normalize_command", ] diff --git a/packages/tangle-cli/src/tangle_cli/cli.py b/packages/tangle-cli/src/tangle_cli/cli.py index ad9541c..225b3b5 100644 --- a/packages/tangle-cli/src/tangle_cli/cli.py +++ b/packages/tangle-cli/src/tangle_cli/cli.py @@ -17,6 +17,7 @@ secrets_cli, ) from .api_transport import configure_cli_verify +from .cli_helpers import dispatching from .cli_options import CaBundleOption, VerifyTlsOption @@ -61,6 +62,33 @@ def _configure_tls_from_argv(argv: list[str]) -> None: configure_cli_verify(ca_bundle, verify_tls) +#: Canonical program name for command identities (``tangle-cli`` is an alias). +PROGRAM_NAME = "tangle" + + +def _command_identity(app: App, tokens: tuple[str, ...]) -> str | None: + """The dispatched command path, e.g. ``"tangle sdk secrets delete"``. + + Accumulated as Cyclopts resolves the command line: the program name, then + each group, then the leaf, with options and positional arguments excluded. + Each level is canonicalized to its first registered name, so an alias + resolves to the real command. Selects the command's + ``TANGLE_ROOT_CONFIG`` entry. + """ + + try: + chain, _apps, _unused = app.parse_commands(list(tokens)) + path = [PROGRAM_NAME] + current = app + for token in chain: + target = current[token] + path.append(next(name for name in current if current[name] is target)) + current = target + except Exception: # an unparsable command line reports its own error later + return None + return " ".join(path) if chain else None + + def build_sdk_app() -> App: """Build the SDK command group.""" @@ -105,7 +133,8 @@ def launcher( """Apply global TLS options, then dispatch the requested command.""" configure_cli_verify(ca_bundle, verify_tls) - app(tokens) + with dispatching(_command_identity(app, tokens)): + app(tokens) return app diff --git a/packages/tangle-cli/src/tangle_cli/cli_helpers.py b/packages/tangle-cli/src/tangle_cli/cli_helpers.py index 558069c..f9211ec 100644 --- a/packages/tangle-cli/src/tangle_cli/cli_helpers.py +++ b/packages/tangle-cli/src/tangle_cli/cli_helpers.py @@ -4,16 +4,44 @@ import json import pathlib +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any -from .args_container import ArgsContainer, ConfigFileError +from .args_container import ArgsContainer, ConfigFileError, load_config + +# Internal plumbing only: the dispatcher records the resolved command path here +# and the helpers below hand it to ArgsContainer explicitly as ``command=``. +_DISPATCHED_COMMAND: ContextVar[str | None] = ContextVar("tangle_dispatched_command", default=None) + + +@contextmanager +def dispatching(command: str | None) -> Iterator[None]: + """Record the command the CLI dispatcher resolved for the duration of a run.""" + + token = _DISPATCHED_COMMAND.set(command) + try: + yield + finally: + _DISPATCHED_COMMAND.reset(token) + + +def dispatched_command() -> str | None: + """The command path recorded by :func:`dispatching`, if any.""" + + return _DISPATCHED_COMMAND.get() def load_args_or_exit(config: str | None, **kwargs: Any) -> list[ArgsContainer]: - """Load ArgsContainer values from CLI/config specs, exiting with CLI errors.""" + """Load ArgsContainer values from CLI/config specs, exiting with CLI errors. + + Passes the dispatched command path to ``ArgsContainer.load(command=...)`` + so the command's ``TANGLE_ROOT_CONFIG`` entry applies. + """ try: - return ArgsContainer.load(config, **kwargs) + return ArgsContainer.load(config, command=dispatched_command(), **kwargs) except ConfigFileError as exc: raise SystemExit(f"Config error: {exc}") from exc @@ -25,15 +53,16 @@ def print_json(payload: object) -> None: def load_config_or_exit(config: str | None) -> dict[str, object]: - """Load the first YAML/JSON config mapping for commands with custom merging.""" + """Load the first YAML/JSON config mapping for commands with custom merging. + + Layered over ``TANGLE_ROOT_CONFIG`` like every ``--config`` (see ``load_config``). + """ - if config is None: - return {} try: - configs = ArgsContainer._load_config_file(config) + configs = load_config(config, command=dispatched_command()) except ConfigFileError as exc: raise SystemExit(f"Config error: {exc}") from exc - return configs[0] if configs else {} + return configs[0].values if configs else {} def optional_path(value: str | pathlib.Path | object | None) -> pathlib.Path | None: diff --git a/pyproject.toml b/pyproject.toml index 6607490..bc4bcb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.23" +version = "0.1.24" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/conftest.py b/tests/conftest.py index 969c353..ab18efa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,3 +95,10 @@ def downstream_authoring_surface(): sys.modules.pop(DOWNSTREAM_AUTHORING_MODULE, None) if not had_parent: sys.modules.pop(parent_name, None) + + +@pytest.fixture(autouse=True) +def _no_ambient_root_config(monkeypatch): + """Keep a developer's ``TANGLE_ROOT_CONFIG`` from layering into test configs.""" + + monkeypatch.delenv("TANGLE_ROOT_CONFIG", raising=False) diff --git a/tests/test_args_container_env.py b/tests/test_args_container_env.py index ca74792..e01d2d5 100644 --- a/tests/test_args_container_env.py +++ b/tests/test_args_container_env.py @@ -368,14 +368,35 @@ def test_env_field_conversion_error_does_not_echo_value(monkeypatch) -> None: assert SECRET not in "".join(traceback.format_exception(excinfo.value)) -def test_origin_reports_env_name_not_value(monkeypatch) -> None: +def test_env_tier_value_appears_only_as_its_field_value(tmp_path, monkeypatch) -> None: monkeypatch.setenv("TOKEN", SECRET) + config = _write(tmp_path, "other: 1\n") - [args] = ArgsContainer.load(None, token=EnvField("TOKEN", (None, None))) + [args] = ArgsContainer.load(config, token=EnvField("TOKEN", (None, None)), other=(None, None)) - assert args.token == SECRET - assert args.origin("token") == "env:TOKEN" - assert "_origins" not in args.to_dict() + assert args.to_dict() == {"token": SECRET, "other": 1} + bookkeeping = {k: v for k, v in vars(args).items() if k not in args.to_dict()} + assert set(bookkeeping) == set(ArgsContainer._PRIVATE_ATTRS) + assert SECRET not in repr(bookkeeping) + assert SECRET not in repr(args) + assert args.config_source("token") is None + + +def test_env_directive_value_stays_out_of_provenance(tmp_path, monkeypatch) -> None: + from tangle_cli.args_container import load_config + + monkeypatch.setenv("TOKEN", SECRET) + config = _write(tmp_path, "token: {_env: TOKEN}\nother: 1\n") + + [entry] = load_config(config) + [args] = ArgsContainer.load(config, token=(None, None)) + + # The resolved value is config data; provenance names only files and variables. + assert entry.values["token"] == SECRET and args.token == SECRET + assert entry.env_sources == {"token": "TOKEN"} + assert SECRET not in repr(entry.sources) + repr(entry.env_sources) + assert SECRET not in repr(args._config_sources) + repr(args._field_config_keys) + assert args.config_source("token") == config.resolve() # --- EnvField tier --------------------------------------------------------------------------- @@ -406,7 +427,8 @@ def test_env_field_precedence_matrix( ) assert args.token == expected - assert args.origin("token") == origin + expected_source = config_path.resolve() if origin == "config" else None + assert args.config_source("token") == expected_source def test_config_env_directive_beats_env_field_tier(tmp_path, monkeypatch) -> None: @@ -417,7 +439,7 @@ def test_config_env_directive_beats_env_field_tier(tmp_path, monkeypatch) -> Non [args] = ArgsContainer.load(config, token=EnvField("OTHER", (None, None))) assert args.token == "from-directive" - assert args.origin("token") == "config" + assert args.config_source("token") == config.resolve() def test_env_field_empty_string_counts_as_set(monkeypatch) -> None: @@ -426,7 +448,7 @@ def test_env_field_empty_string_counts_as_set(monkeypatch) -> None: [args] = ArgsContainer.load(None, token=EnvField("TOKEN", ("dflt", "dflt"))) assert args.token == "" - assert args.origin("token") == "env:TOKEN" + assert args.config_source("token") is None def test_env_field_satisfies_required_and_names_var_when_missing(monkeypatch) -> None: @@ -528,16 +550,13 @@ def test_plain_specs_never_read_same_named_env(tmp_path, monkeypatch) -> None: [args] = ArgsContainer.load(_write(tmp_path, "other: 1\n"), token=(None, None)) assert args.token is None - assert args.origin("token") == "default" -def test_plain_specs_origins(tmp_path) -> None: +def test_plain_specs_values(tmp_path) -> None: [args] = ArgsContainer.load( _write(tmp_path, "b: config\n"), a=("cli", None), b=(None, None), c=(None, None) ) - assert (args.origin("a"), args.origin("b"), args.origin("c")) == ("cli", "config", "default") - assert args.origin("unknown") is None assert args.to_dict() == {"a": "cli", "b": "config", "c": None} diff --git a/tests/test_args_container_typed.py b/tests/test_args_container_typed.py index d577f49..2b42e1b 100644 --- a/tests/test_args_container_typed.py +++ b/tests/test_args_container_typed.py @@ -141,7 +141,6 @@ def test_bool_field_accepts_documented_spellings_from_env(monkeypatch, text, exp [args] = ArgsContainer.load(None, flag=EnvField("FLAG", (False, False))) assert args.flag is expected - assert args.origin("flag") == "env:FLAG" @pytest.mark.parametrize("text", ["", "maybe", "on", "off", "t", "2", " yes", "tru\u0435", SECRET]) @@ -196,7 +195,7 @@ def test_explicit_strict_bool_accepts_cli_bools(tmp_path) -> None: [args] = ArgsContainer.load(None, dry_run=(True, None, strict_bool)) assert args.dry_run is True - assert args.origin("dry_run") == "cli" + assert args.config_source("dry_run") is None # came from the CLI, not a file # --- int / float fields ---------------------------------------------------------------------- @@ -356,7 +355,7 @@ def test_precedence_matrix_with_typing(tmp_path, monkeypatch, cli, config, env, [args] = ArgsContainer.load(path, flag=EnvField("FLAG", (cli, False))) assert args.flag is expected - assert args.origin("flag") == origin + assert args.config_source("flag") == (path.resolve() if origin == "config" else None) def test_defaults_env_source_is_named(tmp_path, monkeypatch) -> None: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 5a5635f..184bbd6 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.23" in metadata + assert "Version: 0.1.24" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/tests/test_pipeline_runs_cli.py b/tests/test_pipeline_runs_cli.py index 557c8ad..f6f854a 100644 --- a/tests/test_pipeline_runs_cli.py +++ b/tests/test_pipeline_runs_cli.py @@ -4745,3 +4745,38 @@ def refuse(*args: Any, **kwargs: Any): app(["sdk", "pipeline-runs", "submit-from-python", str(script), "--log-type", "none"]) assert "Cannot create a compile output" in str(exc_info.value) + + +def test_submit_from_python_root_config_entry_is_per_command(monkeypatch, tmp_path: Path, capsys): + """Only the running command's TANGLE_ROOT_CONFIG entry applies, and its keys + are checked like the command's own --config.""" + + script = _python_project(tmp_path) + root = tmp_path / "root.yaml" + root.write_text( + yaml.safe_dump( + { + "commands": { + # Valid for `submit`, never applied to submit-from-python. + "tangle sdk pipeline-runs submit": {"hydrate": True, "pipeline_path": "x.yaml"}, + "tangle sdk pipeline-runs submit-from-python": {"log_type": "none"}, + } + } + ) + ) + monkeypatch.setenv("TANGLE_ROOT_CONFIG", str(root)) + fake_client = FakeClient() + monkeypatch.setattr(pipeline_runs_cli, "LazyTangleApiClient", lambda **kwargs: fake_client) + + run_app(cli.build_app().meta, ["sdk", "pipeline-runs", "submit-from-python", str(script)]) + + assert len(fake_client.created) == 1 + assert capsys.readouterr().err == "" # log_type: none came from the root entry + + root.write_text( + yaml.safe_dump({"commands": {"tangle sdk pipeline-runs submit-from-python": {"hydrate": True}}}) + ) + with pytest.raises(SystemExit) as exc_info: + cli.build_app().meta(["sdk", "pipeline-runs", "submit-from-python", str(script)]) + assert "'hydrate' is not supported by submit-from-python" in str(exc_info.value) + assert len(fake_client.created) == 1 diff --git a/tests/test_root_config.py b/tests/test_root_config.py new file mode 100644 index 0000000..15277fd --- /dev/null +++ b/tests/test_root_config.py @@ -0,0 +1,657 @@ +"""``TANGLE_ROOT_CONFIG``: per-command root config layered beneath ``--config``.""" + +from __future__ import annotations + +import json +import traceback +from pathlib import Path + +import pytest +import yaml + +from tangle_cli import api_cli, cli, secrets_cli +from tangle_cli.args_container import ( + ArgsContainer, + ConfigFileError, + EnvField, + load_config, + normalize_command, + strict_bool, +) +from tangle_cli.cli_helpers import ( + dispatched_command, + dispatching, + include_env_credentials_for_args, + load_args_or_exit, + load_config_or_exit, +) +from tangle_cli.python_pipeline.cfg import load_cfg + +SECRET = "FAKE-SECRET-do-not-echo-7f3a9c2e1b" +ENV_VARS = ("RC_ENV", "RC_TOKEN", "RC_FLAG", "RC_TIER") +CMD = "tangle sdk pipeline-runs submit" +OTHER = "tangle-deploy pipeline-run submit" + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for name in ENV_VARS: + monkeypatch.delenv(name, raising=False) + + +def _write(directory: Path, name: str, text: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_text(text, encoding="utf-8") + return path + + +def _commands(entries: dict) -> str: + return yaml.safe_dump({"commands": entries}, sort_keys=False) + + +@pytest.fixture +def root(tmp_path, monkeypatch): + def set_root(text: str, name: str = "root.yaml") -> Path: + path = _write(tmp_path / "root", name, text) + monkeypatch.setenv("TANGLE_ROOT_CONFIG", str(path)) + return path + + return set_root + + +def _cmd(tmp_path: Path, text: str, name: str = "cmd.yaml") -> Path: + return _write(tmp_path / "cmd", name, text) + + +def _load(config, command=CMD, **specs): + return ArgsContainer.load(config, command=command, **specs) + + +# --- unset / empty / no identity ----------------------------------------------------------- + + +def test_unset_root_config_leaves_loading_unchanged(tmp_path) -> None: + config = _cmd(tmp_path, "a: 1\nannotations: {team: x}\n") + + [entry] = load_config(config, command=CMD) + [args] = _load(config, a=(None, None), annotations=(None, None)) + + assert entry.values == ArgsContainer._load_config_file(config)[0] + assert (args.a, args.annotations) == (1, {"team": "x"}) + assert load_config(None, command=CMD)[0].values == {} + + +def test_empty_root_config_is_a_no_op(monkeypatch) -> None: + monkeypatch.setenv("TANGLE_ROOT_CONFIG", "") + + [args] = _load(None, a=("dflt", "dflt")) + + assert args.a == "dflt" + + +def test_library_call_without_command_identity_ignores_root(tmp_path, root) -> None: + root(_commands({CMD: {"a": "from-root"}})) + config = _cmd(tmp_path, "b: 2\n") + + [args] = ArgsContainer.load(config, a=("dflt", "dflt"), b=(None, None)) + assert (args.a, args.b) == ("dflt", 2) + assert load_config(None)[0].values == {} + assert ArgsContainer._load_config_file(config) == [{"b": 2}] + + +def test_no_identity_does_not_even_read_a_broken_root(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("TANGLE_ROOT_CONFIG", str(tmp_path / "missing.yaml")) + + [args] = ArgsContainer.load(None, a=("dflt", "dflt")) + + assert args.a == "dflt" + + +# --- command selection ----------------------------------------------------------------------- + + +def test_matching_command_entry_is_the_config(root) -> None: + path = root(_commands({CMD: {"base_url": "https://api.root", "annotations": {"team": "search"}}})) + + [args] = _load(None, base_url=(None, None), annotations=(None, None)) + + assert args.base_url == "https://api.root" + assert args.annotations == {"team": "search"} + assert args.config_source("base_url") == path.resolve() + + +def test_non_matching_command_gets_no_root_config(root, tmp_path) -> None: + root(_commands({OTHER: {"annotations": {"team": "search"}}})) + config = _cmd(tmp_path, "limit: 3\n") + + [args] = _load(config, annotations=(None, None), limit=(None, None)) + + assert (args.annotations, args.limit) == (None, 3) + + +def test_both_clis_have_distinct_entries(root) -> None: + root(_commands({CMD: {"who": "tangle"}, OTHER: {"who": "tangle-deploy"}})) + + [upstream] = _load(None, who=(None, None)) + [downstream] = _load(None, command=OTHER, who=(None, None)) + + assert (upstream.who, downstream.who) == ("tangle", "tangle-deploy") + + +@pytest.mark.parametrize( + ("key", "identity"), + [ + ("tangle sdk\tpipeline-runs submit", CMD), + ("tangle-cli sdk pipeline-runs submit", CMD), + (CMD, "tangle-cli sdk pipeline-runs submit"), + (CMD, " tangle sdk pipeline-runs submit "), + ], +) +def test_command_names_are_normalized(root, key, identity) -> None: + root(_commands({key: {"a": 1}})) + + [args] = _load(None, command=identity, a=(None, None)) + + assert args.a == 1 + + +def test_normalize_command_maps_alias_and_deprecated_prefixes() -> None: + aliases = {"td": "tangle-deploy", "tangle-deploy pipeline-runs": "tangle-deploy pipeline-run"} + + assert normalize_command("td pipeline-runs submit", aliases) == "tangle-deploy pipeline-run submit" + assert normalize_command("tangle-deploy pipeline-runs", aliases) == "tangle-deploy pipeline-run" + assert normalize_command("tangle-deploy pipeline-runsx submit", aliases) == ( + "tangle-deploy pipeline-runsx submit" + ) + assert normalize_command("tangle-cli sdk secrets list") == "tangle sdk secrets list" + # A cyclic alias table terminates instead of looping. + assert normalize_command("a x", {"a": "b", "b": "a"}) in ("a x", "b x") + + +def test_command_match_is_case_sensitive_and_exact(root) -> None: + root(_commands({"Tangle sdk pipeline-runs submit": {"a": 1}, "tangle sdk pipeline-runs": {"a": 2}})) + + [args] = _load(None, a=(None, None)) + + assert args.a is None + + +def test_near_miss_command_key_warns_without_values(root, capsys) -> None: + root(_commands({"tangle sdk pipeline-run submit": {"token": SECRET}})) + + [args] = _load(None, token=(None, None)) + + assert args.token is None + err = capsys.readouterr().err + assert "has no entry for 'tangle sdk pipeline-runs submit'" in err + assert "did you mean 'tangle sdk pipeline-run submit'" in err + assert SECRET not in err + + +def test_normalize_command_rejects_empty() -> None: + with pytest.raises(ConfigFileError): + normalize_command(" ") + + +# --- layering -------------------------------------------------------------------------------- + + +def test_command_config_deep_merges_over_root_entry(tmp_path, root) -> None: + root( + _commands( + { + CMD: { + "base_url": "https://api.root", + "annotations": {"team": "search", "tier": "gold"}, + "header": ["X-Root: 1"], + "nested": {"a": {"x": 1, "y": 2}, "keep": True}, + } + } + ) + ) + config = _cmd( + tmp_path, + "annotations: {owner: me, tier: silver}\nheader: ['X-Cmd: 2']\nnested: {a: {y: 20, z: 30}}\n", + ) + + [args] = _load( + config, + base_url=(None, None), + annotations=(None, None), + header=(None, None), + nested=(None, None), + ) + + assert args.base_url == "https://api.root" + assert args.annotations == {"team": "search", "tier": "silver", "owner": "me"} + assert args.header == ["X-Cmd: 2"] # lists replace wholesale + assert args.nested == {"a": {"x": 1, "y": 20, "z": 30}, "keep": True} + + +def test_scalar_and_type_changes_replace(tmp_path, root) -> None: + root(_commands({CMD: {"limit": 5, "annotations": {"team": "x"}, "name": "root"}})) + config = _cmd(tmp_path, "limit: 7\nannotations: [not-a-map]\n") + + [entry] = load_config(config, command=CMD) + + assert entry.values == {"limit": 7, "annotations": ["not-a-map"], "name": "root"} + + +def test_cli_beats_both_and_replaces_whole_field(tmp_path, root) -> None: + root(_commands({CMD: {"annotations": {"team": "x"}, "limit": 5}})) + config = _cmd(tmp_path, "annotations: {owner: y}\n") + + [args] = _load(config, annotations=({"cli": "z"}, None), limit=(9, None)) + + assert (args.annotations, args.limit) == ({"cli": "z"}, 9) + assert args.config_source("annotations") is None + + +def test_precedence_cli_config_root_env_default(tmp_path, root, monkeypatch) -> None: + root(_commands({CMD: {"a": "root", "b": "root", "c": "root"}})) + config = _cmd(tmp_path, "a: config\nb: config\n") + monkeypatch.setenv("RC_TIER", "env") + + [args] = _load( + config, + a=EnvField("RC_TIER", ("cli", "dflt")), + b=EnvField("RC_TIER", ("dflt", "dflt")), + c=EnvField("RC_TIER", ("dflt", "dflt")), + d=EnvField("RC_TIER", ("dflt", "dflt")), + e=("dflt", "dflt"), + ) + + assert (args.a, args.b, args.c, args.d, args.e) == ("cli", "config", "root", "env", "dflt") + + +def test_typed_coercion_applies_after_merge(tmp_path, root, monkeypatch) -> None: + root(_commands({CMD: {"force": {"_env": "RC_FLAG", "default": False}, "retries": "3", "dry_run": "yes"}})) + monkeypatch.setenv("RC_FLAG", "false") + + [args] = _load(None, force=(False, False), retries=(0, 0), dry_run=(None, None, strict_bool)) + assert (args.force, args.retries, args.dry_run) == (False, 3, True) + + monkeypatch.setenv("RC_FLAG", "maybe") + unrelated = _cmd(tmp_path, "other: 1\n", name="unrelated.yaml") + with pytest.raises( + ConfigFileError, match=r"environment variable RC_FLAG \(_env at config key 'force'\)" + ): + _load(unrelated, force=(False, False)) + + [args] = _load(_cmd(tmp_path, "force: 'true'\n"), force=(False, False)) + assert args.force is True + + +# --- null ------------------------------------------------------------------------------------ + + +def test_null_in_command_config_unsets_an_inherited_key(tmp_path, root, monkeypatch) -> None: + root(_commands({CMD: {"token": "root-token", "annotations": {"team": "x", "owner": "y"}}})) + config = _cmd(tmp_path, "token: null\nannotations: {team: null}\n") + monkeypatch.setenv("RC_TIER", "from-env") + + [entry] = load_config(config, command=CMD) + [args] = _load( + config, + token=EnvField("RC_TIER", ("dflt", "dflt")), + annotations=(None, None), + ) + + assert entry.values == {"annotations": {"owner": "y"}} + assert "token" not in entry.sources + assert args.token == "from-env" # falls through to the env tier, then the default + assert args.annotations == {"owner": "y"} + [no_env] = _load(config, token=("dflt", "dflt")) + assert no_env.token == "dflt" + + +def test_null_where_root_sets_nothing_is_ignored_while_layering(tmp_path, root, monkeypatch) -> None: + root(_commands({CMD: {"annotations": {"team": "x"}}})) + config = _cmd( + tmp_path, + "token: null\nhydrate: null\nannotations: {owner: null}\nextra: {a: null, b: 1}\nlist: [1, null]\n", + ) + monkeypatch.setenv("RC_TIER", "from-env") + + [entry] = load_config(config, command=CMD) + [args] = _load( + config, + token=EnvField("RC_TIER", (None, None)), + hydrate=(True, True), + annotations=(None, None), + ) + + # Absent, not None: mapping leaves drop; list items are values and stay. + assert entry.values == {"annotations": {"team": "x"}, "extra": {"b": 1}, "list": [1, None]} + assert (args.token, args.hydrate) == ("from-env", True) + + +def test_null_without_an_active_root_entry_keeps_todays_meaning(tmp_path, root) -> None: + config = _cmd(tmp_path, "token: null\nhydrate: null\nannotations: {owner: null}\n") + expected = {"token": None, "hydrate": None, "annotations": {"owner": None}} + + [unset] = ArgsContainer.load(config, command=CMD, token=("dflt", "dflt"), hydrate=(True, True)) + assert load_config(config, command=CMD)[0].values == expected + assert (unset.token, unset.hydrate) == (None, None) + + root(_commands({OTHER: {"token": "other"}})) # set, but no entry for this command + assert load_config(config, command=CMD)[0].values == expected + assert load_config(config)[0].values == expected # no command identity + + +@pytest.mark.parametrize( + ("document", "location"), + [ + ({"commands": {CMD: {"token": None}}}, r"commands\.tangle sdk pipeline-runs submit\.token"), + ({"commands": {OTHER: {"a": {"b": [1, None]}}}}, r"commands\.tangle-deploy pipeline-run submit\.a\.b\[1\]"), + ({"_shared": None, "commands": {}}, r"_shared"), + ], +) +def test_null_anywhere_in_root_is_rejected(root, document, location) -> None: + path = root(yaml.safe_dump(document)) + + with pytest.raises(ConfigFileError, match=rf"^TANGLE_ROOT_CONFIG \({path}\): null is not allowed"): + _load(None, token=(None, None)) + with pytest.raises(ConfigFileError, match=rf"\(at {location}\)$"): + _load(None, token=(None, None)) + + +# --- _select / _env -------------------------------------------------------------------------- + + +def test_document_select_picks_a_command_map(root, monkeypatch) -> None: + root( + "_select:\n" + " env: RC_ENV\n" + " cases:\n" + f" prod: {{commands: {{'{CMD}': {{base_url: https://api.prod, token: {{_env: RC_TOKEN}}}}}}}}\n" + f" dev: {{commands: {{'{CMD}': {{base_url: https://api.dev, token: {{_env: RC_DORMANT}}}}}}}}\n" + ) + monkeypatch.setenv("RC_ENV", "prod") + monkeypatch.setenv("RC_TOKEN", "tok") + + [args] = _load(None, base_url=(None, None), token=(None, None)) + + assert (args.base_url, args.token) == ("https://api.prod", "tok") + + +def test_entry_select_and_env_resolve_inside_the_entry(tmp_path, root, monkeypatch) -> None: + root( + "commands:\n" + f" '{CMD}':\n" + " _select:\n" + " env: RC_ENV\n" + " cases:\n" + " prod: {base_url: https://api.prod, annotations: {env: prod}, token: {_env: RC_TOKEN}}\n" + " default: {base_url: https://api.dev}\n" + f" '{OTHER}': {{token: {{_env: RC_UNSET_FOR_OTHER_COMMAND}}}}\n" + ) + monkeypatch.setenv("RC_ENV", "prod") + monkeypatch.setenv("RC_TOKEN", "tok") + config = _cmd(tmp_path, "annotations: {owner: me}\n") + + [args] = _load(config, base_url=(None, None), token=(None, None), annotations=(None, None)) + assert (args.base_url, args.token) == ("https://api.prod", "tok") + assert args.annotations == {"env": "prod", "owner": "me"} + + monkeypatch.delenv("RC_ENV") + [dev] = _load(None, base_url=(None, None)) + assert dev.base_url == "https://api.dev" + + +@pytest.mark.parametrize("selection", [None, "prod"]) +def test_malformed_directive_in_any_command_fails_in_every_env(root, monkeypatch, selection) -> None: + if selection: + monkeypatch.setenv("RC_ENV", selection) + path = root(_commands({CMD: {"a": 1}, OTHER: {"token": {"_env": "RC_TOKEN", "fallback": "x"}}})) + + with pytest.raises(ConfigFileError, match=rf"^TANGLE_ROOT_CONFIG \({path}\): _env at commands\.tangle-deploy"): + _load(None, a=(None, None)) + + +def test_root_select_unmatched_fails_closed_without_echo(root, monkeypatch) -> None: + path = root(f"commands:\n '{CMD}':\n _select: {{env: RC_ENV, cases: {{prod: {{a: 1}}}}}}\n") + monkeypatch.setenv("RC_ENV", SECRET) + + with pytest.raises(ConfigFileError) as excinfo: + _load(None, a=(None, None)) + + assert str(excinfo.value).startswith(f"TANGLE_ROOT_CONFIG ({path}): Environment variable RC_ENV") + assert SECRET not in "".join(traceback.format_exception(excinfo.value)) + + +# --- multi-config documents ------------------------------------------------------------------ + + +def test_list_and_defaults_documents_merge_per_entry(tmp_path, root) -> None: + root(_commands({CMD: {"base_url": "https://api.root", "limit": 1, "annotations": {"team": "x"}}})) + listed = _cmd(tmp_path, "- {name: a}\n- {name: b, annotations: {owner: y}}\n", name="list.yaml") + defaults = _cmd( + tmp_path, + "_defaults:\n limit: 5\n annotations: {owner: d}\n" + "configs:\n - {name: a}\n - {name: b, limit: 9, annotations: {env: z}}\n", + name="defaults.yaml", + ) + + base = {"base_url": "https://api.root"} + assert [e.values for e in load_config(listed, command=CMD)] == [ + {**base, "limit": 1, "annotations": {"team": "x"}, "name": "a"}, + {**base, "limit": 1, "annotations": {"team": "x", "owner": "y"}, "name": "b"}, + ] + # root < _defaults (still a shallow merge in its own file) < entry. + assert [e.values for e in load_config(defaults, command=CMD)] == [ + {**base, "limit": 5, "annotations": {"team": "x", "owner": "d"}, "name": "a"}, + {**base, "limit": 9, "annotations": {"team": "x", "env": "z"}, "name": "b"}, + ] + + +# --- relative paths -------------------------------------------------------------------------- + + +def test_config_source_names_the_file_each_value_came_from(tmp_path, root) -> None: + root_path = root(_commands({CMD: {"module": "pipelines/root.py", "output": "out/root.yaml"}})) + config = _cmd(tmp_path, "output: out/cmd.yaml\n") + + [args] = _load(config, module=(None, None), output=(None, None), other=(None, None)) + + assert args.config_source("module") == root_path.resolve() + assert args.config_source("output") == config.resolve() + assert args.config_source("other") is None + source = args.config_source("module") + assert source is not None and source.parent != config.resolve().parent + + +def test_config_source_without_root_config(tmp_path) -> None: + config = _cmd(tmp_path, "output: out.yaml\n") + + [args] = ArgsContainer.load(config, output=(None, None)) + + assert args.config_source("output") == config.resolve() + + +# --- shape errors ---------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("base_url: https://x\n", "must be a mapping with a top-level 'commands' selector"), + ("", "must be a mapping with a top-level 'commands' selector"), + ("- {a: 1}\n", "must be a mapping with a top-level 'commands' selector"), + (_commands({CMD: {}}) + "base_url: x\n", "only 'commands' and underscore-prefixed helper keys"), + ("commands: [a]\n", "'commands' must map command names to config objects"), + ("commands: {_env: RC_TOKEN}\n", "'commands' must map command names to config objects"), + (_commands({CMD: [1]}), f"'commands' entry '{CMD}' must be a config object"), + (_commands({CMD: {"_env": "RC_TOKEN"}}), f"'commands' entry '{CMD}' must be a config object"), + ("commands: {' ': {}}\n", "'commands' keys must be non-empty command names"), + ( + _commands({CMD: {}, "tangle-cli sdk pipeline-runs submit": {}}), + f"'commands' names command '{CMD}' more than once", + ), + ], +) +def test_malformed_root_shapes_fail_closed(root, text, message) -> None: + path = root(text) + + with pytest.raises(ConfigFileError) as excinfo: + _load(None, a=(None, None)) + + assert str(excinfo.value).startswith(f"TANGLE_ROOT_CONFIG ({path}): ") + assert message in str(excinfo.value) + + +def test_underscore_helper_keys_allow_yaml_anchors(root) -> None: + root( + "_shared: &shared {annotations: {team: search}}\n" + "commands:\n" + f" '{CMD}': *shared\n" + f" '{OTHER}': {{<<: *shared, extra: 1}}\n" + ) + + [upstream] = _load(None, annotations=(None, None)) + [downstream] = _load(None, command=OTHER, annotations=(None, None), extra=(None, None)) + + assert upstream.annotations == downstream.annotations == {"team": "search"} + assert downstream.extra == 1 + + +def test_missing_and_directory_root_fail_closed(tmp_path, monkeypatch) -> None: + missing = tmp_path / "nope.yaml" + monkeypatch.setenv("TANGLE_ROOT_CONFIG", str(missing)) + with pytest.raises(ConfigFileError) as excinfo: + _load(None, a=(None, None)) + assert str(excinfo.value) == f"TANGLE_ROOT_CONFIG names a config file that does not exist: {missing}" + + monkeypatch.setenv("TANGLE_ROOT_CONFIG", str(tmp_path)) + with pytest.raises(ConfigFileError, match=r"^TANGLE_ROOT_CONFIG must name a config file: "): + _load(None, a=(None, None)) + + +@pytest.mark.parametrize( + ("name", "text"), + [("root.yaml", f"commands: {{x: '{SECRET}\n"), ("root.json", f'{{"commands": "{SECRET}",}}')], +) +def test_unparsable_root_names_var_and_path_without_echo(root, name, text) -> None: + path = root(text, name=name) + + with pytest.raises(ConfigFileError) as excinfo: + _load(None, a=(None, None)) + + assert str(excinfo.value).startswith(f"TANGLE_ROOT_CONFIG ({path}): ") + assert SECRET not in "".join(traceback.format_exception(excinfo.value)) + + +# --- CLI integration and other consumers ------------------------------------------------------- + + +def test_tangle_launcher_sets_the_dispatched_command_identity(root, monkeypatch, capsys) -> None: + root( + _commands( + { + "tangle sdk secrets delete": {"secret_name": "FROM_ROOT", "force": True, "log_type": "none"}, + "tangle sdk secrets list": {"force": False}, + } + ) + ) + + class Client: + deleted: list[str] = [] + + def __init__(self, **kwargs) -> None: + pass + + def secrets_delete(self, secret_name: str): + Client.deleted.append(secret_name) + + monkeypatch.setattr(secrets_cli, "LazyTangleApiClient", Client) + + try: + cli.build_app().meta(["sdk", "secrets", "delete"]) + except SystemExit as exc: + assert exc.code in (0, None) + + assert Client.deleted == ["FROM_ROOT"] + assert json.loads(capsys.readouterr().out)["secret_name"] == "FROM_ROOT" + assert dispatched_command() is None # the identity does not leak past dispatch + + +def test_direct_app_call_without_dispatcher_gets_no_root(root, monkeypatch) -> None: + root(_commands({"tangle sdk secrets delete": {"secret_name": "FROM_ROOT"}})) + + with pytest.raises(SystemExit, match="secret_name is required"): + cli.build_app()(["sdk", "secrets", "delete"]) + + +def test_launcher_identity_ignores_options_and_arguments() -> None: + app = cli.build_app() + + assert cli._command_identity(app, ("sdk", "secrets", "delete", "--force", "NAME")) == ( + "tangle sdk secrets delete" + ) + assert cli._command_identity(app, ()) is None + + +def test_launcher_identity_canonicalizes_cyclopts_aliases() -> None: + from cyclopts import App + + app = App(name="t") + group = App(name=["group", "group-old"]) + app.command(group) + + @group.command(name=["run", "run-old"]) + def run() -> None: # pragma: no cover - never invoked + return None + + assert cli._command_identity(app, ("group-old", "run-old", "--x", "1")) == "tangle group run" + + +def test_load_args_or_exit_passes_the_dispatched_command(root) -> None: + root(_commands({CMD: {"a": "from-root"}})) + + [outside] = load_args_or_exit(None, a=(None, None)) + with dispatching(CMD): + [inside] = load_args_or_exit(None, a=(None, None)) + + assert (outside.a, inside.a) == (None, "from-root") + + +def test_load_config_or_exit_and_api_preparse_use_the_command_entry(tmp_path, root) -> None: + root( + _commands( + { + CMD: {"base_url": "https://api.submit", "token": "root-token"}, + "tangle api pipeline-runs list": {"base_url": "https://api.list"}, + } + ) + ) + config = _cmd(tmp_path, "token: cmd-token\n") + + with dispatching(CMD): + assert load_config_or_exit(None) == {"base_url": "https://api.submit", "token": "root-token"} + assert load_config_or_exit(str(config))["token"] == "cmd-token" + assert load_config_or_exit(None) == {} + assert api_cli._config_value_from_argv(["pipeline-runs", "list", "--limit", "1"], "base_url") == ( + "https://api.list" + ) + assert api_cli._config_value_from_argv(["pipeline-runs", "get", "ID"], "base_url") is None + + +def test_root_base_url_counts_as_config_for_credential_isolation(root) -> None: + root(_commands({CMD: {"base_url": "https://api.root"}})) + + [args] = _load(None, base_url=(None, None)) + + assert include_env_credentials_for_args(args, cli_base_url=None) is False + + +def test_pipeline_cfg_is_not_layered(tmp_path, root) -> None: + root(_commands({CMD: {"value": "from-root"}})) + cfg_path = _write(tmp_path / "pipe", "config.yaml", "own: 1\n") + + with dispatching(CMD): + cfg = load_cfg(cfg_path) + + assert cfg.own == 1 + with pytest.raises(Exception, match="unknown config key"): + _ = cfg.value diff --git a/uv.lock b/uv.lock index 02cfbe1..f9d1a87 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.23" +version = "0.1.24" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" },