From f90f8cbdef673c7dce18d107bb1023bcf64d7854 Mon Sep 17 00:00:00 2001 From: Bardh Hoxha Date: Sun, 20 Sep 2026 13:07:47 -0400 Subject: [PATCH 1/3] fix: validate QP inputs and reject non-finite solver success --- .../quadratic_program/_validation.py | 44 +++++++ .../quadratic_program/qp_solver_casadi.py | 7 +- .../quadratic_program/qp_solver_cvxopt.py | 5 +- .../quadratic_program/qp_solver_jaxopt.py | 33 +---- .../quadratic_program/solver_registry.py | 23 ++-- .../test_solver_validation.py | 124 ++++++++++++++++++ 6 files changed, 195 insertions(+), 41 deletions(-) create mode 100644 src/cbfkit/optimization/quadratic_program/_validation.py create mode 100644 tests/test_optimization/test_solver_validation.py diff --git a/src/cbfkit/optimization/quadratic_program/_validation.py b/src/cbfkit/optimization/quadratic_program/_validation.py new file mode 100644 index 00000000..e64ec091 --- /dev/null +++ b/src/cbfkit/optimization/quadratic_program/_validation.py @@ -0,0 +1,44 @@ +"""Static QP shape checks shared by native and JAX solver adapters.""" + +from jax import Array + + +def validate_qp_shapes( + h_mat: Array, + f_vec: Array, + g_mat: Array | None = None, + h_vec: Array | None = None, + a_mat: Array | None = None, + b_vec: Array | None = None, +) -> None: + """Reject incomplete constraints and incompatible shapes, including under JIT. + + These checks use only static shapes, never traced array values. Numerical + convergence remains the backend's responsibility. + """ + if f_vec.ndim != 1 or f_vec.shape[0] == 0: + raise ValueError( + "Linear cost 'f_vec' must be a 1D array of shape (n_vars,) with n_vars > 0." + ) + n = f_vec.shape[0] + if h_mat.shape != (n, n): + raise ValueError( + f"Quadratic cost 'h_mat' must be a 2D array of shape ({n}, {n}), got {h_mat.shape}." + ) + for matrix, vector, matrix_name, vector_name, kind in ( + (g_mat, h_vec, "g_mat", "h_vec", "Inequality"), + (a_mat, b_vec, "a_mat", "b_vec", "Equality"), + ): + if (matrix is None) != (vector is None): + raise ValueError(f"'{matrix_name}' and '{vector_name}' must be supplied together.") + if matrix is not None and vector is not None: + if matrix.ndim != 2 or matrix.shape[1] != n: + raise ValueError( + f"{kind} constraint matrix '{matrix_name}' must be a 2D array " + f"of shape (n_constraints, {n}), got {matrix.shape}." + ) + if vector.shape != (matrix.shape[0],): + raise ValueError( + f"{kind} constraint bounds '{vector_name}' must be a 1D array " + f"of shape ({matrix.shape[0]},), got {vector.shape}." + ) diff --git a/src/cbfkit/optimization/quadratic_program/qp_solver_casadi.py b/src/cbfkit/optimization/quadratic_program/qp_solver_casadi.py index c0182b87..f6ee3ec5 100644 --- a/src/cbfkit/optimization/quadratic_program/qp_solver_casadi.py +++ b/src/cbfkit/optimization/quadratic_program/qp_solver_casadi.py @@ -15,6 +15,8 @@ from cbfkit.utils.user_types.solvers import QpSolution +from ._validation import validate_qp_shapes + def solve( h_mat: Array, @@ -39,6 +41,7 @@ def solve( solution: Solution to the QP status: True if optimal solution found """ + validate_qp_shapes(h_mat, f_vec, g_mat, h_vec, a_mat, b_vec) if ca is None: raise ImportError( "CasADi is not installed. Please install it with `pip install cbfkit[casadi]`." @@ -94,7 +97,9 @@ def solve( solution = solver(lbg=lbg, ubg=ubg) success = solver.stats()["success"] - return success * jnp.array(solution["x"]).reshape((n,)), success + primal = jnp.array(solution["x"]).reshape((n,)) + success = success and bool(np.all(np.isfinite(primal))) + return success * primal, success def solve_with_details( diff --git a/src/cbfkit/optimization/quadratic_program/qp_solver_cvxopt.py b/src/cbfkit/optimization/quadratic_program/qp_solver_cvxopt.py index ed761be6..27004a6a 100644 --- a/src/cbfkit/optimization/quadratic_program/qp_solver_cvxopt.py +++ b/src/cbfkit/optimization/quadratic_program/qp_solver_cvxopt.py @@ -9,6 +9,7 @@ import numpy as np from jax import Array +from cbfkit.optimization.quadratic_program._validation import validate_qp_shapes from cbfkit.utils.user_types.solvers import QpSolution # Resolve the optional, platform-specific backend once at import time. Keep its @@ -54,6 +55,7 @@ def solve( ------- (sol, success): Solution array and boolean success flag. """ + validate_qp_shapes(p_mat, q_vec, g_mat, h_vec, a_mat, b_vec) backend = _ensure_cvxopt() matrix, solvers = backend.matrix, backend.solvers @@ -89,7 +91,8 @@ def solve( if sol["status"] == "unknown": success = bool(np.all(np.array(g_mat) @ np.array(sol["x"]) - np.array(h_vec) <= 0)) - return jnp.array(sol["x"]).reshape((len(sol["x"]),)), success + primal = jnp.array(sol["x"]).reshape((len(sol["x"]),)) + return primal, success and bool(np.all(np.isfinite(primal))) def solve_with_details( diff --git a/src/cbfkit/optimization/quadratic_program/qp_solver_jaxopt.py b/src/cbfkit/optimization/quadratic_program/qp_solver_jaxopt.py index 8b650846..cc941e35 100644 --- a/src/cbfkit/optimization/quadratic_program/qp_solver_jaxopt.py +++ b/src/cbfkit/optimization/quadratic_program/qp_solver_jaxopt.py @@ -30,6 +30,8 @@ class QpStatus(enum.IntEnum): from cbfkit.utils.jit_monitor import JitMonitor +from ._validation import validate_qp_shapes + # Instantiate QP solver objects MAX_ITER = 10000 QP = OSQP(maxiter=MAX_ITER, tol=1e-4) @@ -121,6 +123,9 @@ def solve_inequality_constrained_qp( ) # We return the raw status code to allow callers to inspect failure reason. + status = jnp.where( + (status == QpStatus.SOLVED) & ~jnp.all(jnp.isfinite(sol.primal)), QpStatus.UNSOLVED, status + ) return QpSolution(primal=sol.primal, status=status, params=(sol, state)) @@ -151,33 +156,7 @@ def solve_with_details( """ JitMonitor.increment("qp_solver_jaxopt.solve_with_details") - if f_vec.ndim != 1: - raise ValueError( - f"Linear cost 'f_vec' must be a 1D array of shape (n_vars,), but got {f_vec.shape}. " - "Ensure it is a flat array, not a column vector." - ) - if h_mat.ndim != 2: - raise ValueError( - f"Quadratic cost 'h_mat' must be a 2D array of shape (n_vars, n_vars), but got {h_mat.shape}." - ) - if h_vec is not None and h_vec.ndim != 1: - raise ValueError( - f"Inequality constraint bounds 'h_vec' must be a 1D array of shape (n_ineq,), but got {h_vec.shape}. " - "Ensure it is a flat array, not a column vector." - ) - if g_mat is not None and g_mat.ndim != 2: - raise ValueError( - f"Inequality constraint matrix 'g_mat' must be a 2D array of shape (n_ineq, n_vars), but got {g_mat.shape}." - ) - if b_vec is not None and b_vec.ndim != 1: - raise ValueError( - f"Equality constraint bounds 'b_vec' must be a 1D array of shape (n_eq,), but got {b_vec.shape}. " - "Ensure it is a flat array, not a column vector." - ) - if a_mat is not None and a_mat.ndim != 2: - raise ValueError( - f"Equality constraint matrix 'a_mat' must be a 2D array of shape (n_eq, n_vars), but got {a_mat.shape}." - ) + validate_qp_shapes(h_mat, f_vec, g_mat, h_vec, a_mat, b_vec) params_obj = (h_mat, 0.5 * f_vec) params_eq = None if (a_mat is None or b_vec is None) else (a_mat, b_vec) diff --git a/src/cbfkit/optimization/quadratic_program/solver_registry.py b/src/cbfkit/optimization/quadratic_program/solver_registry.py index 92274150..fdebd645 100644 --- a/src/cbfkit/optimization/quadratic_program/solver_registry.py +++ b/src/cbfkit/optimization/quadratic_program/solver_registry.py @@ -40,6 +40,8 @@ from cbfkit.utils.user_types.solvers import QpSolution, QpSolverCallable, with_solver_metadata +from ._validation import validate_qp_shapes + # --------------------------------------------------------------------------- # Factory functions — each returns a QpSolverCallable # --------------------------------------------------------------------------- @@ -80,16 +82,7 @@ def solve_with_details( ) -> QpSolution: JitMonitor.increment("qp_solver_jaxopt.solve_with_details") - if f_vec.ndim != 1: - raise ValueError( - f"Linear cost 'f_vec' must be a 1D array of shape (n_vars,), " - f"but got {f_vec.shape}." - ) - if h_mat.ndim != 2: - raise ValueError( - f"Quadratic cost 'h_mat' must be a 2D array of shape (n_vars, n_vars), " - f"but got {h_mat.shape}." - ) + validate_qp_shapes(h_mat, f_vec, g_mat, h_vec, a_mat, b_vec) params_obj = (h_mat, 0.5 * f_vec) params_eq = None if (a_mat is None or b_vec is None) else (a_mat, b_vec) @@ -113,6 +106,7 @@ def solve_with_details( status, ) + status = jnp.where((status == 1) & ~jnp.all(jnp.isfinite(sol.primal)), 0, status) return QpSolution(primal=sol.primal, status=status, params=(sol, state)) return with_solver_metadata(solve_with_details, name="jaxopt", jit_compatible=True) @@ -200,10 +194,14 @@ def solve_with_details( "support equality constraints." ) - if g_mat is None or h_vec is None: + validate_qp_shapes(h_mat, f_vec, g_mat, h_vec, a_mat, b_vec) + + if g_mat is None or g_mat.shape[0] == 0: # Registry convention min x'Hx + f'x => 2H x* = -f. x = jnp.linalg.solve(2.0 * h_mat, -f_vec) - return QpSolution(primal=x, status=1, params=None) + return QpSolution( + primal=x, status=jnp.where(jnp.all(jnp.isfinite(x)), 1, 0), params=None + ) # Extract warm-start state from previous QpSolution.params warm: Optional[PdipmState] = None @@ -245,6 +243,7 @@ def _keep(_): bad = (status != 1) | ~jnp.all(jnp.isfinite(sol)) sol, status, state = jax.lax.cond(bad, _cold, _keep, None) + status = jnp.where((status == 1) & ~jnp.all(jnp.isfinite(sol)), 0, status) return QpSolution(primal=sol, status=status, params=(sol, state)) return with_solver_metadata(solve_with_details, name="fast", jit_compatible=True) diff --git a/tests/test_optimization/test_solver_validation.py b/tests/test_optimization/test_solver_validation.py new file mode 100644 index 00000000..072ffc2c --- /dev/null +++ b/tests/test_optimization/test_solver_validation.py @@ -0,0 +1,124 @@ +"""Malformed QPs must not silently become different, successful problems.""" + +from types import SimpleNamespace + +import jax +import jax.numpy as jnp +import pytest + +from cbfkit.optimization.quadratic_program import get_solver + + +@pytest.fixture(params=["fast", "jaxopt", "cvxopt", "casadi"]) +def solver(request): + return get_solver(request.param) + + +@pytest.mark.parametrize("given", ["g_mat", "h_vec"]) +def test_incomplete_inequality_is_rejected(solver, given): + value = jnp.ones((1, 2)) if given == "g_mat" else jnp.ones(1) + with pytest.raises(ValueError, match="supplied together"): + solver(jnp.eye(2), jnp.zeros(2), **{given: value}) + + +@pytest.mark.parametrize("given", ["a_mat", "b_vec"]) +def test_incomplete_equality_is_rejected(solver, given): + value = jnp.ones((1, 2)) if given == "a_mat" else jnp.ones(1) + error = NotImplementedError if solver.solver_name == "fast" else ValueError + with pytest.raises(error): + solver(jnp.eye(2), jnp.zeros(2), **{given: value}) + + +@pytest.mark.parametrize( + "h_shape,f_shape,g_shape,b_shape", + [ + ((2, 3), (2,), (1, 2), (1,)), + ((2, 2), (3,), (1, 2), (1,)), + ((2, 2), (2, 1), (1, 2), (1,)), + ((0, 0), (0,), (1, 0), (1,)), + ((2, 2), (2,), (1, 3), (1,)), + ((2, 2), (2,), (2,), (1,)), + ((2, 2), (2,), (1, 2), (2,)), + ((2, 2), (2,), (1, 2), (1, 1)), + ], +) +def test_incompatible_shapes_are_rejected(solver, h_shape, f_shape, g_shape, b_shape): + with pytest.raises(ValueError): + solver(jnp.ones(h_shape), jnp.zeros(f_shape), jnp.ones(g_shape), jnp.ones(b_shape)) + + +@pytest.mark.parametrize("name", ["fast", "jaxopt"]) +def test_incomplete_inequality_is_rejected_under_jit(name): + solver = get_solver(name) + + @jax.jit + def call(h, f, g): + return solver(h, f, g_mat=g).primal + + with pytest.raises(ValueError, match="supplied together"): + call(jnp.eye(2), jnp.zeros(2), jnp.ones((1, 2))) + + +@pytest.mark.parametrize("name", ["jaxopt", "cvxopt", "casadi"]) +@pytest.mark.parametrize("a_shape,b_shape", [((1, 3), (1,)), ((1, 2), (2,)), ((2,), (1,))]) +def test_equality_dimensions_are_checked(name, a_shape, b_shape): + with pytest.raises(ValueError): + get_solver(name)(jnp.eye(2), jnp.zeros(2), a_mat=jnp.ones(a_shape), b_vec=jnp.ones(b_shape)) + + +@pytest.mark.parametrize("name", ["fast", "jaxopt"]) +def test_incompatible_shapes_are_rejected_under_jit(name): + solver = get_solver(name) + + @jax.jit + def call(h, f): + return solver(h, f, jnp.ones((1, 2)), jnp.ones(1)).primal + + with pytest.raises(ValueError, match="h_mat"): + call(jnp.eye(3), jnp.zeros(2)) + + +@pytest.mark.parametrize("name", ["fast", "jaxopt"]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_nonfinite_constrained_problem_cannot_succeed(name, compiled): + solver = get_solver(name, max_iter=20) + + def call(f): + return solver(jnp.eye(1), f, jnp.ones((1, 1)), jnp.ones(1)).status + + call = jax.jit(call) if compiled else call + assert int(call(jnp.array([jnp.nan]))) != 1 + + +@pytest.mark.parametrize("backend_status,expected", [(1, 0), (2, 2), (3, 3)]) +def test_nonfinite_guard_preserves_backend_failure_codes(monkeypatch, backend_status, expected): + from jaxopt import OSQP + + def run(self, **kwargs): + return ( + SimpleNamespace(primal=jnp.array([jnp.nan])), + SimpleNamespace(status=jnp.asarray(backend_status), iter_num=jnp.asarray(1)), + ) + + monkeypatch.setattr(OSQP, "run", run) + result = get_solver("jaxopt")(jnp.eye(1), jnp.zeros(1), jnp.ones((1, 1)), jnp.ones(1)) + assert int(result.status) == expected + + +@pytest.mark.parametrize("compiled", [False, True]) +@pytest.mark.parametrize("empty_constraints", [False, True]) +def test_fast_unconstrained_failure_status(compiled, empty_constraints): + solver = get_solver("fast") + + def call(h, f): + kwargs = {"g_mat": jnp.empty((0, 1)), "h_vec": jnp.empty(0)} if empty_constraints else {} + result = solver(h, f, **kwargs) + return result.primal, result.status + + call = jax.jit(call) if compiled else call + primal, status = call(jnp.zeros((1, 1)), jnp.array([-1.0])) + assert not jnp.all(jnp.isfinite(primal)) + assert int(status) != 1 + primal, status = call(jnp.eye(1), jnp.array([-4.0])) + assert int(status) == 1 + assert jnp.allclose(primal, jnp.array([2.0])) From 9d054339991318bc3d45498048c6ee54e1d8b64a Mon Sep 17 00:00:00 2001 From: Bardh Hoxha Date: Sun, 20 Sep 2026 13:11:22 -0400 Subject: [PATCH 2/3] fix: reject incomplete benchmark metrics and empty runs --- src/cbfkit/benchmarks/metrics.py | 58 ++++++++--- src/cbfkit/benchmarks/registry.py | 2 +- src/cbfkit/benchmarks/runner.py | 27 ++++-- src/cbfkit/benchmarks/solver_stress.py | 10 +- src/cbfkit/benchmarks/sweep.py | 25 +++-- src/cbfkit/benchmarks/sweep_viz.py | 27 +++--- src/cbfkit/cli/bench.py | 23 +++-- tests/test_benchmarks/test_metrics.py | 95 +++++++++++++++++++ .../test_benchmarks/test_sweep_validation.py | 72 ++++++++++++++ 9 files changed, 281 insertions(+), 58 deletions(-) create mode 100644 tests/test_benchmarks/test_metrics.py create mode 100644 tests/test_benchmarks/test_sweep_validation.py diff --git a/src/cbfkit/benchmarks/metrics.py b/src/cbfkit/benchmarks/metrics.py index 30e8fedd..d41167c8 100644 --- a/src/cbfkit/benchmarks/metrics.py +++ b/src/cbfkit/benchmarks/metrics.py @@ -4,31 +4,58 @@ __all__ = ["mean", "rate", "summarize"] +import math from typing import Mapping, Sequence -Record = Mapping[str, float | int | bool | str] +Record = Mapping[str, float | int | bool | str | None] def _to_float(value: float | int | bool | str) -> float: if isinstance(value, bool): return float(value) if isinstance(value, (int, float)): - return float(value) - raise TypeError(f"Non-numeric metric value: {value!r}") + result = float(value) + if math.isfinite(result): + return result + raise ValueError(f"Non-finite metric value: {value!r}") + raise ValueError(f"Non-numeric metric value: {value!r}") -def mean(records: Sequence[Record], key: str) -> float: - if not records: - return 0.0 - return sum(_to_float(record.get(key, 0.0)) for record in records) / len(records) +def metric_value(record: Record, key: str) -> float: + """Require a measured, finite metric for comparisons and decisions.""" + if key not in record: + raise ValueError(f"Unknown or missing metric {key!r}.") + value = record[key] + if value is None: + raise ValueError(f"Metric {key!r} is unavailable.") + return _to_float(value) + +def mean(records: Sequence[Record], key: str) -> float | None: + """Average a metric; explicit None means the measurement is unavailable. -def rate(records: Sequence[Record], key: str) -> float: + Every record must declare the key. Partial measurements produce None, + rather than averaging a selected subset or treating absent values as zero. + """ + if not records: + raise ValueError("Cannot summarize an empty benchmark run.") + values = [] + for index, record in enumerate(records): + if key not in record: + raise ValueError(f"Record {index} is missing required metric {key!r}.") + value = record[key] + values.append(None if value is None else _to_float(value)) + if any(value is None for value in values): + return None + return math.fsum(value / len(values) for value in values if value is not None) + + +def rate(records: Sequence[Record], key: str) -> float | None: """Alias for mean over binary or count-like key values.""" return mean(records, key) -def summarize(records: Sequence[Record]) -> dict[str, float]: +def summarize(records: Sequence[Record]) -> dict[str, float | None]: """Produce common summary statistics for benchmark outputs. In addition to the fixed metrics, any extra numeric keys found in the @@ -36,7 +63,7 @@ def summarize(records: Sequence[Record]) -> dict[str, float]: (e.g. ``final_goal_distance``, ``time_to_goal``) are carried through to the sweep summary without requiring changes here. """ - fixed: dict[str, float] = { + fixed: dict[str, float | None] = { "num_runs": float(len(records)), "success_rate": rate(records, "success"), "safety_violation_rate": rate(records, "safety_violations"), @@ -44,15 +71,16 @@ def summarize(records: Sequence[Record]) -> dict[str, float]: "avg_step_ms": mean(records, "avg_step_ms"), } - # Collect extra numeric keys from the first record (all records share the same schema) + # Include optional metrics from every record. Missing optional measurements + # remain unavailable in the summary instead of being silently discarded. fixed_keys = set(fixed.keys()) | {"success", "safety_violations", "solver_failures"} extra_keys: set[str] = set() - if records: - for k, v in records[0].items(): - if k not in fixed_keys and isinstance(v, (int, float, bool)): + for record in records: + for k, v in record.items(): + if k not in fixed_keys and (v is None or isinstance(v, (int, float, bool))): extra_keys.add(k) for k in sorted(extra_keys): - fixed[k] = mean(records, k) + fixed[k] = mean([{k: record.get(k)} for record in records], k) return fixed diff --git a/src/cbfkit/benchmarks/registry.py b/src/cbfkit/benchmarks/registry.py index a130b8e4..1ef4ba5c 100644 --- a/src/cbfkit/benchmarks/registry.py +++ b/src/cbfkit/benchmarks/registry.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Sequence -BenchmarkResult = Mapping[str, float | int | bool | str] +BenchmarkResult = Mapping[str, float | int | bool | str | None] BenchmarkScenario = Callable[[int], BenchmarkResult] SweepableRunner = Callable[[int, Dict[str, Any]], BenchmarkResult] BatchSweepableRunner = Callable[[Sequence[int], Dict[str, Any]], Sequence[BenchmarkResult]] diff --git a/src/cbfkit/benchmarks/runner.py b/src/cbfkit/benchmarks/runner.py index 82861e2b..9589a174 100644 --- a/src/cbfkit/benchmarks/runner.py +++ b/src/cbfkit/benchmarks/runner.py @@ -10,7 +10,7 @@ from typing import Iterable, Sequence from ._progress import make_progress -from .metrics import summarize +from .metrics import metric_value, summarize from .registry import registry @@ -18,24 +18,29 @@ class BenchmarkRun: scenario: str seeds: Sequence[int] - records: list[dict[str, float | int | bool | str]] - summary: dict[str, float] + records: list[dict[str, float | int | bool | str | None]] + summary: dict[str, float | None] def _parse_seeds(seeds: str | Iterable[int]) -> list[int]: if isinstance(seeds, str): if ":" in seeds: start, end = seeds.split(":", maxsplit=1) - return list(range(int(start), int(end) + 1)) - return [int(token) for token in seeds.split(",") if token] - return [int(seed) for seed in seeds] + parsed = list(range(int(start), int(end) + 1)) + else: + parsed = [int(token) for token in seeds.split(",")] + else: + parsed = [int(seed) for seed in seeds] + if not parsed: + raise ValueError("A benchmark run requires at least one seed.") + return parsed def run_scenario(name: str, seeds: str | Iterable[int]) -> BenchmarkRun: parsed_seeds = _parse_seeds(seeds) spec = registry.scenario(name) - records: list[dict[str, float | int | bool | str]] = [] + records: list[dict[str, float | int | bool | str | None]] = [] with make_progress() as progress: task = progress.add_task(f"Seeds ({name})", total=len(parsed_seeds)) for seed in parsed_seeds: @@ -62,7 +67,9 @@ def write_artifacts(run: BenchmarkRun, output_dir: str | Path) -> None: "summary": run.summary, "records": run.records, } - (out / "results.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + (out / "results.json").write_text( + json.dumps(payload, indent=2, allow_nan=False), encoding="utf-8" + ) keys = sorted({key for rec in run.records for key in rec.keys()}) with (out / "records.csv").open("w", newline="", encoding="utf-8") as handle: @@ -76,8 +83,8 @@ def compare_runs( right: BenchmarkRun, metric: str, ) -> dict[str, float]: - left_value = float(left.summary.get(metric, 0.0)) - right_value = float(right.summary.get(metric, 0.0)) + left_value = metric_value(left.summary, metric) + right_value = metric_value(right.summary, metric) return { "left": left_value, "right": right_value, diff --git a/src/cbfkit/benchmarks/solver_stress.py b/src/cbfkit/benchmarks/solver_stress.py index 830a142f..3f90d8aa 100644 --- a/src/cbfkit/benchmarks/solver_stress.py +++ b/src/cbfkit/benchmarks/solver_stress.py @@ -128,9 +128,9 @@ def nominal_wrapper(t, x, k, ref): if k.endswith("solver_status"): status_key = k - avg_iter = 0.0 - max_iter = 0.0 - failures = 0 + avg_iter = None + max_iter = None + failures = None if iter_key and len(values) > 0: idx = keys.index(iter_key) @@ -153,6 +153,8 @@ def nominal_wrapper(t, x, k, ref): "avg_solver_iter": avg_iter, "max_solver_iter": max_iter, "solver_failures": failures, - "success": int(failures == 0), # Simple success definition + "success": None if failures is None else int(failures == 0), + # This scenario measures solver behavior, not barrier violations. + "safety_violations": None, "final_dist": float(jnp.linalg.norm(results.states[-1, :2] - jnp.array([10.0, 10.0]))), } diff --git a/src/cbfkit/benchmarks/sweep.py b/src/cbfkit/benchmarks/sweep.py index 816cce75..72a551fe 100644 --- a/src/cbfkit/benchmarks/sweep.py +++ b/src/cbfkit/benchmarks/sweep.py @@ -29,7 +29,7 @@ from ._progress import console as _console from ._progress import make_progress as _make_progress -from .metrics import summarize +from .metrics import metric_value, summarize from .registry import BatchSweepableRunner, SweepableRunner from .sweep_viz import SweepViz @@ -121,10 +121,7 @@ def _resolve_falsifier_kwargs( def _is_failure(result: dict[str, Any], metric: str) -> bool: """Check if a result indicates a failure.""" - val = result.get(metric, 0) - if isinstance(val, bool): - return val - return float(val) > 0 + return metric_value(result, metric) > 0 def _run_combo( @@ -158,6 +155,8 @@ def _run_combo( # Use batch runner when available and falsifier is off if batch_runner is not None and not falsifier: batch_results = batch_runner(seeds, combo) + if len(batch_results) != len(seeds): + raise ValueError("Batch runner must return one result per requested seed.") for seed, result in zip(seeds, batch_results): result = dict(result) result["seed"] = seed @@ -298,6 +297,9 @@ def run_sweep( Optional live visualization. When provided the sweep renders a colour-coded results table and scatter plot in the terminal. """ + if not seeds: + raise ValueError("A benchmark sweep requires at least one seed.") + falsifier, falsifier_metric = _resolve_falsifier_kwargs( falsifier, falsifier_metric, @@ -305,6 +307,9 @@ def run_sweep( failure_metric, ) + if not param_combos: + raise ValueError("A benchmark sweep requires at least one parameter combination.") + records: list[dict[str, Any]] = [] per_combo_summaries: list[dict[str, Any]] = [] skipped = 0 @@ -437,6 +442,9 @@ def run_optuna_sweep( Requires ``pip install cbfkit[optuna]``. """ + if not seeds: + raise ValueError("A benchmark sweep requires at least one seed.") + falsifier, falsifier_metric = _resolve_falsifier_kwargs( falsifier, falsifier_metric, @@ -444,6 +452,9 @@ def run_optuna_sweep( failure_metric, ) + if n_trials <= 0: + raise ValueError("An Optuna sweep requires at least one trial.") + try: import optuna except ImportError as exc: @@ -490,10 +501,10 @@ def objective(trial) -> float: ) summary = per_combo_summaries[-1] - obj_val = summary.get(objective_metric, 0.0) + obj_val = metric_value(summary, objective_metric) if safety_constraint is not None: sc_metric, sc_max = safety_constraint - if summary.get(sc_metric, 0.0) > sc_max: + if metric_value(summary, sc_metric) > sc_max: obj_val = float("inf") if viz is not None: diff --git a/src/cbfkit/benchmarks/sweep_viz.py b/src/cbfkit/benchmarks/sweep_viz.py index 554a4309..fe80d7f2 100644 --- a/src/cbfkit/benchmarks/sweep_viz.py +++ b/src/cbfkit/benchmarks/sweep_viz.py @@ -144,10 +144,14 @@ def __init__( self.n_seeds = n_seeds def add_result(self, combo: dict[str, Any], summary: dict[str, Any]) -> None: + val = summary.get(self.objective_metric) + if not isinstance(val, (int, float)): + raise ValueError( + f"Objective metric {self.objective_metric!r} is missing or unavailable." + ) idx = len(self.trials) self.trials.append({"combo": dict(combo), "summary": dict(summary)}) - val = summary.get(self.objective_metric, 0.0) - if summary.get("safety_violations", 0) == 0: + if summary.get("safety_violation_rate", summary.get("safety_violations")) == 0: self._safe_count += 1 if not math.isfinite(val): return @@ -260,8 +264,8 @@ def _render_table(self) -> Table: for i, trial in enumerate(self.trials): combo = trial["combo"] summary = trial["summary"] - obj_val = summary.get(self.objective_metric, 0.0) - violations = summary.get("safety_violations", 0) + obj_val = summary[self.objective_metric] + violations = summary.get("safety_violation_rate", summary.get("safety_violations")) safe = violations == 0 color = _color_for_value(obj_val, lo, hi, self.direction) is_best = i == self._best_idx @@ -301,11 +305,12 @@ def _render_table(self) -> Table: row.append(Text("--", style="dim")) # Safety with violation count - if safe: + if violations is None or not math.isfinite(violations): + row.append(Text("N/A", style="dim")) + elif safe: row.append(Text("\u2713", style="green")) else: - count = int(violations) if isinstance(violations, (int, float)) else violations - row.append(Text(f"\u2717 {count}", style="red bold")) + row.append(Text(f"\u2717 {violations:g}", style="red bold")) table.add_row(*row) @@ -374,7 +379,7 @@ def _scatter_2d(self, x_param: str, y_param: str) -> Text: W, H = 50, 15 xs = [float(t["combo"][x_param]) for t in self.trials] ys = [float(t["combo"][y_param]) for t in self.trials] - objs = [t["summary"].get(self.objective_metric, 0.0) for t in self.trials] + objs = [t["summary"][self.objective_metric] for t in self.trials] x_range = self._safe_range(min(xs), max(xs)) y_range = self._safe_range(min(ys), max(ys)) @@ -403,7 +408,7 @@ def _scatter_2d(self, x_param: str, y_param: str) -> Text: def _scatter_1d(self, x_param: str) -> Text: W, H = 50, 10 xs = [float(t["combo"][x_param]) for t in self.trials] - objs = [t["summary"].get(self.objective_metric, 0.0) for t in self.trials] + objs = [t["summary"][self.objective_metric] for t in self.trials] if not math.isfinite(self._finite_lo): return Text("") @@ -438,7 +443,7 @@ def _scatter_1d(self, x_param: str) -> Text: def _scatter_convergence(self) -> Text: """Render a convergence plot: trial # on x-axis, best-so-far on y-axis.""" W, H = 50, 12 - objs = [t["summary"].get(self.objective_metric, 0.0) for t in self.trials] + objs = [t["summary"][self.objective_metric] for t in self.trials] if not math.isfinite(self._finite_lo): return Text("") @@ -507,7 +512,7 @@ def _render_summary(self) -> Panel: if not self.trials: return Panel("No trials completed.", border_style="dim") - objs = [t["summary"].get(self.objective_metric, 0.0) for t in self.trials] + objs = [t["summary"][self.objective_metric] for t in self.trials] finite_objs = [o for o in objs if math.isfinite(o)] lines = [] diff --git a/src/cbfkit/cli/bench.py b/src/cbfkit/cli/bench.py index 63456b15..401ef01a 100644 --- a/src/cbfkit/cli/bench.py +++ b/src/cbfkit/cli/bench.py @@ -294,16 +294,19 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) - if args.command == "list": - return _cmd_list() - if args.command == "run": - return _cmd_run(args) - if args.command == "compare": - return _cmd_compare(args) - if args.command == "sweep": - return _cmd_sweep(args) - if args.command == "sweep-plot": - return _cmd_sweep_plot(args) + try: + if args.command == "list": + return _cmd_list() + if args.command == "run": + return _cmd_run(args) + if args.command == "compare": + return _cmd_compare(args) + if args.command == "sweep": + return _cmd_sweep(args) + if args.command == "sweep-plot": + return _cmd_sweep_plot(args) + except ValueError as exc: + parser.error(str(exc)) parser.error(f"Unsupported command: {args.command}") return 2 diff --git a/tests/test_benchmarks/test_metrics.py b/tests/test_benchmarks/test_metrics.py new file mode 100644 index 00000000..f0f5ebe8 --- /dev/null +++ b/tests/test_benchmarks/test_metrics.py @@ -0,0 +1,95 @@ +"""Incomplete benchmark evidence must not become a successful zero measurement.""" + +import json + +import pytest + +from cbfkit.benchmarks.metrics import mean, summarize +from cbfkit.benchmarks.runner import BenchmarkRun, _parse_seeds, compare_runs, write_artifacts +from cbfkit.cli.bench import main + + +@pytest.fixture +def record(): + return {"success": 1, "safety_violations": 0, "solver_failures": 0, "avg_step_ms": 2.0} + + +@pytest.mark.parametrize("key", ["success", "safety_violations", "solver_failures", "avg_step_ms"]) +def test_required_metrics_cannot_be_omitted(record, key): + del record[key] + with pytest.raises(ValueError, match=key): + summarize([record]) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), "unknown"]) +def test_invalid_measurement_is_rejected(record, value): + record["avg_step_ms"] = value + with pytest.raises(ValueError, match="metric value"): + summarize([record]) + + +def test_unavailable_measurement_is_not_zero(record, tmp_path): + record["safety_violations"] = None + summary = summarize([record]) + assert summary["safety_violation_rate"] is None + run = BenchmarkRun("test", [0], [record], summary) + write_artifacts(run, tmp_path) + data = json.loads((tmp_path / "results.json").read_text()) + assert data["summary"]["safety_violation_rate"] is None + with pytest.raises(ValueError, match="unavailable"): + compare_runs(run, run, "safety_violation_rate") + + +def test_optional_metrics_from_later_records_remain_unavailable(record): + summary = summarize([record, dict(record, final_distance=3.0)]) + assert "final_distance" in summary + assert summary["final_distance"] is None + + +def test_partial_measurement_is_not_averaged_over_a_subset(record): + summary = summarize([dict(record, avg_step_ms=None), record]) + assert summary["avg_step_ms"] is None + + +def test_complete_measurements_average_normally(record): + summary = summarize([record, dict(record, avg_step_ms=4.0, safety_violations=1)]) + assert summary["avg_step_ms"] == 3.0 + assert summary["safety_violation_rate"] == 0.5 + + +def test_unknown_comparison_metric_fails(record): + run = BenchmarkRun("test", [0], [record], summarize([record])) + with pytest.raises(ValueError, match="avg_steps_ms"): + compare_runs(run, run, "avg_steps_ms") + assert compare_runs(run, run, "avg_step_ms")["delta"] == 0.0 + + +def test_empty_summary_fails(): + with pytest.raises(ValueError, match="empty"): + summarize([]) + with pytest.raises(ValueError, match="empty"): + mean([], "success") + + +@pytest.mark.parametrize("seeds", [[], "", "3:1", "1,,2"]) +def test_invalid_seed_selection_fails(seeds): + with pytest.raises(ValueError): + _parse_seeds(seeds) + + +def test_cli_reports_invalid_metric_without_traceback(capsys): + with pytest.raises(SystemExit) as exc: + main( + [ + "compare", + "sanity_random_safety", + "--left-seeds", + "0", + "--right-seeds", + "1", + "--metric", + "typo", + ] + ) + assert exc.value.code == 2 + assert "typo" in capsys.readouterr().err diff --git a/tests/test_benchmarks/test_sweep_validation.py b/tests/test_benchmarks/test_sweep_validation.py new file mode 100644 index 00000000..492ed291 --- /dev/null +++ b/tests/test_benchmarks/test_sweep_validation.py @@ -0,0 +1,72 @@ +"""Sweep decisions require complete evidence and a nonempty experiment.""" + +import pytest +from rich.console import Console + +from cbfkit.benchmarks.sweep import run_optuna_sweep, run_sweep +from cbfkit.benchmarks.sweep_viz import SweepViz + + +def runner(seed, params): + return {"success": 1, "safety_violations": 0, "solver_failures": 0, "avg_step_ms": 1.0} + + +@pytest.mark.parametrize("seeds,combos", [([], [{}]), ([0], [])]) +def test_empty_grid_sweep_is_rejected(seeds, combos): + with pytest.raises(ValueError, match="at least one"): + run_sweep("test", seeds, combos, runner) + + +def test_missing_falsifier_metric_is_rejected(): + with pytest.raises(ValueError, match="typo"): + run_sweep("test", [0], [{}], runner, falsifier=True, falsifier_metric="typo") + + +@pytest.mark.parametrize("count", [0, 2]) +def test_batch_result_count_must_match_seeds(count): + with pytest.raises(ValueError, match="one result per requested seed"): + run_sweep( + "test", [0], [{}], runner, batch_runner=lambda seeds, params: [runner(0, {})] * count + ) + + +@pytest.mark.parametrize( + "kwargs", [{"objective_metric": "typo"}, {"safety_constraint": ("typo", 0)}] +) +def test_optuna_cannot_optimize_or_accept_missing_metrics(kwargs): + pytest.importorskip("optuna") + with pytest.raises(ValueError, match="typo"): + run_optuna_sweep("test", [0], {}, runner, n_trials=1, **kwargs) + + +@pytest.mark.parametrize("seeds,trials", [([], 1), ([0], 0)]) +def test_empty_optuna_sweep_is_rejected(seeds, trials): + with pytest.raises(ValueError, match="at least one"): + run_optuna_sweep("test", seeds, {}, runner, n_trials=trials) + + +def test_sweep_display_does_not_label_unknown_safety_as_safe(): + viz = SweepViz(objective_metric="avg_step_ms") + viz.add_result({}, {"avg_step_ms": 1.0, "safety_violation_rate": None}) + console = Console(width=100) + with console.capture() as capture: + console.print(viz.render()) + assert "N/A" in capture.get() + assert viz._safe_count == 0 + + +def test_sweep_display_uses_aggregated_violation_rate(): + viz = SweepViz(objective_metric="avg_step_ms") + viz.add_result({}, {"avg_step_ms": 1.0, "safety_violation_rate": 0.5}) + assert viz._safe_count == 0 + console = Console(width=100) + with console.capture() as capture: + console.print(viz.render()) + assert "0.5" in capture.get() + + +def test_sweep_display_rejects_unknown_objective(): + viz = SweepViz(objective_metric="typo") + with pytest.raises(ValueError, match="typo"): + viz.add_result({}, {"avg_step_ms": 1.0}) + assert not viz.trials From 5db84ae3994a4f43bc2f328423d111445d5cadfb Mon Sep 17 00:00:00 2001 From: Bardh Hoxha Date: Sun, 20 Sep 2026 13:17:20 -0400 Subject: [PATCH 3/3] test: replace obsolete scaffolding and expand lint coverage --- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 4 +- CONTRIBUTING.md | 16 +++- scripts/render_showcase.py | 1 - .../test_constraint_normalization.py | 1 - .../test_quadratic_program_solvers.py | 78 ++----------------- .../test_relaxable_constraints.py | 1 - .../test_robust_normalization.py | 1 - .../test_solver_constraint_cases.py | 39 ++++++++++ .../test_stiff_constraint_safety.py | 1 - tests/test_tutorial_migration_aliases.py | 42 ---------- 11 files changed, 61 insertions(+), 125 deletions(-) create mode 100644 tests/test_optimization/test_solver_constraint_cases.py delete mode 100644 tests/test_tutorial_migration_aliases.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8902ccc3..0c51bed5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: pip install uv==0.9.12 uv sync --python 3.10 --locked --extra quality --extra all --extra mujoco - run: | - uv run --no-sync ruff check src/cbfkit + uv run --no-sync ruff check src/cbfkit scripts tests/test_optimization tests/test_benchmarks uv run --no-sync black --check src/cbfkit uv run --no-sync isort --check-only src/cbfkit uv run --no-sync mypy src/cbfkit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1739f853..58e662db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,10 @@ repos: files: ^(src/cbfkit/|pyproject.toml) - id: ruff name: ruff - entry: uv run --no-sync ruff check src/cbfkit + entry: uv run --no-sync ruff check src/cbfkit scripts tests/test_optimization tests/test_benchmarks language: system pass_filenames: false - files: ^(src/cbfkit/|pyproject.toml) + files: ^(src/cbfkit/|scripts/|tests/test_optimization/|tests/test_benchmarks/|pyproject.toml) - id: mypy name: mypy entry: uv run --no-sync mypy src/cbfkit diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28c162e3..4ab547ae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,14 +48,18 @@ separate pytest processes and report that limitation. ## Linting, formatting and type checks ```bash -uv run --no-sync ruff check src/cbfkit +uv run --no-sync ruff check src/cbfkit scripts tests/test_optimization tests/test_benchmarks uv run --no-sync black --check src/cbfkit uv run --no-sync isort --check-only src/cbfkit uv run --no-sync mypy src/cbfkit uv run --no-sync pre-commit run --all-files ``` -All four source checks are blocking in CI and use the same scope in pre-commit. To apply +Ruff also checks maintained scripts and the optimization/benchmark tests. Formatting +and type checks remain scoped to the library; extend those checks to other test +directories incrementally as they are maintained. + +All four checks are blocking in CI and use the same scope in pre-commit. To apply formatting, run `uv run --no-sync isort src/cbfkit` followed by `uv run --no-sync black src/cbfkit`. Ruff checks unused imports (F401); declare intentional re-exports with `__all__` or explicit aliases, and use a local suppression only for import @@ -140,6 +144,14 @@ under ignored `results/`. Do not compare absolute times across different hardwar noisy latency thresholds on shared CI runners. Compare safety failures and constraint violations alongside speed; the baseline command fails if either count is nonzero. +For `cbfkit-bench` scenarios, every result must declare `success`, `safety_violations`, +`solver_failures`, and `avg_step_ms`. Use `None` for an unavailable measurement; +summaries preserve it as JSON `null`, including when only some seeds have a value. +Missing required keys, non-finite values, and empty runs are errors. Comparisons, +optimization objectives, and safety decisions require a measured, finite metric. +Optional numeric metrics are collected across all records and remain unavailable +when any seed lacks a measurement. + ## Submitting a change 1. Branch from `main` and keep the change focused. Separate refactors from behaviour changes. diff --git a/scripts/render_showcase.py b/scripts/render_showcase.py index 29cedb8b..8cfaeed2 100644 --- a/scripts/render_showcase.py +++ b/scripts/render_showcase.py @@ -965,7 +965,6 @@ def render_fixed_wing_3d() -> str: """Fixed-wing UAV 3D reach-avoid — load cached pickle or run a short sim.""" import pickle - import jax.numpy as jnp import matplotlib.pyplot as plt import numpy as np from matplotlib.animation import FuncAnimation, PillowWriter diff --git a/tests/test_optimization/test_constraint_normalization.py b/tests/test_optimization/test_constraint_normalization.py index 448cbfd9..cd755a9b 100644 --- a/tests/test_optimization/test_constraint_normalization.py +++ b/tests/test_optimization/test_constraint_normalization.py @@ -1,5 +1,4 @@ -import pytest import jax.numpy as jnp from jax import random from cbfkit.controllers.cbf_clf import vanilla_cbf_clf_qp_controller diff --git a/tests/test_optimization/test_quadratic_program_solvers.py b/tests/test_optimization/test_quadratic_program_solvers.py index 3f2531eb..3a3850c7 100644 --- a/tests/test_optimization/test_quadratic_program_solvers.py +++ b/tests/test_optimization/test_quadratic_program_solvers.py @@ -1,36 +1,8 @@ -""" -Test Module for cbfkit.optimization.quadratic_program solvers. - -========================= - -This module contains unit tests for functionalities in 'quadratic_program' -from 'cbfkit.optimization'. - -Tests ------ -The following test that the given solver correctly computes the solution, -or lack thereof, of the posed optimization problem: -- test_qp_cvxopt_feasible_inequality_only: feasible QP with inequality constraints -- test_qp_jaxopt_feasible_inequality_only: feasible QP with inequality constraints -- test_qp_casadi_feasible_inequality_only: feasible QP with inequality constraints -- test_qp_cvxopt_feasible_equality_only: feasible QP with equality constraints -- test_qp_jaxopt_feasible_iquality_only: feasible QP with equality constraints -- test_qp_casadi_feasible_iquality_only: feasible QP with equality constraints -- test_qp_cvxopt_feasible: feasible QP with inequality and equality constraints -- test_qp_jaxopt_feasible: feasible QP with inequality and equality constraints -- test_qp_casadi_feasible: feasible QP with inequality and equality constraints -- test_qp_cvxopt_infeasible: infeasible QP with inequality and equality constraints -- test_qp_jaxopt_infeasible: infeasible QP with inequality and equality constraints -- test_qp_casadi_infeasible: infeasible QP with inequality and equality constraints - -Setup ------ -- No set up required - -Examples --------- -To run all tests in this module (from the root of the repository): - $ python -m unittest tests.test_optimization.test_quadratic_program_solvers +"""Legacy JAXopt tuple API regression tests. + +Registry backend parity, equality feasibility, and infeasibility are covered in +``test_solver_convention_parity.py`` and ``test_solver_constraint_cases.py``. +These tests retain coverage for the native JAXopt solve/solve_with_state APIs. """ import os @@ -53,56 +25,21 @@ class TestQuadraticProgramSolvers(unittest.TestCase): eps = 1e-2 - # def test_qp_cvxopt_feasible_inequality_only(self): - # """Tests that the CVXOPT-based quadratic program solver computes - # the correct solution for a sequence of problems with inequality constraints only.""" - # self._test_qp_feasible_inequality_only(qp_cvxopt.solve) - def test_qp_jaxopt_feasible_inequality_only(self): """Tests that the JAXOPT-based quadratic program solver computes the correct solution for a sequence of problems with inequality constraints only.""" self._test_qp_feasible_inequality_only(qp_jaxopt.solve) - # def test_qp_casadi_feasible_inequality_only(self): - # """Tests that the Casadi-based quadratic program solver computes - # the correct solution for a sequence of problems with inequality constraints only.""" - # self._test_qp_feasible_inequality_only(qp_casadi.solve) - - # def test_qp_cvxopt_feasible_equality_only(self): - # """Tests that the CVXOPT-based quadratic program solver computes the correct solution - # for a sequence of problems with equality constraints only.""" - # self._test_qp_feasible_equality_only(qp_cvxopt.solve) - def test_qp_jaxopt_feasible_equality_only(self): """Tests that the JAXOPT-based quadratic program solver computes the correct solution for a sequence of problems with equality constraints only.""" self._test_qp_feasible_equality_only(qp_jaxopt.solve) - # def test_qp_casadi_feasible_equality_only(self): - # """Tests that the Casadi-based quadratic program solver computes the correct solution - # for a sequence of problems with equality constraints only.""" - # self._test_qp_feasible_equality_only(qp_casadi.solve) - - # def test_qp_cvxopt_feasible(self): - # """Tests that the CVXOPT-based quadratic program solver computes the correct solution - # for a sequence of problems with both inequality and equality constraints.""" - # self._test_qp_feasible(qp_cvxopt.solve) - def test_qp_jaxopt_feasible(self): """Tests that the JAXOPT-based quadratic program solver computes the correct solution for a sequence of problems with both inequality and equality constraints.""" self._test_qp_feasible(qp_jaxopt.solve) - # def test_qp_casadi_feasible(self): - # """Tests that the Casadi-based quadratic program solver computes the correct solution - # for a sequence of problems with both inequality and equality constraints.""" - # self._test_qp_feasible(qp_casadi.solve) - - # def test_qp_cvxopt_infeasible(self): - # """Tests that the CVXOPT-based quadratic program solver correctly identifies that - # the posed inequality/equality constrained problem is infeasible.""" - # self._test_qp_infeasible(qp_cvxopt.solve) - def test_qp_jaxopt_infeasible(self): """Tests that the JAXOPT-based quadratic program solver correctly identifies that the posed inequality/equality constrained problem is infeasible.""" @@ -113,11 +50,6 @@ def test_qp_jaxopt_feasible_with_state(self): and returns state when using solve_with_state.""" self._test_qp_feasible_with_state(qp_jaxopt.solve_with_state) - # def test_qp_casadi_infeasible(self): - # """Tests that the Casadi-based quadratic program solver correctly identifies that - # the posed inequality/equality constrained problem is infeasible.""" - # self._test_qp_infeasible(qp_casadi.solve) - def _test_qp_feasible_inequality_only(self, solver: QpSolverCallable): """Tests that the quadratic program solver computes the correct solution for a sequence of problems with inequality constraints only.""" diff --git a/tests/test_optimization/test_relaxable_constraints.py b/tests/test_optimization/test_relaxable_constraints.py index fd747751..3cf9e384 100644 --- a/tests/test_optimization/test_relaxable_constraints.py +++ b/tests/test_optimization/test_relaxable_constraints.py @@ -1,5 +1,4 @@ -import pytest import jax.numpy as jnp from jax import random diff --git a/tests/test_optimization/test_robust_normalization.py b/tests/test_optimization/test_robust_normalization.py index 6eeec916..b92fb048 100644 --- a/tests/test_optimization/test_robust_normalization.py +++ b/tests/test_optimization/test_robust_normalization.py @@ -1,5 +1,4 @@ -import pytest import jax.numpy as jnp from jax import random from cbfkit.controllers.cbf_clf import vanilla_cbf_clf_qp_controller diff --git a/tests/test_optimization/test_solver_constraint_cases.py b/tests/test_optimization/test_solver_constraint_cases.py new file mode 100644 index 00000000..db797742 --- /dev/null +++ b/tests/test_optimization/test_solver_constraint_cases.py @@ -0,0 +1,39 @@ +"""Active coverage for equality and infeasibility cases formerly commented out.""" + +import jax.numpy as jnp +import pytest + +from cbfkit.optimization.quadratic_program import get_solver + + +@pytest.mark.parametrize("name", ["jaxopt", "cvxopt", "casadi"]) +@pytest.mark.parametrize("with_inequality", [False, True]) +def test_equality_feasible_solution(name, with_inequality): + # min ||x - [1, 2]||^2, subject to x0 + x1 = 1, has minimizer [0, 1]. + kwargs = {"a_mat": jnp.ones((1, 2)), "b_vec": jnp.ones(1)} + if with_inequality: + kwargs.update(g_mat=jnp.eye(2), h_vec=2.0 * jnp.ones(2)) + try: + result = get_solver(name)(jnp.eye(2), jnp.array([-2.0, -4.0]), **kwargs) + except ImportError: + pytest.skip(f"Optional solver {name} is not installed") + assert int(result.status) == 1 + assert jnp.allclose(result.primal, jnp.array([0.0, 1.0]), atol=1e-3) + + +@pytest.mark.parametrize("name", ["fast", "jaxopt", "cvxopt", "casadi"]) +def test_contradictory_inequalities_do_not_report_success(name): + # x <= 0 and x >= 1 cannot both hold. + try: + result = get_solver(name)( + jnp.eye(1), jnp.zeros(1), jnp.array([[1.0], [-1.0]]), jnp.array([0.0, -1.0]) + ) + except ImportError: + pytest.skip(f"Optional solver {name} is not installed") + except ValueError as exc: + # CVXOPT can fail its interior-point factorization on this infeasible + # problem instead of returning a status. Both paths explicitly fail. + assert name == "cvxopt" + assert "domain error" in str(exc) + return + assert int(result.status) != 1 diff --git a/tests/test_optimization/test_stiff_constraint_safety.py b/tests/test_optimization/test_stiff_constraint_safety.py index 257434f7..54b199e4 100644 --- a/tests/test_optimization/test_stiff_constraint_safety.py +++ b/tests/test_optimization/test_stiff_constraint_safety.py @@ -1,5 +1,4 @@ -import pytest import jax.numpy as jnp from jax import random from cbfkit.controllers.cbf_clf import vanilla_cbf_clf_qp_controller diff --git a/tests/test_tutorial_migration_aliases.py b/tests/test_tutorial_migration_aliases.py deleted file mode 100644 index 5dbf8319..00000000 --- a/tests/test_tutorial_migration_aliases.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import subprocess -import sys - -import pytest - - -ALIASES = [ - ("tutorials/mppi_cbf_reach_avoid.py", "tutorials/simulate_mppi_cbf.py"), - ("tutorials/mppi_stl_reach_avoid.py", "tutorials/simulate_mppi_stl.py"), - ( - "tutorials/single_integrator_dynamic_obstacles.py", - "tutorials/single_integrator_reach_avoid_dyn_obs.py", - ), -] - - -@pytest.mark.parametrize("alias_path,target_path", ALIASES) -def test_tutorial_alias_prints_mapping(alias_path: str, target_path: str) -> None: - # Newer bolt revisions may keep these scripts as first-class entrypoints - # (no deprecation shim). In that case this compatibility check is not applicable. - if not os.path.exists(target_path): - pytest.skip(f"Target entrypoint {target_path} is not present on this branch.") - - with open(alias_path, "r", encoding="utf-8") as f: - alias_source = f.read().lower() - if "deprecated" not in alias_source: - pytest.skip(f"{alias_path} is not a deprecation shim on this branch.") - - env = os.environ.copy() - env["CBFKIT_TUTORIAL_ALIAS_ONLY"] = "1" - env["PYTHONPATH"] = f"{os.getcwd()}{os.pathsep}{os.path.join(os.getcwd(), 'src')}" - - proc = subprocess.run( - [sys.executable, alias_path], - env=env, - capture_output=True, - text=True, - ) - assert proc.returncode == 0, proc.stderr - assert "deprecated" in proc.stdout.lower() - assert target_path in proc.stdout