From 32bff017bd28ceb3e97e68c1dd0d0566e809dc84 Mon Sep 17 00:00:00 2001 From: juraph Date: Thu, 8 Oct 2026 11:36:01 +1100 Subject: [PATCH 1/4] Add proper pytesting First pass for unit testing of our current architecture. These tests are sort of naive, there should be more rigor put into them, however as a first draft they server fairly well. Adding more tests will be fairly trivial. Of note, two currently reported failures: XFAIL tests/jax_tests/test_plnet_lbdn.py::test_lbdn_init_output_zero - init_output_zero documents a zero output at init, but the output is non-zero (see PR notes) XFAIL tests/torch_tests/test_lipnets.py::test_unitary_no_bias_explicit - Unitary.direct_to_explicit assumes a bias exists --- .github/workflows/tests.yml | 37 ++++ README.md | 21 +- pyproject.toml | 17 ++ run_tests.sh | 10 +- test/plnet/bilipnet_test.py | 89 --------- test/plnet/monlipnet_test.py | 67 ------- test/plnet/orthogonal_test.py | 39 ---- test/plnet/plnet_test.py | 97 --------- test/plnet_torch/bilipnet_test.py | 55 ------ test/plnet_torch/monlipnet_test.py | 52 ----- test/plnet_torch/orthogonal_test.py | 39 ---- test/plnet_torch/plnet_test.py | 55 ------ test/test_bilipren.py | 154 --------------- test/test_bilipren_torch.py | 116 ----------- test/test_lbdn.py | 48 ----- test/test_linren.py | 53 ----- test/test_r2dn.py | 87 -------- test/test_ren.py | 57 ------ tests/conftest.py | 12 ++ test/utils.py => tests/helpers.py | 0 tests/jax_tests/conftest.py | 19 ++ tests/jax_tests/test_composition.py | 155 +++++++++++++++ tests/jax_tests/test_lipnets.py | 130 ++++++++++++ tests/jax_tests/test_orthogonal.py | 67 +++++++ tests/jax_tests/test_plnet_lbdn.py | 105 ++++++++++ tests/jax_tests/test_ren.py | 294 ++++++++++++++++++++++++++++ tests/test_parity.py | 61 ++++++ tests/torch_tests/conftest.py | 18 ++ tests/torch_tests/test_lipnets.py | 171 ++++++++++++++++ tests/torch_tests/test_ren.py | 176 +++++++++++++++++ 30 files changed, 1283 insertions(+), 1018 deletions(-) create mode 100644 .github/workflows/tests.yml delete mode 100644 test/plnet/bilipnet_test.py delete mode 100644 test/plnet/monlipnet_test.py delete mode 100644 test/plnet/orthogonal_test.py delete mode 100644 test/plnet/plnet_test.py delete mode 100644 test/plnet_torch/bilipnet_test.py delete mode 100644 test/plnet_torch/monlipnet_test.py delete mode 100644 test/plnet_torch/orthogonal_test.py delete mode 100644 test/plnet_torch/plnet_test.py delete mode 100644 test/test_bilipren.py delete mode 100644 test/test_bilipren_torch.py delete mode 100644 test/test_lbdn.py delete mode 100644 test/test_linren.py delete mode 100644 test/test_r2dn.py delete mode 100644 test/test_ren.py create mode 100644 tests/conftest.py rename test/utils.py => tests/helpers.py (100%) create mode 100644 tests/jax_tests/conftest.py create mode 100644 tests/jax_tests/test_composition.py create mode 100644 tests/jax_tests/test_lipnets.py create mode 100644 tests/jax_tests/test_orthogonal.py create mode 100644 tests/jax_tests/test_plnet_lbdn.py create mode 100644 tests/jax_tests/test_ren.py create mode 100644 tests/test_parity.py create mode 100644 tests/torch_tests/conftest.py create mode 100644 tests/torch_tests/test_lipnets.py create mode 100644 tests/torch_tests/test_ren.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..9e238f1 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,37 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + # uv.lock is not tracked, so key the cache on pyproject.toml + cache-dependency-glob: pyproject.toml + + - name: Install dependencies (CPU) + run: uv sync --extra cpu + + - name: Run tests + run: uv run pytest -n auto --junitxml=report.xml + + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: pytest-report + path: report.xml diff --git a/README.md b/README.md index 65e579a..d7039d5 100644 --- a/README.md +++ b/README.md @@ -34,18 +34,31 @@ This will create a Python virtual environment at `./.venv`, install the pinned P ./run_tests.sh ``` +### Testing + +Tests are written with [pytest](https://pytest.org) and live in `tests/`: `tests/jax_tests/` for the JAX/Flax implementations, `tests/torch_tests/` for the PyTorch implementations, and `tests/test_parity.py` for JAX-vs-PyTorch checks. `./run_tests.sh` is a thin wrapper around `uv run pytest`, so any pytest arguments work: + +```bash +./run_tests.sh # everything +./run_tests.sh -m torch # PyTorch tests only (or -m jax) +./run_tests.sh -n auto # run in parallel +./run_tests.sh -k bilip # select by name +``` + +The same suite runs on every push to `main` and every pull request through GitHub Actions (`.github/workflows/tests.yml`, CPU only). + ### A Note on Dependencies Dependencies are declared in `pyproject.toml` and locked to exact versions in `uv.lock`, which is committed to the repository so that everyone builds the same environment. They are split into two sets: - The **core dependencies** are those required by the `robustnn` package itself (`jax`, `flax`, `numpy`, and `torch`). These are the only packages a user installing `robustnn` will get. -- The **`examples` extra** adds everything needed to run the demos in `examples/` and the scripts in `test/` (`matplotlib`, `optax`, `pandas`, `scipy`, `tensorflow`, etc.). The `install.sh` script installs this extra by default for development. +- The **`examples` extra** adds everything needed to run the demos in `examples/` (`matplotlib`, `optax`, `pandas`, `scipy`, `tensorflow`, etc.). The `install.sh` script installs this extra by default for development. -There is also a `cuda12` extra, which `install.sh` selects automatically when CUDA is detected. If you would rather manage the environment yourself, the equivalent commands are: +The `cpu` and `cuda13` extras select the CPU or CUDA 13 builds of `jax` and `torch`; `install.sh` picks one automatically when CUDA is detected. The test tools (`pytest`, `pytest-xdist`, `optax`) are in the `dev` dependency group, which `uv sync` installs by default. If you would rather manage the environment yourself, the equivalent commands are: ```bash -uv sync --extra examples # CPU -uv sync --extra examples --extra cuda12 # CUDA 12 +uv sync --extra examples --extra cpu # CPU +uv sync --extra examples --extra cuda13 # CUDA 13 ``` All code was tested and developed in Ubuntu 22.04 with CUDA 12.4 and Python 3.12. diff --git a/pyproject.toml b/pyproject.toml index 5da9aa0..147a625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,13 @@ examples = [ "cycler>=0.12.1", ] +[dependency-groups] +dev = [ + "pytest>=8", + "pytest-xdist>=3", + "optax>=0.2.8", +] + [build-system] requires = ["uv_build>=0.12.5,<0.13"] build-backend = "uv_build" @@ -83,3 +90,13 @@ torch = [ { index = "pytorch-cpu", extra = "cpu" }, { index = "pytorch-cu130", extra = "cuda13"}, ] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] +addopts = "-ra --strict-markers --import-mode=importlib" +markers = [ + "jax: tests for the JAX/Flax implementations", + "torch: tests for the PyTorch implementations", + "slow: slow tests (e.g. optimisation-based Lipschitz estimates)", +] diff --git a/run_tests.sh b/run_tests.sh index 1d10132..2e8e27c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,9 +1,7 @@ #!/bin/bash +# Run the test suite. Extra arguments are passed to pytest, e.g. +# ./run_tests.sh -k torch +# ./run_tests.sh -m jax -n auto set -euo pipefail -uv run python test/test_lbdn.py -uv run python test/test_linren.py -uv run python test/test_ren.py -uv run python test/test_r2dn.py -uv run python test/test_bilipren.py -uv run python test/test_bilipren_torch.py +uv run pytest "$@" diff --git a/test/plnet/bilipnet_test.py b/test/plnet/bilipnet_test.py deleted file mode 100644 index 63bd671..0000000 --- a/test/plnet/bilipnet_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.bilipnet_jax import BiLipNet -from flax import linen as nn -import jax -import jax.numpy as jnp - -# use some random key -rng = jax.random.key(0) -rng, key = jax.random.split(rng, 2) - -# Generate random input -batches = 1 -input_size = 2 -inputs = jax.numpy.ones((batches, input_size)) -random_input = jax.numpy.array([[1,2],[3,4]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -bilipnet_layer = BiLipNet(input_size=input_size, - units=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=True, - is_nu_fixed=True, - is_tau_fixed=False, - depth=depth, - act_fn=nn.relu, - ) - -# Initialize parameters -params = bilipnet_layer.init(key, inputs) -explicit_params = bilipnet_layer.direct_to_explicit(params) - -# output -# parameters inside the model -# Parameters: {'params': {'mon_0': {'by': Array([0., 0.], dtype=float32), 'Fq': Array([[-0.35264897, 0.4984822 , -0.47076553, -0.5240039 ], -# [ 0.8253973 , 0.18583922, -0.5114892 , -1.1876136 ]], dtype=float32), 'fq': Array([1.8046912], dtype=float32), 'Fab0': Array([[-0.5248157 , -0.40291667], -# [-0.78404427, 1.1737523 ]], dtype=float32), 'fab0': Array([1.5589076], dtype=float32), 'b0': Array([0., 0.], dtype=float32), 'Fab1': Array([[ 0.87369543, -0.2151519 ], -# [ 0.49886596, -1.2565557 ], -# [ 0.6950717 , 0.331631 ], -# [ 0.1621341 , 0.02383308]], dtype=float32), 'fab1': Array([1.8048248], dtype=float32), 'b1': Array([0., 0.], dtype=float32)}, 'mon_1': {'by': Array([0., 0.], dtype=float32), 'Fq': Array([[ 0.6417317 , 0.27034405, -1.1292694 , -0.35397807], -# [ 0.30808014, 0.29465917, -0.1229675 , -0.5643722 ]], dtype=float32), 'fq': Array([1.549461], dtype=float32), 'Fab0': Array([[-0.10914532, 0.28966933], -# [-0.4470564 , -0.16007964]], dtype=float32), 'fab0': Array([0.56683856], dtype=float32), 'b0': Array([0., 0.], dtype=float32), 'Fab1': Array([[-0.5976882 , -0.52727705], -# [-0.15430154, 0.30250213], -# [ 0.5753947 , 0.7232377 ], -# [ 0.08675962, 0.22860019]], dtype=float32), 'fab1': Array([1.2901573], dtype=float32), 'b1': Array([0., 0.], dtype=float32)}, 'uni_0': {'W': Array([[-0.45691052, 0.27073306], -# [ 0.30126202, 1.4242083 ]], dtype=float32), 'a': Array([1.5495778], dtype=float32), 'b': Array([0., 0.], dtype=float32)}, 'uni_1': {'W': Array([[ 1.3580841 , -0.20097394], -# [ 0.79159266, -0.11509188]], dtype=float32), 'a': Array([1.588914], dtype=float32), 'b': Array([0., 0.], dtype=float32)}, 'uni_2': {'W': Array([[0.32905164, 1.3335835 ], -# [0.71546155, 1.1022835 ]], dtype=float32), 'a': Array([1.900956], dtype=float32), 'b': Array([0., 0.], dtype=float32)}}} -print("Parameters:", params) - -# Bound: (1.0, 2.0000000000000004, 2.0000000000000004) -print("Bound: ", bilipnet_layer.get_bounds(params)) - -# Explicit Parameter: ExplicitBiLipParams(monlip_layers=[ExplicitMonLipParams(mu=1.0, nu=1.4142135623730951, units=(2, 2), V=[Array([[-0.8374117 , 0.35082638], -# [-1.3102895 , 0.8824698 ]], dtype=float32)], S=Array([[ 0.22833245, -0.6014742 ], -# [-0.6312865 , 0.38985923], -# [-0.17338762, -0.92611027], -# [ 0.8809698 , -0.2251715 ]], dtype=float32), by=Array([0., 0.], dtype=float32), bh=[], sqrt_g2=Array(0.45508987, dtype=float32, weak_type=True), sqrt_2g=Array(0.91017973, dtype=float32, weak_type=True), STks=[Array([[ 0.22833245, -0.6312865 ], -# [-0.6014742 , 0.38985923]], dtype=float32), Array([[-0.17338762, 0.8809698 ], -# [-0.92611027, -0.2251715 ]], dtype=float32)], Ak_1s=[Array([], shape=(0, 0), dtype=float32), Array([[ 0.74633104, 0.66557497], -# [-0.66557497, 0.74633104]], dtype=float32)], BTks=[Array([], shape=(0, 2), dtype=float32), Array([[-0.42924377, -0.7826297 ], -# [-0.14776382, -0.10674066]], dtype=float32)], bs=[Array([0., 0.], dtype=float32), Array([0., 0.], dtype=float32)]), ExplicitMonLipParams(mu=1.0, nu=1.4142135623730951, units=(2, 2), V=[Array([[-0.09556141, 0.6244243 ], -# [-1.094813 , 1.5568984 ]], dtype=float32)], S=Array([[-0.22651373, 0.45749414], -# [ 0.07242526, 0.39718354], -# [ 0.317936 , -0.7688174 ], -# [-0.17970872, 0.18454075]], dtype=float32), by=Array([0., 0.], dtype=float32), bh=[], sqrt_g2=Array(0.45508987, dtype=float32, weak_type=True), sqrt_2g=Array(0.91017973, dtype=float32, weak_type=True), STks=[Array([[-0.22651373, 0.07242526], -# [ 0.45749414, 0.39718354]], dtype=float32), Array([[ 0.317936 , -0.17970872], -# [-0.7688174 , 0.18454075]], dtype=float32)], Ak_1s=[Array([], shape=(0, 0), dtype=float32), Array([[ 0.29637387, 0.955072 ], -# [-0.955072 , 0.29637387]], dtype=float32)], BTks=[Array([], shape=(0, 2), dtype=float32), Array([[-0.312346 , -0.905712 ], -# [ 0.04689751, -0.29210064]], dtype=float32)], bs=[Array([0., 0.], dtype=float32), Array([0., 0.], dtype=float32)])], unitary_layers=[ExplicitOrthogonalParams(R=Array([[ 0.9981377 , 0.06100107], -# [-0.06100107, 0.9981378 ]], dtype=float32), b=Array([0., 0.], dtype=float32)), ExplicitOrthogonalParams(R=Array([[ 0.00746098, 0.99997216], -# [-0.9999722 , 0.00746104]], dtype=float32), b=Array([0., 0.], dtype=float32)), ExplicitOrthogonalParams(R=Array([[ 0.44709978, -0.8944841 ], -# [ 0.8944841 , 0.44709978]], dtype=float32), b=Array([0., 0.], dtype=float32))], lipmin=1.0, lipmax=2.0000000000000004, distortion=2.0000000000000004) -print("Explicit Parameter:", explicit_params) - -# call results: [[2.5733566 1.0719225] -# [6.596445 1.4032273]] -print("call results:", bilipnet_layer.apply(params, random_input)) - -# explict call results: [[2.5733566 1.0719225] -# [6.596445 1.4032273]] -print("explict call results:", bilipnet_layer.explicit_call(params=params, x=random_input, explicit=explicit_params)) diff --git a/test/plnet/monlipnet_test.py b/test/plnet/monlipnet_test.py deleted file mode 100644 index 27d75e8..0000000 --- a/test/plnet/monlipnet_test.py +++ /dev/null @@ -1,67 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.monlipnet_jax import MonLipNet -from flax import linen as nn -import jax -import jax.numpy as jnp - -# use some random key -rng = jax.random.key(0) -rng, key = jax.random.split(rng, 2) - -# Generate random input -batches = 1 -input_size = 2 -inputs = jax.numpy.ones((batches, input_size)) -random_input = jax.numpy.array([[1,2],[3,4]]) -units = [2,2] -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -monlipnet_layer = MonLipNet(input_size=input_size, - units=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=False, - is_nu_fixed=False, - is_tau_fixed=True, - ) - -# Initialize parameters -params = monlipnet_layer.init(key, inputs) -explicit_params = monlipnet_layer.direct_to_explicit(params) - -# output -# parameters inside the model -# Parameters: {'params': {'logmu': Array([0.], dtype=float32), 'by': Array([0., 0.], dtype=float32), 'Fq': Array([[-0.67808104, 0.30136618, 0.15980107, 0.42550763], -# [-0.44976526, 0.41513145, 0.10287036, -0.29770726]], dtype=float32), 'fq': Array([1.1095239], dtype=float32), 'Fab0': Array([[-0.80490816, -0.13022006], -# [ 0.8404852 , 0.00755759]], dtype=float32), 'fab0': Array([1.1710281], dtype=float32), 'b0': Array([0., 0.], dtype=float32), 'Fab1': Array([[ 0.8647551 , -0.14875135], -# [ 0.43040338, 0.32212225], -# [-0.6120695 , -0.5580562 ], -# [-0.09665938, -0.82898444]], dtype=float32), 'fab1': Array([1.5625466], dtype=float32), 'b1': Array([0., 0.], dtype=float32)}} -print("Parameters:", params) - -# Bound: (Array([1.], dtype=float32), Array([2.], dtype=float32), 2) -print("Bound: ", monlipnet_layer.get_bounds(params)) - -# Explicit Parameter: ExplicitMonLipParams(mu=Array([1.], dtype=float32), nu=Array([2.], dtype=float32), units=(2, 2), V=[Array([[ 0.8974826 , 0.88322675], -# [-1.5544597 , 1.2352736 ]], dtype=float32)], S=Array([[ 0.6871649 , -0.2573519 ], -# [ 0.15004435, 0.9252147 ], -# [ 0.06253868, -0.3901289 ], -# [ 0.4661716 , -0.8010758 ]], dtype=float32), by=Array([0., 0.], dtype=float32), bh=[], sqrt_g2=Array([0.70710677], dtype=float32), sqrt_2g=Array([1.4142135], dtype=float32), STks=[Array([[ 0.6871649 , 0.15004435], -# [-0.2573519 , 0.9252147 ]], dtype=float32), Array([[ 0.06253868, 0.4661716 ], -# [-0.3901289 , -0.8010758 ]], dtype=float32)], Ak_1s=[Array([], shape=(0, 0), dtype=float32), Array([[ 0.02972363, -0.99955815], -# [ 0.99955815, 0.02972363]], dtype=float32)], BTks=[Array([], shape=(0, 2), dtype=float32), Array([[ 0.45475647, 0.5942618 ], -# [-0.43541664, 0.7952448 ]], dtype=float32)], bs=[Array([0., 0.], dtype=float32), Array([0., 0.], dtype=float32)]) -print("Explicit Parameter:", explicit_params) - -# call results: [[1.9913986 2.4819925] -# [5.3340416 4.9430366]] -print("call results:", monlipnet_layer.apply(params, random_input)) - -# explict call results: [[1.9913986 2.4819925] -# [5.3340416 4.9430366]] -print("explict call results:", monlipnet_layer.explicit_call(params=params, x=random_input, explicit=explicit_params)) diff --git a/test/plnet/orthogonal_test.py b/test/plnet/orthogonal_test.py deleted file mode 100644 index ae44f5e..0000000 --- a/test/plnet/orthogonal_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.orthogonal_jax import Unitary -from flax import linen as nn -import jax -import jax.numpy as jnp - -# use some random key -rng = jax.random.key(0) -rng, key = jax.random.split(rng, 2) - -# Generate random input -batches = 1 -input_size = 2 -inputs = jax.numpy.ones((batches, input_size)) -random_input = jax.numpy.array([[1,2]]) - -# Initialize a unitary layer -unitary_layer = Unitary(input_size=input_size, use_bias=True) - -# Initialize parameters -params = unitary_layer.init(key, inputs) -explicit_params = unitary_layer.direct_to_explicit(params) - -# output -# parameters inside the model -# Parameters: {'params': {'W': Array([[-1.4166745 , -0.39387214], -# [ 1.3889172 , 0.8457249 ]], dtype=float32), 'a': Array([2.1923602], dtype=float32), 'b': Array([0., 0.], dtype=float32)}} -print("Parameters:", params) - -# Explicit Parameter: ExplicitOrthogonalParams(R=Array([[-0.5213407, 0.8533486], -# [-0.8533486, -0.5213408]], dtype=float32), b=Array([0., 0.], dtype=float32)) -print("Explicit Parameter:", explicit_params) - -# call results: [[ 1.1853565 -1.8960302]] -print("call results:", unitary_layer.apply(params, random_input)) - -# explict call results: [[ 1.1853565 -1.8960302]] -print("explict call results:", unitary_layer.explicit_call(params=params, x=random_input, explicit=explicit_params)) diff --git a/test/plnet/plnet_test.py b/test/plnet/plnet_test.py deleted file mode 100644 index c57e994..0000000 --- a/test/plnet/plnet_test.py +++ /dev/null @@ -1,97 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.bilipnet_jax import BiLipNet -from robustnn.plnet_jax import PLNet -from flax import linen as nn -import jax -import jax.numpy as jnp - -# use some random key -rng = jax.random.key(0) -rng, key = jax.random.split(rng, 2) - -# Generate random input -batches = 1 -input_size = 2 -inputs = jax.numpy.ones((batches, input_size)) -random_input = jax.numpy.array([[1,2],[3,4]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -bilipnet_layer = BiLipNet(input_size=input_size, - units=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=True, - is_nu_fixed=True, - is_tau_fixed=False, - depth=depth, - act_fn=nn.relu, - ) -plnet_layer = PLNet(BiLipBlock=bilipnet_layer) - -# Initialize parameters -params = plnet_layer.init(key, inputs) -explicit_params = plnet_layer.direct_to_explicit(params) - -# output -# Parameters: {'params': {'BiLipBlock': {'mon_0': {'by': Array([0., 0.], dtype=float32), 'Fq': Array([[ 0.4767888 , 0.35946247, 0.88449204, 0.33577147], -# [ 0.07560676, 0.04496887, -0.48031786, 0.04774935]], dtype=float32), 'fq': Array([1.2216109], dtype=float32), 'Fab0': Array([[-0.72828573, -0.25070167], -# [ 0.13120073, -1.6032633 ]], dtype=float32), 'fab0': Array([1.7835128], dtype=float32), 'b0': Array([0., 0.], dtype=float32), 'Fab1': Array([[ 0.9240948 , 0.3495114 ], -# [ 0.05895175, 1.1558645 ], -# [-0.73868793, -0.30372804], -# [ 0.37575445, 0.35212633]], dtype=float32), 'fab1': Array([1.7940742], dtype=float32), 'b1': Array([0., 0.], dtype=float32)}, 'mon_1': {'by': Array([0., 0.], dtype=float32), 'Fq': Array([[ 0.55890423, 0.71112454, 0.11298353, -0.07829883], -# [-0.42325065, -0.45026565, 0.44604003, -0.03375688]], dtype=float32), 'fq': Array([1.1911924], dtype=float32), 'Fab0': Array([[ 0.24955161, 0.1769813 ], -# [ 0.41612598, -0.6867631 ]], dtype=float32), 'fab0': Array([0.85930365], dtype=float32), 'b0': Array([0., 0.], dtype=float32), 'Fab1': Array([[-0.05376392, -0.37428084], -# [ 0.29803494, -0.3588037 ], -# [ 0.9440928 , 0.37645176], -# [ 0.3438681 , 0.56135315]], dtype=float32), 'fab1': Array([1.3516402], dtype=float32), 'b1': Array([0., 0.], dtype=float32)}, 'uni_0': {'W': Array([[ 0.8779757 , 0.63369995], -# [-0.06714347, -0.88191175]], dtype=float32), 'a': Array([1.3981035], dtype=float32), 'b': Array([0., 0.], dtype=float32)}, 'uni_1': {'W': Array([[-0.12902264, 0.7436014 ], -# [ 0.17159157, -0.7215747 ]], dtype=float32), 'a': Array([1.0581604], dtype=float32), 'b': Array([0., 0.], dtype=float32)}, 'uni_2': {'W': Array([[ 0.94838375, -0.7258084 ], -# [ 0.07415643, -0.43732023]], dtype=float32), 'a': Array([1.2739614], dtype=float32), 'b': Array([0., 0.], dtype=float32)}}}} -print("Parameters:", params) - -# Explicit Parameter: ExplicitPLParams(bilip_layer=ExplicitBiLipParams(monlip_layers=[ExplicitMonLipParams(mu=1.0, nu=1.4142135623730951, units=(2, 2), V=[Array([[ 1.879422 , 0.522046 ], -# [ 0.5164222, -0.4752227]], dtype=float32)], S=Array([[-0.00249775, 0.00269252], -# [ 0.14675875, 0.9206321 ], -# [-0.2045758 , -0.16814956], -# [ 0.37216145, -0.14749956]], dtype=float32), by=Array([0., 0.], dtype=float32), bh=[], sqrt_g2=Array(0.45508987, dtype=float32, weak_type=True), sqrt_2g=Array(0.91017973, dtype=float32, weak_type=True), STks=[Array([[-0.00249775, 0.14675875], -# [ 0.00269252, 0.9206321 ]], dtype=float32), Array([[-0.2045758 , 0.37216145], -# [-0.16814956, -0.14749956]], dtype=float32)], Ak_1s=[Array([], shape=(0, 0), dtype=float32), Array([[ 0.74543 , -0.6665839], -# [ 0.6665839, 0.74543 ]], dtype=float32)], BTks=[Array([], shape=(0, 2), dtype=float32), Array([[ 0.8744825 , 0.03409042], -# [-0.43182182, -0.349242 ]], dtype=float32)], bs=[Array([0., 0.], dtype=float32), Array([0., 0.], dtype=float32)]), ExplicitMonLipParams(mu=1.0, nu=1.4142135623730951, units=(2, 2), V=[Array([[-1.1694525 , -0.32475257], -# [-0.32392204, -1.8697786 ]], dtype=float32)], S=Array([[ 0.39068747, 0.8540329 ], -# [-0.8641052 , 0.22347406], -# [ 0.03718062, 0.4850111 ], -# [-0.706138 , 0.33923262]], dtype=float32), by=Array([0., 0.], dtype=float32), bh=[], sqrt_g2=Array(0.45508987, dtype=float32, weak_type=True), sqrt_2g=Array(0.91017973, dtype=float32, weak_type=True), STks=[Array([[ 0.39068747, -0.8641052 ], -# [ 0.8540329 , 0.22347406]], dtype=float32), Array([[ 0.03718062, -0.706138 ], -# [ 0.4850111 , 0.33923262]], dtype=float32)], Ak_1s=[Array([], shape=(0, 0), dtype=float32), Array([[ 0.8918072 , -0.45241565], -# [ 0.45241565, 0.8918072 ]], dtype=float32)], BTks=[Array([], shape=(0, 2), dtype=float32), Array([[-0.5949247 , -0.5673965 ], -# [ 0.11973099, -0.7604673 ]], dtype=float32)], bs=[Array([0., 0.], dtype=float32), Array([0., 0.], dtype=float32)])], unitary_layers=[ExplicitOrthogonalParams(R=Array([[ 0.34121832, -0.9399841 ], -# [ 0.9399841 , 0.34121832]], dtype=float32), b=Array([0., 0.], dtype=float32)), ExplicitOrthogonalParams(R=Array([[ 0.5069373 , -0.86198294], -# [ 0.86198294, 0.5069373 ]], dtype=float32), b=Array([0., 0.], dtype=float32)), ExplicitOrthogonalParams(R=Array([[ 0.21955402, 0.97560036], -# [-0.9756004 , 0.21955411]], dtype=float32), b=Array([0., 0.], dtype=float32))], lipmin=1.0, lipmax=2.0000000000000004, distortion=2.0000000000000004), f_function=.f_function at 0x718d6b5ff6a0>, c=0.0, optimal_point=None, lipmin=1.0, lipmax=2.0000000000000004, distortion=2.0000000000000004) -print("Explicit Parameter:", explicit_params) - -# call results: [ 2.73808 15.734728] -print("call results:", plnet_layer.apply(params, random_input)) - -# explict call results: [ 2.73808 15.734728] -print("explict call results:", plnet_layer.explicit_call(params=params, x=random_input, explicit=explicit_params)) - -# test for know equilibrium case -random_input_min = jax.numpy.array([[3,2],[4,1]]) - -# supposed to map to all dim -random_input_min_fixed = jax.numpy.array([[1,1]]) - -# explict call with gt: [2.72025 3.4215217] -print("explict call with gt: ", plnet_layer.apply(params, random_input, random_input_min)) - -# explict call with mapping: [0.3416915 7.868343 ] -print("explict call with mapping: ", plnet_layer.apply(params, random_input, random_input_min_fixed)) diff --git a/test/plnet_torch/bilipnet_test.py b/test/plnet_torch/bilipnet_test.py deleted file mode 100644 index 6ee64b3..0000000 --- a/test/plnet_torch/bilipnet_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.bilipnet_torch import BiLipNet -import torch -import numpy as np - -# Set seeds for all RNGs -seed = 42 -torch.manual_seed(seed) -torch.cuda.manual_seed(seed) # for current GPU -torch.cuda.manual_seed_all(seed) # for all GPUs -np.random.seed(seed) - -# Generate random input -batches = 1 -input_size = 2 -inputs = torch.zeros((batches, input_size)) -random_input = torch.tensor([[1.0,2.0],[3.0,4.0]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a bi-lip layer -bilipnet_layer = BiLipNet(features=input_size, - unit_features=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=True, - is_nu_fixed=True, - is_tau_fixed=False, - depth=depth, - ) - -# Bound: (1.0, 1.9999999315429164, 1.9999999315429164) -print("Bound: ", bilipnet_layer.get_bounds()) - -# call results: tensor([[1.3719, 2.7746], -# [2.7158, 5.8810]], grad_fn=) -print("call results:", bilipnet_layer(random_input)) - -# explict call results: [[1.3719137 2.774568 ] -# [2.7157617 5.8809733]] -explict_call_res = bilipnet_layer.explicit_call(random_input.numpy(force=True), bilipnet_layer.direct_to_explicit()) -print("explict call results:", explict_call_res ) - -# inverse: [[0.99999946 1.9999995 ] -# [2.9999998 3.9999983 ]] -print("inverse:", bilipnet_layer.inverse(explict_call_res, - alphas=[0.1]*2, - inverse_activation_fns=[lambda x: np.maximum(0, x), lambda x: np.maximum(0, x)], - iterations=[500, 500], - Lambdas=[1, 1] ) ) \ No newline at end of file diff --git a/test/plnet_torch/monlipnet_test.py b/test/plnet_torch/monlipnet_test.py deleted file mode 100644 index 74c234e..0000000 --- a/test/plnet_torch/monlipnet_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.monlipnet_torch import MonLipNet -import torch -import numpy as np - -# Set seeds for all RNGs -seed = 42 -torch.manual_seed(seed) -torch.cuda.manual_seed(seed) # for current GPU -torch.cuda.manual_seed_all(seed) # for all GPUs -np.random.seed(seed) - -# Generate random input -batches = 1 -input_size = 2 -inputs = torch.zeros((batches, input_size)) -random_input = torch.tensor([[1.0,2.0],[3.0,4.0]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -monlipnet_layer = MonLipNet(features=input_size, - unit_features=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=False, - is_nu_fixed=False, - is_tau_fixed=True, - ) - -# Bound: (1.0, 2.0, 2.0) -print("Bound: ", monlipnet_layer.get_bounds()) - -# call results: tensor([[1.2342, 3.2445], - # [3.5813, 6.4334]], grad_fn=) -print("call results:", monlipnet_layer( random_input)) - -# explict call results: [[1.2342471 3.2444794] -# [3.5812564 6.433382 ]] -explict_call_res = monlipnet_layer.explicit_call( random_input.numpy(force=True), monlipnet_layer.direct_to_explicit()) -print("explict call results:", explict_call_res) - -# inverse: [[1. 2.0000005] -# [3. 4. ]] -print("inverse:", monlipnet_layer.inverse(explict_call_res)) - - \ No newline at end of file diff --git a/test/plnet_torch/orthogonal_test.py b/test/plnet_torch/orthogonal_test.py deleted file mode 100644 index f985799..0000000 --- a/test/plnet_torch/orthogonal_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.orthogonal_torch import Unitary -import torch -import numpy as np - -# Set seeds for all RNGs -seed = 42 -torch.manual_seed(seed) -torch.cuda.manual_seed(seed) # for current GPU -torch.cuda.manual_seed_all(seed) # for all GPUs -np.random.seed(seed) - -# Generate random input -batches = 1 -input_size = 2 -inputs = torch.zeros((batches, input_size)) -random_input = torch.tensor([[1.0,2.0],[3.0,4.0]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -unitary_layer = Unitary(in_features=input_size, out_features=input_size, bias=True) - -# call results: tensor([[-1.7999, 1.6573] - # [-3.1680, 4.1328]], grad_fn=) -print("call results:", unitary_layer( random_input)) - -# explict call results: [[-1.7998953 1.6573124] -# [-3.168004 4.1328483]] -explict_call_res = unitary_layer.explicit_call( random_input.numpy(force=True), unitary_layer.direct_to_explicit()) -print("explict call results:", explict_call_res) - -# inverse: [[1. 2.0000005] -# [3. 4. ]] -print("inverse:", unitary_layer.inverse(explict_call_res)) \ No newline at end of file diff --git a/test/plnet_torch/plnet_test.py b/test/plnet_torch/plnet_test.py deleted file mode 100644 index 5741732..0000000 --- a/test/plnet_torch/plnet_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -from robustnn.bilipnet_torch import BiLipNet -from robustnn.plnet_torch import PLNet -import torch -import numpy as np - -# Set seeds for all RNGs -seed = 42 -torch.manual_seed(seed) -torch.cuda.manual_seed(seed) # for current GPU -torch.cuda.manual_seed_all(seed) # for all GPUs -np.random.seed(seed) - -# Generate random input -batches = 1 -input_size = 2 -inputs = torch.zeros((batches, input_size)) -random_input = torch.tensor([[1.0,2.0],[3.0,4.0]]) -units = [2,2] -depth = 2 -mu = 1 -nu = 2 -tau = 2 - -# Initialize a unitary layer -bilipnet_layer = BiLipNet(features=input_size, - unit_features=units, - mu=mu, - nu=nu, - tau=tau, - is_mu_fixed=True, - is_nu_fixed=False, - is_tau_fixed=False, - depth=depth, - ) -plnet_layer = PLNet(BiLipBlock=bilipnet_layer) - -# call results: tensor([ 2.3951, 10.4903], grad_fn=) -print("call results:", plnet_layer(random_input)) - -# explict call results: [ 2.3950937 10.490302 ] -explict_call_res = plnet_layer.explicit_call(random_input.numpy(force=True), plnet_layer.direct_to_explicit()) -print("explict call results:", explict_call_res ) - -random_input_min = torch.tensor([[2.0,3.0],[4.0,5.0]]) -# explict call with gt: [0.7159801 0.7159799] -print("explict call with gt: ", plnet_layer.explicit_call(random_input.numpy(force=True), - plnet_layer.direct_to_explicit(x_optimal=random_input_min.numpy(force=True)) )) - -random_input_min = random_input -# explict call with same gt: [0. 0.] -print("explict call with same gt: ", plnet_layer.explicit_call(random_input.numpy(force=True), - plnet_layer.direct_to_explicit(x_optimal=random_input_min.numpy(force=True)) )) - diff --git a/test/test_bilipren.py b/test/test_bilipren.py deleted file mode 100644 index e3993ab..0000000 --- a/test/test_bilipren.py +++ /dev/null @@ -1,154 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -''' -Tests for the JAX bi-Lipschitz REN (`BiLipschitzREN`) and its composition -(`CompositionREN`): forward evaluation, inverse round-trip, empirical -bi-Lipschitz bounds, and gradient flow. -''' - -import jax -import jax.numpy as jnp -from jax.flatten_util import ravel_pytree - -from robustnn import ren_jax as ren -from robustnn.ren_composition_jax import CompositionREN - -# Avoid matmul precision discrepancies (see issue #15). -jax.config.update("jax_default_matmul_precision", "highest") - - -def test_bilipschitz_ren(): - rng = jax.random.key(0) - rng, k1, k2 = jax.random.split(rng, 3) - - nu_io, nx, nv = 3, 4, 8 - mu, nu = 0.5, 4.0 - model = ren.BiLipschitzREN(nu_io, nx, nv, nu_io, mu=mu, nu=nu, - init_method="long_memory") - model.check_valid_qsr() - - batches = 5 - states = model.initialize_carry(k1, (batches, nu_io)) - inputs = jax.random.normal(k2, (batches, nu_io)) - params = model.init(rng, states, inputs) - - explicit = model.direct_to_explicit(params) - _, y = model.explicit_call(params, states, inputs, explicit) - assert y.shape == (batches, nu_io) - - # Inverse round-trip recovers the inputs. - explicit_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call(params, states, y, explicit_inv) - rel_err = jnp.linalg.norm(u_rec - inputs) / jnp.linalg.norm(inputs) - assert rel_err < 1e-4, f"inverse round-trip error too large: {rel_err}" - - # Empirical gains lie within the bi-Lipschitz bounds. - rng, ka, kb = jax.random.split(rng, 3) - ua = jax.random.normal(ka, (300, nu_io)) - ub = jax.random.normal(kb, (300, nu_io)) - z = jnp.zeros((300, nx)) - _, ya = model.explicit_call(params, z, ua, explicit) - _, yb = model.explicit_call(params, z, ub, explicit) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.min() >= mu - 1e-3 - assert gain.max() <= nu + 1e-3 - - # Gradients are finite. - def loss(p): - e = model.direct_to_explicit(p) - _, yy = model.explicit_call(p, states, inputs, e) - return jnp.sum(yy ** 2) - g = jax.grad(loss)(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def test_composition_ren(): - rng = jax.random.key(1) - rng, k1, k2 = jax.random.split(rng, 3) - - io, nx, nv, L = 2, 4, 8, 3 - mu, nu = 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, - init_method="long_memory") - - batches = 6 - states = model.initialize_carry(k1, (batches, io)) - inputs = jax.random.normal(k2, (batches, io)) - params = model.init(rng, states, inputs) - - new_states, y = model.apply(params, states, inputs) - assert len(new_states["rens"]) == L - assert y.shape == (batches, io) - - # Inverse round-trip. - e_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call(params, states, y, e_inv) - rel_err = jnp.linalg.norm(u_rec - inputs) / jnp.linalg.norm(inputs) - assert rel_err < 1e-4, f"composition inverse error too large: {rel_err}" - - # Empirical gains within bounds. - rng, ka, kb = jax.random.split(rng, 3) - ua = jax.random.normal(ka, (300, io)) - ub = jax.random.normal(kb, (300, io)) - z = {"rens": [jnp.zeros((300, nx)) for _ in range(L)], - "dyn_in": None, "dyn_out": None} - e = model.direct_to_explicit(params) - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.min() >= mu - 1e-3 - assert gain.max() <= nu + 1e-3 - assert model.get_bounds(params) == (mu, nu) - - def loss(p): - _, yy = model.apply(p, states, inputs) - return jnp.sum(yy ** 2) - g = jax.grad(loss)(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def test_composition_ren_dyn_orth(): - """Composition with a dynamic-orthogonal input layer: both inverse modes.""" - rng = jax.random.key(2) - rng, k1, k2 = jax.random.split(rng, 3) - - io, nx, nv, L = 2, 3, 6, 2 - mu, nu = 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, - init_method="long_memory", - dyn_orth_at_input=True, dyn_state_multiplier=4) - - batches = 5 - carry = model.initialize_carry(k1, (batches, io)) - assert carry["dyn_in"] is not None and carry["dyn_out"] is None - inputs = jax.random.normal(k2, (batches, io)) - params = model.init(rng, carry, inputs) - - new_carry, y = model.apply(params, carry, inputs) - assert len(new_carry["rens"]) == L - assert new_carry["dyn_in"].shape == (batches, 4 * nx) - assert y.shape == (batches, io) - - e_inv = model.direct_to_explicit_inverse(params) - - # Non-causal inverse (uses the saved dyn-orth next state) recovers the - # true input exactly. - _, u_rec = model.inverse_call_noncausal(params, carry, new_carry, y, e_inv) - rel_err = jnp.linalg.norm(u_rec - inputs) / jnp.linalg.norm(inputs) - assert rel_err < 1e-4, f"noncausal inverse error too large: {rel_err}" - - # Causal "ignore" inverse recovers the signal at the dyn-orth output (i.e. - # the dyn-orth forward output), not the true input. - _, sig_rec = model.inverse_call(params, carry, y, e_inv) - # The dyn-orth forward output: y_dyn = state @ C.T + inputs @ D.T + by. - ed = e_inv.dyn_in - expected = carry["dyn_in"] @ ed.C.T + inputs @ ed.D.T + ed.by - rel_err2 = jnp.linalg.norm(sig_rec - expected) / jnp.linalg.norm(expected) - assert rel_err2 < 1e-4, f"ignore inverse mismatch: {rel_err2}" - - -if __name__ == "__main__": - test_bilipschitz_ren() - test_composition_ren() - test_composition_ren_dyn_orth() - print("All JAX bi-Lipschitz REN tests passed.") diff --git a/test/test_bilipren_torch.py b/test/test_bilipren_torch.py deleted file mode 100644 index 7c5f7cf..0000000 --- a/test/test_bilipren_torch.py +++ /dev/null @@ -1,116 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -''' -Tests for the PyTorch bi-Lipschitz REN (`BiLipschitzREN`) and its composition -(`CompositionREN`): forward evaluation, inverse round-trip, empirical -bi-Lipschitz bounds, and gradient flow. -''' - -import torch - -from robustnn.ren_torch import BiLipschitzREN -from robustnn.ren_composition_torch import CompositionREN - - -def test_bilipschitz_ren_torch(): - torch.manual_seed(0) - io, nx, nv = 3, 4, 8 - mu, nu = 0.5, 4.0 - model = BiLipschitzREN(io, nx, nv, mu=mu, nu=nu) - - batches = 5 - state = model.initialize_carry(batches) - u = torch.randn(batches, io) - _, y = model(state, u) - assert tuple(y.shape) == (batches, io) - - with torch.no_grad(): - _, u_rec = model.inverse(state, y) - rel_err = ((u_rec - u).norm() / u.norm()).item() - assert rel_err < 1e-4, f"inverse round-trip error too large: {rel_err}" - - z = torch.zeros(300, nx) - ua, ub = torch.randn(300, io), torch.randn(300, io) - with torch.no_grad(): - e = model.direct_to_explicit() - _, ya = model.explicit_call(z, ua, e) - _, yb = model.explicit_call(z, ub, e) - gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - - (y ** 2).sum().backward() - gnorm = sum(p.grad.abs().sum() for p in model.parameters() if p.grad is not None) - assert bool(torch.isfinite(gnorm)) - - -def test_composition_ren_torch(): - torch.manual_seed(1) - io, nx, nv, L = 2, 4, 8, 3 - mu, nu = 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu) - - batches = 6 - states = model.initialize_carry(batches) - u = torch.randn(batches, io) - new_states, y = model(states, u) - assert len(new_states["rens"]) == L - assert tuple(y.shape) == (batches, io) - - with torch.no_grad(): - _, u_rec = model.inverse(states, y) - rel_err = ((u_rec - u).norm() / u.norm()).item() - assert rel_err < 1e-4, f"composition inverse error too large: {rel_err}" - - zc = {"rens": [torch.zeros(300, nx) for _ in range(L)], - "dyn_in": None, "dyn_out": None} - ua, ub = torch.randn(300, io), torch.randn(300, io) - with torch.no_grad(): - _, ca = model(zc, ua) - _, cb = model(zc, ub) - gain = (ca - cb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - assert model.get_bounds() == (mu, nu) - - (y ** 2).sum().backward() - gnorm = sum(p.grad.abs().sum() for p in model.parameters() if p.grad is not None) - assert bool(torch.isfinite(gnorm)) - - -def test_composition_ren_torch_dyn_orth(): - """Composition with a dynamic-orthogonal input layer: both inverse modes.""" - torch.manual_seed(2) - io, nx, nv, L = 2, 3, 6, 2 - mu, nu = 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, - dyn_orth_at_input=True, dyn_state_multiplier=4) - - batches = 5 - carry = model.initialize_carry(batches) - assert carry["dyn_in"] is not None and carry["dyn_out"] is None - u = torch.randn(batches, io) - new_carry, y = model(carry, u) - assert len(new_carry["rens"]) == L - assert tuple(new_carry["dyn_in"].shape) == (batches, 4 * nx) - assert tuple(y.shape) == (batches, io) - - # Non-causal inverse recovers the true input exactly. - with torch.no_grad(): - _, u_rec = model.inverse_noncausal(carry, new_carry, y) - rel_err = ((u_rec - u).norm() / u.norm()).item() - assert rel_err < 1e-4, f"noncausal inverse error too large: {rel_err}" - - # Causal "ignore" inverse recovers the dyn-orth forward output. - with torch.no_grad(): - _, sig_rec = model.inverse(carry, y) - _, dyn_out = model.dyn_in(carry["dyn_in"], u) - rel_err2 = ((sig_rec - dyn_out).norm() / dyn_out.norm()).item() - assert rel_err2 < 1e-4, f"ignore inverse mismatch: {rel_err2}" - - -if __name__ == "__main__": - test_bilipschitz_ren_torch() - test_composition_ren_torch() - test_composition_ren_torch_dyn_orth() - print("All PyTorch bi-Lipschitz REN tests passed.") diff --git a/test/test_lbdn.py b/test/test_lbdn.py deleted file mode 100644 index be941b3..0000000 --- a/test/test_lbdn.py +++ /dev/null @@ -1,48 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -import jax -import jax.numpy as jnp -import flax.linen as nn - -from robustnn.lbdn_jax import LBDN - -# Need this to avoid matrix multiplication discrepancy (see issue #15) -jax.config.update("jax_default_matmul_precision", "highest") - -rng = jax.random.key(0) -rng, key = jax.random.split(rng, 2) - -# Model size and Lipschitz bound -nu, ny = 5, 2 -layers = (8, 16) -gamma = jnp.float32(10) - -# Create LBDN model -model = LBDN(nu, layers, ny, gamma=gamma, activation=nn.tanh) - -# Dummy inputs -batches = 4 -inputs = jnp.ones((batches, nu)) -params = model.init(key, inputs) - -# Forward mode -# Test separate parameter conversion and model call. This is the -# same as defining jit_call = jax.jit(model.apply) -@jax.jit -def jit_call(params, inputs): - explicit = model.direct_to_explicit(params) - return model.explicit_call(params, inputs, explicit) - -out = jit_call(params, inputs) -print(out) - -# Test taking a gradient -def loss(inputs): - out = jit_call(params, inputs) - return jnp.sum(out**2) - -grad_func = jax.jit(jax.grad(loss)) -gs = grad_func(inputs) - -print(loss(inputs)) -print("Output grad: ", gs[0]) diff --git a/test/test_linren.py b/test/test_linren.py deleted file mode 100644 index 71cea5e..0000000 --- a/test/test_linren.py +++ /dev/null @@ -1,53 +0,0 @@ -import jax -import jax.numpy as jnp -import flax.linen as nn - -from robustnn import linear_ren as ren - -jax.config.update("jax_default_matmul_precision", "highest") - -# Random seeds -rng = jax.random.key(0) -rng, keyX, keyY, keyS, key1, key2 = jax.random.split(rng, 6) - -# Initialise a QSR-constrained REN -nu, nx, ny = 5, 3, 2 -X = jax.random.normal(keyX, (ny, ny)) -Y = jax.random.normal(keyY, (nu, nu)) -S = jax.random.normal(keyS, (nu, ny)) - -Q = -X.T @ X -R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y - -model = ren.GeneralLinREN(nu, nx, 0, ny, Q=Q, S=S, R=R, - activation=nn.tanh, init_method="long_memory") -model.check_valid_qsr() - -# Dummy inputs and states -batches = 4 -states = model.initialize_carry(key1, (batches, nu)) + 1 -inputs = jnp.ones((batches, nu)) -params = model.init(key2, states, inputs) - -# Forward mode -# jit_call = jax.jit(model.apply) -@jax.jit -def jit_call(params, states, inputs): - explicit = model.direct_to_explicit(params) - return model.explicit_call(params, states, inputs, explicit) - -new_state, out = jit_call(params, states, inputs) -print(new_state) -print(out) - -# Test taking a gradient -def loss(states, inputs): - nstate, out = jit_call(params, states, inputs) - return jnp.sum(nstate**2) + jnp.sum(out**2) - -grad_func = jax.jit(jax.grad(loss, argnums=(0,1))) -gs = grad_func(states, inputs) - -print(loss(states, inputs)) -print("States grad: ", gs[0]) -print("Input grad: ", gs[1]) diff --git a/test/test_r2dn.py b/test/test_r2dn.py deleted file mode 100644 index e55497d..0000000 --- a/test/test_r2dn.py +++ /dev/null @@ -1,87 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -import jax -import jax.numpy as jnp - -from robustnn import r2dn_jax as r2dn -from robustnn.utils import count_num_params -from utils import compute_p_contractingr2dn - -# Need this to avoid matrix multiplication discrepancy (see issue #15) -jax.config.update("jax_default_matmul_precision", "highest") - -# Problem size -nu = 5 -ny = 2 -batches = 4 -horizon = 2 - -# Initialise an R2DN -nx = 3 # Number of states -nv = 4 # Number of equilibirum layer states -nh = (2,) * 2 # Number of hidden layers in the LBDN -init_method = "long_memory" -model = r2dn.ContractingR2DN(nu, nx, nv, ny, nh, init_method=init_method) - -# Random seeds -rng = jax.random.key(0) -rng, key1, key2, key3, key4 = jax.random.split(rng, 5) - -# Dummy inputs and states -states = model.initialize_carry(key1, (batches, None)) -states = jax.random.normal(key2, states.shape) -inputs = jax.random.normal(key3, (horizon, batches, nu)) - -# Initialise the model and check how many params -params = model.init(key4, states, inputs[0]) -print("Number of params: ", count_num_params(params)) - -# Call forward pass -jit_call = jax.jit(model.simulate_sequence) -new_state, out = jit_call(params, states, inputs) -print(new_state) -print(out) - -# Test taking a gradient -@jax.jit -def loss(params, x0, u): - x1, y = model.simulate_sequence(params, x0, u) - return jnp.sum(x1**2) + jnp.sum(y**2) - -grad_func = jax.jit(jax.grad(loss, argnums=(0,1,2))) -gs = grad_func(params, states, inputs) - -print(loss(params, states, inputs)) -print("States grad: ", gs[1]) -print("Input grad: ", gs[2]) - - -############################################################ -# Quick test of contraction condition - -def mat_norm2(x, A): - return jnp.sum((x @ A.T) * x, axis=-1) - -# Random seeds -rng = jax.random.key(0) -rng, key1, key2, key3, key4, key5 = jax.random.split(rng, 6) - -# Dummy inputs and states -states = model.initialize_carry(key1, (batches, nu)) -x0 = 10*jax.random.normal(key2, states.shape) -x1 = jax.random.normal(key3, states.shape) -inputs = jax.random.normal(key4, (batches, nu)) - -params = model.init(key5, states, inputs) - -# Simulate one step -x0n, _ = model.apply(params, x0, inputs) -x1n, _ = model.apply(params, x1, inputs) - -# Test for contraction -P = compute_p_contractingr2dn(model, params) -lhs = mat_norm2(x0n - x1n, P) - mat_norm2(x0 - x1, P) -rhs = 0 - -assert jnp.all(lhs <= rhs) -print("Passed (very basic) contraction test.") diff --git a/test/test_ren.py b/test/test_ren.py deleted file mode 100644 index e65ca0b..0000000 --- a/test/test_ren.py +++ /dev/null @@ -1,57 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -import jax -import jax.numpy as jnp -import flax.linen as nn - -from robustnn import ren_jax as ren - -# Need this to avoid matrix multiplication discrepancy (see issue #15) -jax.config.update("jax_default_matmul_precision", "highest") - -# Random seeds -rng = jax.random.key(0) -rng, keyX, keyY, keyS, key1, key2 = jax.random.split(rng, 6) - -# Initialise a QSR-constrained REN -nu, nx, nv, ny = 5, 3, 4, 2 -X = jax.random.normal(keyX, (ny, ny)) -Y = jax.random.normal(keyY, (nu, nu)) -S = jax.random.normal(keyS, (nu, ny)) - -Q = -X.T @ X -R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y - -model = ren.GeneralREN(nu, nx, nv, ny, Q=Q, S=S, R=R, - activation=nn.tanh, init_method="long_memory") -model.check_valid_qsr() - -# Dummy inputs and states -batches = 4 -states = model.initialize_carry(key1, (batches, nu)) + 1 -inputs = jnp.ones((batches, nu)) -params = model.init(key2, states, inputs) - -# Forward mode -# Test separate parameter conversion and model call. This is the -# same as defining jit_call = jax.jit(model.apply) -@jax.jit -def jit_call(params, states, inputs): - explicit = model.direct_to_explicit(params) - return model.explicit_call(params, states, inputs, explicit) - -new_state, out = jit_call(params, states, inputs) -print(new_state) -print(out) - -# Test taking a gradient -def loss(states, inputs): - nstate, out = jit_call(params, states, inputs) - return jnp.sum(nstate**2) + jnp.sum(out**2) - -grad_func = jax.jit(jax.grad(loss, argnums=(0,1))) -gs = grad_func(states, inputs) - -print(loss(states, inputs)) -print("States grad: ", gs[0]) -print("Input grad: ", gs[1]) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a6fec62 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Shared pytest fixtures. Backend-specific fixtures live in the +`jax_tests/` and `torch_tests/` conftest files.""" + +import numpy as np +import pytest + + +@pytest.fixture(autouse=True) +def _seed_numpy(): + np.random.seed(0) diff --git a/test/utils.py b/tests/helpers.py similarity index 100% rename from test/utils.py rename to tests/helpers.py diff --git a/tests/jax_tests/conftest.py b/tests/jax_tests/conftest.py new file mode 100644 index 0000000..daad96a --- /dev/null +++ b/tests/jax_tests/conftest.py @@ -0,0 +1,19 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +import pytest + +pytest.importorskip("jax") +pytest.importorskip("flax") + +import jax + + +def pytest_collection_modifyitems(items): + for item in items: + if "jax_tests" in str(item.fspath): + item.add_marker(pytest.mark.jax) + + +@pytest.fixture +def key(): + return jax.random.key(0) diff --git a/tests/jax_tests/test_composition.py b/tests/jax_tests/test_composition.py new file mode 100644 index 0000000..c4d8242 --- /dev/null +++ b/tests/jax_tests/test_composition.py @@ -0,0 +1,155 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `DynUnitary` and `CompositionREN`.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +from robustnn.dyn_orthogonal_jax import DynUnitary +from robustnn.ren_composition_jax import CompositionREN + + +# ------------------------------ DynUnitary -------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_dyn_unitary(key, batches, io, nx, use_bias): + layer = DynUnitary(io, nx, use_bias=use_bias) + x = jax.random.normal(key, (batches, nx)) + u = jax.random.normal(jax.random.key(1), (batches, io)) + params = layer.init(key, x, u) + + assert layer.initialize_carry(key, (batches, io)).shape == (batches, nx) + e = layer.direct_to_explicit(params) + x1, y = layer.explicit_call(params, x, u, e) + assert x1.shape == (batches, nx) and y.shape == (batches, io) + x1b, yb = layer.apply(params, x, u) + np.testing.assert_allclose(x1, x1b, atol=1e-5) + np.testing.assert_allclose(y, yb, atol=1e-5) + + # Stacked state-space matrix is orthogonal. + G = jnp.block([[e.A, e.B], [e.C, e.D]]) + np.testing.assert_allclose(G @ G.T, jnp.eye(nx + io), atol=1e-5) + + # Non-causal inverse recovers the previous state and the input. + x_rec, u_rec = layer.inverse_call(params, x1, y, e) + np.testing.assert_allclose(x_rec, x, atol=1e-4) + np.testing.assert_allclose(u_rec, u, atol=1e-4) + + g = jax.grad(lambda p: sum(jnp.sum(a ** 2) for a in layer.apply(p, x, u)))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +# ----------------------------- CompositionREN ----------------------------- +def _model(io=2, nx=3, nv=6, L=2, mu=0.5, nu=5.0, **kw): + return CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, + init_method="long_memory", **kw) + + +def _setup(model, key, batches, io): + k1, k2 = jax.random.split(key) + carry = model.initialize_carry(k1, (batches, io)) + inputs = jax.random.normal(k2, (batches, io)) + params = model.init(key, carry, inputs) + return params, carry, inputs + + +def _zero_carry(model, n, nx, L): + c = model.initialize_carry(jax.random.key(0), (n, model.input_size)) + return jax.tree.map(jnp.zeros_like, c) + + +@pytest.mark.parametrize("batches", [1, 6]) +@pytest.mark.parametrize("L", [1, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +@pytest.mark.parametrize("activation", [nn.relu, nn.tanh]) +def test_composition_ren(key, batches, L, use_bias, activation): + io, nx, mu, nu = 2, 3, 0.5, 5.0 + model = _model(io=io, nx=nx, L=L, mu=mu, nu=nu, use_bias=use_bias, + activation=activation) + params, carry, inputs = _setup(model, key, batches, io) + + new_carry, y = model.apply(params, carry, inputs) + assert len(new_carry["rens"]) == L + assert y.shape == (batches, io) + assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None + + e = model.direct_to_explicit(params) + _, y_exp = model.explicit_call(params, carry, inputs, e) + np.testing.assert_allclose(y, y_exp, atol=1e-5) + + if activation is nn.relu: + # The inverse solver is only accurate to ~1e-3 for smooth activations + # at the default iteration count, so only check it for relu. + e_inv = model.direct_to_explicit_inverse(params) + _, u_rec = model.inverse_call(params, carry, y, e_inv) + np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) + + n = 300 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, io)) + ub = jax.random.normal(kb, (n, io)) + z = {"rens": [jnp.zeros((n, nx)) for _ in range(L)], + "dyn_in": None, "dyn_out": None} + _, ya = model.explicit_call(params, z, ua, e) + _, yb = model.explicit_call(params, z, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.min() >= mu - 1e-3 + assert gain.max() <= nu + 1e-3 + assert model.get_bounds(params) == (mu, nu) + + def loss(p): + return jnp.sum(model.apply(p, carry, inputs)[1] ** 2) + g = jax.grad(loss)(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +@pytest.mark.parametrize("at_input,at_output", + [(True, False), (False, True), (True, True)]) +def test_composition_ren_dyn_orth_noncausal_inverse(key, at_input, at_output): + io, nx, L = 2, 3, 2 + model = _model(io=io, nx=nx, nv=6, L=L, dyn_orth_at_input=at_input, + dyn_orth_at_output=at_output, dyn_state_multiplier=4) + params, carry, inputs = _setup(model, key, 5, io) + assert (carry["dyn_in"] is not None) == at_input + assert (carry["dyn_out"] is not None) == at_output + + new_carry, y = model.apply(params, carry, inputs) + assert y.shape == (5, io) + if at_input: + assert new_carry["dyn_in"].shape == (5, 4 * nx) + + e_inv = model.direct_to_explicit_inverse(params) + _, u_rec = model.inverse_call_noncausal(params, carry, new_carry, y, e_inv) + np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) + + +def test_composition_ren_ignore_inverse_with_dyn_in(key): + model = _model(dyn_orth_at_input=True, dyn_state_multiplier=4) + params, carry, inputs = _setup(model, key, 5, 2) + _, y = model.apply(params, carry, inputs) + e_inv = model.direct_to_explicit_inverse(params) + _, sig_rec = model.inverse_call(params, carry, y, e_inv) + ed = e_inv.dyn_in + expected = carry["dyn_in"] @ ed.C.T + inputs @ ed.D.T + ed.by + np.testing.assert_allclose(sig_rec, expected, atol=1e-3, rtol=1e-3) + + +def test_composition_ren_ignore_inverse_rejects_dyn_out(key): + model = _model(dyn_orth_at_output=True, dyn_state_multiplier=4) + params, carry, inputs = _setup(model, key, 3, 2) + _, y = model.apply(params, carry, inputs) + e_inv = model.direct_to_explicit_inverse(params) + with pytest.raises(ValueError): + model.inverse_call(params, carry, y, e_inv) + + +@pytest.mark.parametrize("kw", [dict(L=0), dict(mu=2.0, nu=1.0), dict(mu=1.0, nu=1.0)]) +def test_composition_ren_invalid_args(key, kw): + with pytest.raises(ValueError): + model = _model(**kw) + _setup(model, key, 2, 2) diff --git a/tests/jax_tests/test_lipnets.py b/tests/jax_tests/test_lipnets.py new file mode 100644 index 0000000..3ab84e9 --- /dev/null +++ b/tests/jax_tests/test_lipnets.py @@ -0,0 +1,130 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `MonLipNet` and `BiLipNet`.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +from robustnn.bilipnet_jax import BiLipNet +from robustnn.monlipnet_jax import MonLipNet + + +def _gains(f, key, n, size): + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, size)) + ub = jax.random.normal(kb, (n, size)) + return jnp.linalg.norm(f(ua) - f(ub), axis=1) / jnp.linalg.norm(ua - ub, axis=1) + + +# ----------------------------- MonLipNet --------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) +@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) +def test_monlipnet_shape_explicit_and_bounds(key, batches, units, fixed): + size, mu, nu, tau = 3, 0.5, 4.0, 4.0 + flags = dict(is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, + is_tau_fixed="tau" in fixed) + net = MonLipNet(input_size=size, units=units, mu=mu, nu=nu, tau=tau, **flags) + x = jax.random.normal(key, (batches, size)) + params = net.init(key, x) + + y = net.apply(params, x) + assert y.shape == (batches, size) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + lo, hi, _ = net.get_bounds(params) + gains = _gains(lambda u: net.apply(params, u), key, 200, size) + assert gains.min() >= float(lo) - 1e-3 + assert gains.max() <= float(hi) + 1e-3 + + +def test_monlipnet_fixing_all_raises(key): + net = MonLipNet(input_size=2, units=[2], is_mu_fixed=True, + is_nu_fixed=True, is_tau_fixed=True) + with pytest.raises(ValueError): + net.init(key, jnp.ones((1, 2))) + + +@pytest.mark.parametrize("units", [[3], [4, 4]]) +def test_monlipnet_inverse_round_trip(key, units): + net = MonLipNet(input_size=3, units=units, mu=0.5, nu=4.0, tau=4.0, + is_tau_fixed=True) + x = jax.random.normal(key, (6, 3)) + params = net.init(key, x) + y = net.apply(params, x) + e_inv = net.direct_to_explicit_inverse(params, alpha=0.1, iterations=500) + x_rec = net.inverse_call(params, y, e_inv) + np.testing.assert_allclose(x_rec, x, atol=1e-3) + + +def test_monlipnet_jit_and_grad(key): + net = MonLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=4.0, + is_tau_fixed=True) + x = jax.random.normal(key, (5, 3)) + params = net.init(key, x) + np.testing.assert_allclose(jax.jit(net.apply)(params, x), + net.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) + flat = ravel_pytree(g)[0] + assert bool(jnp.isfinite(flat).all()) and bool(jnp.any(flat != 0)) + + +# ----------------------------- BiLipNet ---------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("depth", [1, 2, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_bilipnet_shape_explicit_and_bounds(key, batches, depth, use_bias): + size, mu, nu = 3, 0.5, 4.0 + net = BiLipNet(input_size=size, units=[4, 4], mu=mu, nu=nu, tau=nu / mu, + is_mu_fixed=True, is_nu_fixed=True, depth=depth, + use_bias=use_bias) + x = jax.random.normal(key, (batches, size)) + params = net.init(key, x) + + y = net.apply(params, x) + assert y.shape == (batches, size) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + lo, hi, _ = net.get_bounds(params) + np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-5) + gains = _gains(lambda u: net.apply(params, u), key, 200, size) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_bilipnet_inverse_round_trip(key, depth): + net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True, depth=depth, act_fn=nn.relu) + x = jax.random.normal(key, (6, 3)) + params = net.init(key, x) + y = net.apply(params, x) + inv_act = nn.relu + e_inv = net.direct_to_explicit_inverse( + params, [0.1] * depth, [inv_act] * depth, [500] * depth, [1.0] * depth) + x_rec = net.inverse_call(params, y, e_inv) + np.testing.assert_allclose(x_rec, x, atol=1e-3) + + +def test_bilipnet_jit_and_grad(key): + net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True) + x = jax.random.normal(key, (5, 3)) + params = net.init(key, x) + np.testing.assert_allclose(jax.jit(net.apply)(params, x), + net.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +def test_bilipnet_fixing_all_raises(key): + net = BiLipNet(input_size=2, units=[2], is_mu_fixed=True, + is_nu_fixed=True, is_tau_fixed=True) + with pytest.raises(ValueError): + net.init(key, jnp.ones((1, 2))) diff --git a/tests/jax_tests/test_orthogonal.py b/tests/jax_tests/test_orthogonal.py new file mode 100644 index 0000000..85e657f --- /dev/null +++ b/tests/jax_tests/test_orthogonal.py @@ -0,0 +1,67 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `Unitary` (Cayley) layer.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +from robustnn.orthogonal_jax import Unitary + + +@pytest.mark.parametrize("batches", [1, 7]) +@pytest.mark.parametrize("size", [1, 2, 5]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_shape_and_explicit_matches_direct(key, batches, size, use_bias): + layer = Unitary(input_size=size, use_bias=use_bias) + x = jax.random.normal(key, (batches, size)) + params = layer.init(key, x) + + y = layer.apply(params, x) + assert y.shape == (batches, size) + + explicit = layer.direct_to_explicit(params) + y_exp = layer.explicit_call(params, x, explicit) + np.testing.assert_allclose(y, y_exp, atol=1e-5) + + +@pytest.mark.parametrize("size", [2, 5]) +def test_orthogonal_and_norm_preserving(key, size): + layer = Unitary(input_size=size, use_bias=False) + x = jax.random.normal(key, (16, size)) + params = layer.init(key, x) + R = layer.direct_to_explicit(params).R + np.testing.assert_allclose(R @ R.T, jnp.eye(size), atol=1e-5) + + y = layer.apply(params, x) + np.testing.assert_allclose(jnp.linalg.norm(y, axis=1), + jnp.linalg.norm(x, axis=1), rtol=1e-4) + + +@pytest.mark.parametrize("use_bias", [True, False]) +def test_inverse_round_trip(key, use_bias): + layer = Unitary(input_size=4, use_bias=use_bias) + x = jax.random.normal(key, (6, 4)) + params = layer.init(key, x) + # Non-zero bias so that the inverse has to undo it. + if use_bias: + params = jax.tree.map(lambda p: p, params) + params["params"]["b"] = jnp.arange(4, dtype=jnp.float32) + explicit = layer.direct_to_explicit(params) + y = layer.explicit_call(params, x, explicit) + x_rec = layer.inverse_call(params, y, explicit) + np.testing.assert_allclose(x_rec, x, atol=1e-4) + + +def test_jit_and_grad(key): + layer = Unitary(input_size=3) + x = jax.random.normal(key, (5, 3)) + params = layer.init(key, x) + + np.testing.assert_allclose(jax.jit(layer.apply)(params, x), + layer.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(layer.apply(p, x) ** 2))(params) + flat = ravel_pytree(g)[0] + assert bool(jnp.isfinite(flat).all()) diff --git a/tests/jax_tests/test_plnet_lbdn.py b/tests/jax_tests/test_plnet_lbdn.py new file mode 100644 index 0000000..06c636d --- /dev/null +++ b/tests/jax_tests/test_plnet_lbdn.py @@ -0,0 +1,105 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `PLNet` and `LBDN`.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +from robustnn.bilipnet_jax import BiLipNet +from robustnn.lbdn_jax import LBDN +from robustnn.plnet_jax import PLNet + + +def _plnet(size=3, **kw): + bilip = BiLipNet(input_size=size, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True, depth=2) + return PLNet(BiLipBlock=bilip, **kw) + + +# ------------------------------- PLNet ----------------------------------- +@pytest.mark.parametrize("batches", [1, 6]) +def test_plnet_shape_and_explicit(key, batches): + net = _plnet() + x = jax.random.normal(key, (batches, 3)) + params = net.init(key, x) + y = net.apply(params, x) + assert y.shape == (batches,) + assert bool(jnp.all(y >= 0)) + + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + +def test_plnet_zero_at_optimal_point(key): + net = _plnet() + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + explicit = net.direct_to_explicit(params, x_optimal=x[:1]) + y_opt = net.explicit_call(params, x[:1], explicit) + np.testing.assert_allclose(y_opt, 0.0, atol=1e-6) + assert bool(jnp.all(net.explicit_call(params, x, explicit) >= 0)) + + +def test_plnet_add_constant(key): + net = _plnet(add_constant=True, c=1.5) + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), + net.apply(params, x), atol=1e-5) + + +def test_plnet_grad(key): + net = _plnet() + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x)))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +# -------------------------------- LBDN ----------------------------------- +@pytest.mark.parametrize("activation", [nn.relu, nn.tanh, nn.sigmoid, lambda x: x]) +@pytest.mark.parametrize("hidden", [(4,), (8, 16)]) +@pytest.mark.parametrize("batches", [1, 5]) +def test_lbdn_shape_explicit_and_lipschitz(key, activation, hidden, batches): + nin, nout, gamma = 5, 2, 3.0 + net = LBDN(nin, hidden, nout, gamma=gamma, activation=activation) + x = jax.random.normal(key, (batches, nin)) + params = net.init(key, x) + + y = net.apply(params, x) + assert y.shape == (batches, nout) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (200, nin)) + ub = jax.random.normal(kb, (200, nin)) + gain = (jnp.linalg.norm(net.apply(params, ua) - net.apply(params, ub), axis=1) + / jnp.linalg.norm(ua - ub, axis=1)) + assert gain.max() <= gamma + 1e-3 + + +@pytest.mark.parametrize("use_bias", [True, False]) +@pytest.mark.parametrize("trainable_lipschitz", [True, False]) +def test_lbdn_options_and_grad(key, use_bias, trainable_lipschitz): + net = LBDN(3, (6,), 2, gamma=2.0, use_bias=use_bias, + trainable_lipschitz=trainable_lipschitz) + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + np.testing.assert_allclose(jax.jit(net.apply)(params, x), + net.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +@pytest.mark.xfail(reason="init_output_zero documents a zero output at init, but the output is non-zero (see PR notes)") +def test_lbdn_init_output_zero(key): + net = LBDN(3, (6,), 2, init_output_zero=True) + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + np.testing.assert_allclose(net.apply(params, x), 0.0, atol=1e-6) diff --git a/tests/jax_tests/test_ren.py b/tests/jax_tests/test_ren.py new file mode 100644 index 0000000..733d97d --- /dev/null +++ b/tests/jax_tests/test_ren.py @@ -0,0 +1,294 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX RENs: Contracting, Lipschitz, General (QSR) and +BiLipschitz, plus the linear RENs and the contracting R2DN.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +from helpers import compute_p_contractingr2dn, compute_p_contractingren +from robustnn import linear_ren +from robustnn import r2dn_jax as r2dn +from robustnn import ren_jax as ren + +ACTIVATIONS = [nn.relu, nn.tanh, lambda x: x] + + +def _setup(model, key, batches, nin, x_scale=1.0): + k1, k2, k3 = jax.random.split(key, 3) + states = model.initialize_carry(k1, (batches, nin)) + states = x_scale * jax.random.normal(k2, states.shape) + inputs = jax.random.normal(k3, (batches, nin)) + params = model.init(key, states, inputs) + return params, states, inputs + + +def _qsr(key, nu, ny): + kx, ky, ks = jax.random.split(key, 3) + X = jax.random.normal(kx, (ny, ny)) + Y = jax.random.normal(ky, (nu, nu)) + S = jax.random.normal(ks, (nu, ny)) + Q = -X.T @ X + R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y + return Q, S, R + + +def _check_forward(model, params, states, inputs, nx, ny): + batches = inputs.shape[0] + explicit = model.direct_to_explicit(params) + xn, y = model.explicit_call(params, states, inputs, explicit) + assert xn.shape == (batches, nx) + assert y.shape == (batches, ny) + xn2, y2 = model.apply(params, states, inputs) + np.testing.assert_allclose(xn, xn2, atol=1e-5) + np.testing.assert_allclose(y, y2, atol=1e-5) + + # jit gives the same answer, and gradients w.r.t. everything are finite + jit_xn, jit_y = jax.jit(model.apply)(params, states, inputs) + np.testing.assert_allclose(y, jit_y, atol=1e-4) + + def loss(p, x, u): + a, b = model.apply(p, x, u) + return jnp.sum(a ** 2) + jnp.sum(b ** 2) + g = jax.grad(loss, argnums=(0, 1, 2))(params, states, inputs) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +# --------------------------- Contracting REN ----------------------------- +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("activation", ACTIVATIONS) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +@pytest.mark.parametrize("do_polar_param", [True, False]) +@pytest.mark.parametrize("sizes", [(1, 1, 1, 1), (5, 3, 4, 2)]) +def test_contracting_ren(key, batches, activation, init_method, do_polar_param, sizes): + nu, nx, nv, ny = sizes + model = ren.ContractingREN(nu, nx, nv, ny, activation=activation, + init_method=init_method, do_polar_param=do_polar_param) + params, states, inputs = _setup(model, key, batches, nu) + _check_forward(model, params, states, inputs, nx, ny) + + +def test_contracting_ren_contraction(key): + model = ren.ContractingREN(5, 3, 4, 2, activation=nn.tanh) + k1, k2 = jax.random.split(key) + params, x1, u = _setup(model, key, 8, 5) + x0 = 10 * jax.random.normal(k1, x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingren(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) + + +def test_ren_identity_output_and_zero_output(key): + model = ren.ContractingREN(3, 3, 4, 3, identity_output=True) + params, states, inputs = _setup(model, key, 4, 3) + _, y = model.apply(params, states, inputs) + # identity output: y_t = x_t, i.e. the state before the update + np.testing.assert_allclose(y, states, atol=1e-5) + + zero = ren.ContractingREN(3, 3, 4, 2, init_output_zero=True) + params, states, inputs = _setup(zero, key, 4, 3) + _, y = zero.apply(params, states, inputs) + np.testing.assert_allclose(y, 0.0, atol=1e-6) + + +def test_ren_identity_output_size_mismatch(key): + model = ren.ContractingREN(3, 3, 4, 2, identity_output=True) + with pytest.raises(ValueError): + _setup(model, key, 2, 3) + + +def test_ren_bad_init_method(key): + model = ren.ContractingREN(3, 3, 4, 2, init_method="nope") + with pytest.raises(ValueError): + _setup(model, key, 2, 3) + + +# ----------------------------- Lipschitz REN ----------------------------- +@pytest.mark.parametrize("gamma", [0.5, 1.0, 5.0]) +@pytest.mark.parametrize("activation", ACTIVATIONS) +def test_lipschitz_ren(key, gamma, activation): + nu, nx, nv, ny = 4, 3, 5, 2 + model = ren.LipschitzREN(nu, nx, nv, ny, gamma=gamma, activation=activation) + params, states, inputs = _setup(model, key, 4, nu) + _check_forward(model, params, states, inputs, nx, ny) + + # Same initial state => one-step output gain is bounded by gamma. + n = 200 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nu)) + ub = jax.random.normal(kb, (n, nu)) + x = jnp.zeros((n, nx)) + e = model.direct_to_explicit(params) + _, ya = model.explicit_call(params, x, ua, e) + _, yb = model.explicit_call(params, x, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.max() <= gamma + 1e-3 + + +def test_lipschitz_ren_identity_output_unsupported(key): + with pytest.raises(NotImplementedError): + model = ren.LipschitzREN(3, 3, 4, 3, identity_output=True) + _setup(model, key, 2, 3) + + +# ------------------------------ General REN ------------------------------ +@pytest.mark.parametrize("activation", ACTIVATIONS) +@pytest.mark.parametrize("sizes", [(5, 3, 4, 2), (2, 2, 3, 2)]) +def test_general_ren(key, activation, sizes): + nu, nx, nv, ny = sizes + Q, S, R = _qsr(key, nu, ny) + model = ren.GeneralREN(nu, nx, nv, ny, Q=Q, S=S, R=R, activation=activation, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = _setup(model, key, 4, nu) + _check_forward(model, params, states, inputs, nx, ny) + + +def test_general_ren_invalid_qsr(key): + nu, ny = 3, 2 + Q, S, R = _qsr(key, nu, ny) + bad = [ + dict(Q=jnp.eye(ny + 1), S=S, R=R), # wrong Q size + dict(Q=Q, S=jnp.ones((ny, nu)), R=R), # wrong S size + dict(Q=Q, S=S, R=jnp.eye(nu + 1)), # wrong R size + dict(Q=jnp.eye(ny), S=S, R=R), # Q not neg. definite + dict(Q=Q, S=S, R=-jnp.eye(nu)), # Schur complement not PD + ] + for kw in bad: + model = ren.GeneralREN(nu, 3, 4, ny, **kw) + with pytest.raises(ValueError): + model.check_valid_qsr() + + +# ----------------------------- BiLipschitz REN ---------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("nio", [1, 3]) +@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) +def test_bilipschitz_ren(key, batches, nio, mu_nu): + mu, nu = mu_nu + nx, nv = 4, 8 + model = ren.BiLipschitzREN(nio, nx, nv, nio, mu=mu, nu=nu, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = _setup(model, key, batches, nio) + _check_forward(model, params, states, inputs, nx, nio) + + e = model.direct_to_explicit(params) + _, y = model.explicit_call(params, states, inputs, e) + e_inv = model.direct_to_explicit_inverse(params) + _, u_rec = model.inverse_call(params, states, y, e_inv) + np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) + + n = 300 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nio)) + ub = jax.random.normal(kb, (n, nio)) + z = jnp.zeros((n, nx)) + _, ya = model.explicit_call(params, z, ua, e) + _, yb = model.explicit_call(params, z, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.min() >= mu - 1e-3 + assert gain.max() <= nu + 1e-3 + + +def test_bilipschitz_ren_requires_square(key): + model = ren.BiLipschitzREN(3, 3, 4, 2) + with pytest.raises(ValueError): + _setup(model, key, 2, 3) + + +# ------------------------------ simulate_sequence ------------------------- +@pytest.mark.parametrize("horizon", [1, 5]) +def test_simulate_sequence_matches_loop(key, horizon): + nu, nx, nv, ny = 3, 4, 5, 2 + model = ren.ContractingREN(nu, nx, nv, ny, activation=nn.tanh) + params, x0, u0 = _setup(model, key, 4, nu) + seq = jax.random.normal(key, (horizon, 4, nu)) + + xT, ys = model.simulate_sequence(params, x0, seq) + assert ys.shape == (horizon, 4, ny) + + x, outs = x0, [] + for t in range(horizon): + x, y = model.apply(params, x, seq[t]) + outs.append(y) + np.testing.assert_allclose(xT, x, atol=1e-4) + np.testing.assert_allclose(ys, jnp.stack(outs), atol=1e-4) + + +# --------------------------------- Linear RENs ---------------------------- +@pytest.mark.parametrize("cls", ["contracting", "lipschitz", "general"]) +@pytest.mark.parametrize("batches", [1, 4]) +def test_linear_ren(key, cls, batches): + nu, nx, ny = 5, 3, 2 + if cls == "contracting": + model = linear_ren.ContractingLinREN(nu, nx, 0, ny) + elif cls == "lipschitz": + model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=2.0) + else: + Q, S, R = _qsr(key, nu, ny) + model = linear_ren.GeneralLinREN(nu, nx, 0, ny, Q=Q, S=S, R=R, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = _setup(model, key, batches, nu) + _check_forward(model, params, states, inputs, nx, ny) + + +def test_linear_ren_lipschitz_bound(key): + nu, nx, ny, gamma = 4, 3, 2, 2.0 + model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=gamma) + params, _, _ = _setup(model, key, 2, nu) + n = 200 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nu)) + ub = jax.random.normal(kb, (n, nu)) + z = jnp.zeros((n, nx)) + e = model.direct_to_explicit(params) + _, ya = model.explicit_call(params, z, ua, e) + _, yb = model.explicit_call(params, z, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.max() <= gamma + 1e-3 + + +def test_linear_ren_requires_zero_features(key): + with pytest.raises(ValueError): + model = linear_ren.ContractingLinREN(3, 3, 4, 2) + _setup(model, key, 2, 3) + + +# ----------------------------------- R2DN --------------------------------- +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("hidden", [(2,), (2, 2), (4, 3, 2)]) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +def test_contracting_r2dn(key, batches, hidden, init_method): + nu, nx, nv, ny = 5, 3, 4, 2 + model = r2dn.ContractingR2DN(nu, nx, nv, ny, hidden, init_method=init_method) + params, states, inputs = _setup(model, key, batches, nu) + _check_forward(model, params, states, inputs, nx, ny) + + +def test_r2dn_simulate_and_contraction(key): + nu, nx, nv, ny, horizon = 5, 3, 4, 2, 3 + model = r2dn.ContractingR2DN(nu, nx, nv, ny, (2, 2), init_method="long_memory") + params, x1, u = _setup(model, key, 4, nu) + + seq = jax.random.normal(key, (horizon, 4, nu)) + xT, ys = jax.jit(model.simulate_sequence)(params, x1, seq) + assert ys.shape == (horizon, 4, ny) and xT.shape == x1.shape + + x0 = 10 * jax.random.normal(key, x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingr2dn(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) diff --git a/tests/test_parity.py b/tests/test_parity.py new file mode 100644 index 0000000..4e97c22 --- /dev/null +++ b/tests/test_parity.py @@ -0,0 +1,61 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""JAX vs PyTorch parity: build the torch layer, copy its *explicit* +parameters into the JAX layer, and check both give the same outputs. + +Initialisations differ between the two backends, so parameters have to be +copied across explicitly. Only the layers with a simple explicit +parameterisation are covered; MonLipNet / BiLipNet / REN parity is a TODO +(their explicit parameter layouts differ between backends). +""" + +import numpy as np +import pytest + +jax = pytest.importorskip("jax") +jnp = pytest.importorskip("jax.numpy") +torch = pytest.importorskip("torch") + +from robustnn.dyn_orthogonal_jax import DynUnitary as JaxDynUnitary +from robustnn.dyn_orthogonal_jax import ExplicitDynOrthogonalParams +from robustnn.dyn_orthogonal_torch import DynUnitary as TorchDynUnitary +from robustnn.orthogonal_jax import ExplicitOrthogonalParams, Unitary as JaxUnitary +from robustnn.orthogonal_torch import Unitary as TorchUnitary + + +@pytest.mark.parametrize("size", [2, 5]) +def test_unitary_parity(size): + torch.manual_seed(0) + t_layer = TorchUnitary(size, size, bias=True) + with torch.no_grad(): + t_layer.bias.copy_(torch.randn(size)) + x = torch.randn(6, size) + y_torch = t_layer(x).detach().numpy() + + e = t_layer.direct_to_explicit() + j_layer = JaxUnitary(input_size=size, use_bias=True) + params = j_layer.init(jax.random.key(0), jnp.asarray(x.numpy())) + j_explicit = ExplicitOrthogonalParams(R=jnp.asarray(e.Q), b=jnp.asarray(e.b)) + y_jax = j_layer.explicit_call(params, jnp.asarray(x.numpy()), j_explicit) + np.testing.assert_allclose(np.asarray(y_jax), y_torch, atol=1e-5) + + +@pytest.mark.parametrize("io,nx", [(2, 3), (3, 6)]) +def test_dyn_unitary_parity(io, nx): + torch.manual_seed(0) + t_layer = TorchDynUnitary(io, nx) + x, u = torch.randn(5, nx), torch.randn(5, io) + with torch.no_grad(): + x1_t, y_t = t_layer(x, u) + A, B, C, D = [m.numpy() for m in t_layer._blocks()] + + j_layer = JaxDynUnitary(io, nx) + jx, ju = jnp.asarray(x.numpy()), jnp.asarray(u.numpy()) + params = j_layer.init(jax.random.key(0), jx, ju) + e = ExplicitDynOrthogonalParams( + jnp.asarray(A), jnp.asarray(B), jnp.asarray(C), jnp.asarray(D), + jnp.asarray(t_layer.bx.detach().numpy()), + jnp.asarray(t_layer.by.detach().numpy())) + x1_j, y_j = j_layer.explicit_call(params, jx, ju, e) + np.testing.assert_allclose(np.asarray(x1_j), x1_t.numpy(), atol=1e-5) + np.testing.assert_allclose(np.asarray(y_j), y_t.numpy(), atol=1e-5) diff --git a/tests/torch_tests/conftest.py b/tests/torch_tests/conftest.py new file mode 100644 index 0000000..2109078 --- /dev/null +++ b/tests/torch_tests/conftest.py @@ -0,0 +1,18 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +import pytest + +pytest.importorskip("torch") + +import torch + + +def pytest_collection_modifyitems(items): + for item in items: + if "torch_tests" in str(item.fspath): + item.add_marker(pytest.mark.torch) + + +@pytest.fixture(autouse=True) +def _seed_torch(): + torch.manual_seed(0) diff --git a/tests/torch_tests/test_lipnets.py b/tests/torch_tests/test_lipnets.py new file mode 100644 index 0000000..a1e2082 --- /dev/null +++ b/tests/torch_tests/test_lipnets.py @@ -0,0 +1,171 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `Unitary`, `MonLipNet`, `BiLipNet` and `PLNet`.""" + +import numpy as np +import pytest +import torch +import torch.nn as nn + +from robustnn.bilipnet_torch import BiLipNet +from robustnn.monlipnet_torch import MonLipNet +from robustnn.orthogonal_torch import Unitary +from robustnn.plnet_torch import PLNet + + +def _gains(f, n, size): + ua, ub = torch.randn(n, size), torch.randn(n, size) + with torch.no_grad(): + return (f(ua) - f(ub)).norm(dim=1) / (ua - ub).norm(dim=1) + + +def _grads_finite(model): + grads = [p.grad for p in model.parameters() if p.grad is not None] + assert grads + assert all(bool(torch.isfinite(g).all()) for g in grads) + + +relu_np = lambda x: np.maximum(0, x) + + +# -------------------------------- Unitary --------------------------------- +@pytest.mark.parametrize("batches", [1, 7]) +@pytest.mark.parametrize("size", [1, 2, 5]) +def test_unitary(batches, size): + layer = Unitary(size, size, bias=True) + x = torch.randn(batches, size) + y = layer(x) + assert tuple(y.shape) == (batches, size) + + e = layer.direct_to_explicit() + np.testing.assert_allclose(e.Q @ e.Q.T, np.eye(size), atol=1e-5) + np.testing.assert_allclose(layer.explicit_call(x.numpy(), e), + y.detach().numpy(), atol=1e-5) + + with torch.no_grad(): + x_rec = layer.inverse(y) + np.testing.assert_allclose(np.asarray(x_rec), x.numpy(), atol=1e-4) + + (y ** 2).sum().backward() + _grads_finite(layer) + + +@pytest.mark.parametrize("size", [2, 5]) +def test_unitary_no_bias_forward_preserves_norm(size): + layer = Unitary(size, size, bias=False) + x = torch.randn(8, size) + np.testing.assert_allclose(layer(x).norm(dim=1).detach(), x.norm(dim=1), rtol=1e-4) + + +@pytest.mark.xfail(raises=AttributeError, strict=True, + reason="Unitary.direct_to_explicit assumes a bias exists") +def test_unitary_no_bias_explicit(): + Unitary(3, 3, bias=False).direct_to_explicit() + + +# ------------------------------- MonLipNet -------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) +@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) +def test_monlipnet(batches, units, fixed): + size, mu, nu, tau = 3, 0.5, 4.0, 4.0 + net = MonLipNet(size, units, mu=mu, nu=nu, tau=tau, + is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, + is_tau_fixed="tau" in fixed) + x = torch.randn(batches, size) + y = net(x) + assert tuple(y.shape) == (batches, size) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + + lo, hi, _ = net.get_bounds() + gains = _gains(net, 200, size) + assert gains.min() >= float(lo) - 1e-3 + assert gains.max() <= float(hi) + 1e-3 + + (y ** 2).sum().backward() + _grads_finite(net) + + +@pytest.mark.parametrize("units", [[3], [4, 4]]) +def test_monlipnet_inverse_round_trip(units): + net = MonLipNet(3, units, mu=0.5, nu=4.0, tau=4.0, is_tau_fixed=True) + x = torch.randn(6, 3) + with torch.no_grad(): + y = net(x) + x_rec = net.inverse(y.numpy(), alpha=0.1, iterations=500) + np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) + + +@pytest.mark.parametrize("kw", [dict(mu=1.0), dict(), dict(nu=2.0)]) +def test_monlipnet_needs_two_bounds(kw): + with pytest.raises(ValueError): + MonLipNet(2, [2], **kw) + + +# ------------------------------- BiLipNet --------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("depth", [1, 2, 3]) +def test_bilipnet(batches, depth): + size, mu, nu = 3, 0.5, 4.0 + net = BiLipNet(size, [4, 4], mu=mu, nu=nu, tau=nu / mu, + is_mu_fixed=True, is_nu_fixed=True, depth=depth) + x = torch.randn(batches, size) + y = net(x) + assert tuple(y.shape) == (batches, size) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + + lo, hi, _ = net.get_bounds() + np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-4) + gains = _gains(net, 200, size) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + (y ** 2).sum().backward() + _grads_finite(net) + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_bilipnet_inverse_round_trip(depth): + net = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, + is_nu_fixed=True, depth=depth) + x = torch.randn(6, 3) + with torch.no_grad(): + y = net(x) + x_rec = net.inverse(y.numpy(), alphas=[0.1] * depth, + inverse_activation_fns=[relu_np] * depth, + iterations=[500] * depth, Lambdas=[1.0] * depth) + np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) + + +# --------------------------------- PLNet ---------------------------------- +def _plnet(**kw): + bilip = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, + is_nu_fixed=True, depth=2) + return PLNet(BiLipBlock=bilip, **kw) + + +@pytest.mark.parametrize("batches", [1, 6]) +def test_plnet(batches): + net = _plnet() + x = torch.randn(batches, 3) + y = net(x) + assert tuple(y.shape) == (batches,) + assert bool((y >= 0).all()) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + y.sum().backward() + _grads_finite(net) + + +def test_plnet_zero_at_optimal_point(): + net = _plnet() + x = torch.randn(4, 3) + e = net.direct_to_explicit(x_optimal=x[:1].numpy()) + y = net.explicit_call(x[:1].numpy(), e) + np.testing.assert_allclose(y, 0.0, atol=1e-6) + assert bool((net.explicit_call(x.numpy(), e) >= 0).all()) diff --git a/tests/torch_tests/test_ren.py b/tests/torch_tests/test_ren.py new file mode 100644 index 0000000..10e8f4d --- /dev/null +++ b/tests/torch_tests/test_ren.py @@ -0,0 +1,176 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `BiLipschitzREN`, `DynUnitary` and `CompositionREN`.""" + +import pytest +import torch +import torch.nn as nn + +from robustnn.dyn_orthogonal_torch import DynUnitary +from robustnn.ren_composition_torch import CompositionREN +from robustnn.ren_torch import BiLipschitzREN, RENBase + + +def _grads_finite(model): + grads = [p.grad for p in model.parameters() if p.grad is not None] + assert grads + assert all(bool(torch.isfinite(g).all()) for g in grads) + + +def _rel_err(a, b): + return ((a - b).norm() / b.norm()).item() + + +# ------------------------------ BiLipschitzREN ---------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("nio", [1, 3]) +@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) +@pytest.mark.parametrize("activation", [nn.ReLU(), nn.Tanh()]) +def test_bilipschitz_ren(batches, nio, mu_nu, activation): + mu, nu = mu_nu + nx, nv = 4, 8 + model = BiLipschitzREN(nio, nx, nv, mu=mu, nu=nu, activation=activation) + + state = model.initialize_carry(batches) + assert tuple(state.shape) == (batches, nx) + u = torch.randn(batches, nio) + new_state, y = model(state, u) + assert tuple(new_state.shape) == (batches, nx) + assert tuple(y.shape) == (batches, nio) + + with torch.no_grad(): + e = model.direct_to_explicit() + _, y_exp = model.explicit_call(state, u, e) + torch.testing.assert_close(y_exp, y, atol=1e-4, rtol=1e-4) + + if isinstance(activation, nn.ReLU): + with torch.no_grad(): + _, u_rec = model.inverse(state, y) + assert _rel_err(u_rec, u) < 1e-3 + + n = 300 + z = torch.zeros(n, nx) + ua, ub = torch.randn(n, nio), torch.randn(n, nio) + with torch.no_grad(): + _, ya = model.explicit_call(z, ua, e) + _, yb = model.explicit_call(z, ub, e) + gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) + assert gain.min().item() >= mu - 1e-3 + assert gain.max().item() <= nu + 1e-3 + + (y ** 2).sum().backward() + _grads_finite(model) + + +@pytest.mark.parametrize("mu,nu", [(2.0, 1.0), (1.0, 1.0)]) +def test_bilipschitz_ren_invalid_bounds(mu, nu): + with pytest.raises(ValueError): + BiLipschitzREN(2, 3, 4, mu=mu, nu=nu) + + +def test_ren_base_direct_to_explicit_not_implemented(): + base = RENBase(2, 3, 4, 2) + with pytest.raises(NotImplementedError): + base.direct_to_explicit() + + +# ------------------------------- DynUnitary ------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) +@pytest.mark.parametrize("bias", [True, False]) +def test_dyn_unitary(batches, io, nx, bias): + layer = DynUnitary(io, nx, bias=bias) + x, u = torch.randn(batches, nx), torch.randn(batches, io) + assert tuple(layer.initialize_carry(batches).shape) == (batches, nx) + + x1, y = layer(x, u) + assert tuple(x1.shape) == (batches, nx) and tuple(y.shape) == (batches, io) + + A, B, C, D = layer._blocks() + G = torch.cat([torch.cat([A, B], 1), torch.cat([C, D], 1)], 0) + torch.testing.assert_close(G @ G.T, torch.eye(nx + io), atol=1e-5, rtol=0) + + x_rec, u_rec = layer.inverse(x1, y) + torch.testing.assert_close(x_rec, x, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(u_rec, u, atol=1e-4, rtol=1e-4) + + (x1 ** 2).sum().add((y ** 2).sum()).backward() + _grads_finite(layer) + + +# ------------------------------ CompositionREN ---------------------------- +@pytest.mark.parametrize("batches", [1, 6]) +@pytest.mark.parametrize("L", [1, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_composition_ren(batches, L, use_bias): + io, nx, nv, mu, nu = 2, 3, 6, 0.5, 5.0 + model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, use_bias=use_bias) + + carry = model.initialize_carry(batches) + u = torch.randn(batches, io) + new_carry, y = model(carry, u) + assert len(new_carry["rens"]) == L + assert tuple(y.shape) == (batches, io) + assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None + + with torch.no_grad(): + _, u_rec = model.inverse(carry, y) + assert _rel_err(u_rec, u) < 1e-3 + + n = 300 + zc = {"rens": [torch.zeros(n, nx) for _ in range(L)], + "dyn_in": None, "dyn_out": None} + ua, ub = torch.randn(n, io), torch.randn(n, io) + with torch.no_grad(): + _, ya = model(zc, ua) + _, yb = model(zc, ub) + gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) + assert gain.min().item() >= mu - 1e-3 + assert gain.max().item() <= nu + 1e-3 + assert model.get_bounds() == (mu, nu) + + (y ** 2).sum().backward() + _grads_finite(model) + + +@pytest.mark.parametrize("at_input,at_output", + [(True, False), (False, True), (True, True)]) +def test_composition_ren_dyn_orth_noncausal_inverse(at_input, at_output): + io, nx, L = 2, 3, 2 + model = CompositionREN(io, nx, 6, num_layers=L, mu=0.5, nu=5.0, + dyn_orth_at_input=at_input, dyn_orth_at_output=at_output, + dyn_state_multiplier=4) + carry = model.initialize_carry(5) + assert (carry["dyn_in"] is not None) == at_input + assert (carry["dyn_out"] is not None) == at_output + + u = torch.randn(5, io) + new_carry, y = model(carry, u) + assert tuple(y.shape) == (5, io) + if at_input: + assert tuple(new_carry["dyn_in"].shape) == (5, 4 * nx) + + with torch.no_grad(): + _, u_rec = model.inverse_noncausal(carry, new_carry, y) + assert _rel_err(u_rec, u) < 1e-3 + + +def test_composition_ren_ignore_inverse_with_dyn_in(): + model = CompositionREN(2, 3, 6, num_layers=2, mu=0.5, nu=5.0, + dyn_orth_at_input=True, dyn_state_multiplier=4) + carry = model.initialize_carry(5) + u = torch.randn(5, 2) + _, y = model(carry, u) + with torch.no_grad(): + _, sig_rec = model.inverse(carry, y) + _, dyn_out = model.dyn_in(carry["dyn_in"], u) + assert _rel_err(sig_rec, dyn_out) < 1e-3 + + +@pytest.mark.parametrize("kw", [dict(num_layers=0), dict(mu=2.0, nu=1.0), + dict(mu=1.0, nu=1.0)]) +def test_composition_ren_invalid_args(kw): + args = dict(num_layers=2, mu=0.5, nu=5.0) + args.update(kw) + with pytest.raises(ValueError): + CompositionREN(2, 3, 6, **args) From 1da00759b00d1a3ead2d11f8efb320154577f097 Mon Sep 17 00:00:00 2001 From: juraph Date: Thu, 8 Oct 2026 11:53:24 +1100 Subject: [PATCH 2/4] Remove xfails Treating these as bugs. Resolve before merging. --- tests/jax_tests/test_plnet_lbdn.py | 1 - tests/torch_tests/test_lipnets.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/jax_tests/test_plnet_lbdn.py b/tests/jax_tests/test_plnet_lbdn.py index 06c636d..5fcc872 100644 --- a/tests/jax_tests/test_plnet_lbdn.py +++ b/tests/jax_tests/test_plnet_lbdn.py @@ -97,7 +97,6 @@ def test_lbdn_options_and_grad(key, use_bias, trainable_lipschitz): assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) -@pytest.mark.xfail(reason="init_output_zero documents a zero output at init, but the output is non-zero (see PR notes)") def test_lbdn_init_output_zero(key): net = LBDN(3, (6,), 2, init_output_zero=True) x = jax.random.normal(key, (4, 3)) diff --git a/tests/torch_tests/test_lipnets.py b/tests/torch_tests/test_lipnets.py index a1e2082..b215e61 100644 --- a/tests/torch_tests/test_lipnets.py +++ b/tests/torch_tests/test_lipnets.py @@ -57,8 +57,6 @@ def test_unitary_no_bias_forward_preserves_norm(size): np.testing.assert_allclose(layer(x).norm(dim=1).detach(), x.norm(dim=1), rtol=1e-4) -@pytest.mark.xfail(raises=AttributeError, strict=True, - reason="Unitary.direct_to_explicit assumes a bias exists") def test_unitary_no_bias_explicit(): Unitary(3, 3, bias=False).direct_to_explicit() From da47996eaba08fb79458285782d37cb4ff041d75 Mon Sep 17 00:00:00 2001 From: juraph Date: Wed, 14 Oct 2026 13:23:45 +1100 Subject: [PATCH 3/4] Move each network into individual files & json Each network type should have it's own dedicated file. This allows for easily adding additional tests. Further, expected inputs/outputs have been moved into their own *.json file, which has removed the requirement of the dedicated parity testing, as input/outputs are shared between them --- README.md | 8 +- tests/generate_golden.py | 310 +++++++++++++++++ tests/golden.py | 131 +++++++ tests/golden/bilipnet.json | 168 +++++++++ tests/golden/bilipschitz_ren.json | 117 +++++++ tests/golden/contracting_linear_ren.json | 37 ++ tests/golden/contracting_ren.json | 112 ++++++ tests/golden/dyn_unitary.json | 46 +++ tests/golden/general_linear_ren.json | 40 +++ tests/golden/general_ren.json | 82 +++++ tests/golden/lbdn.json | 103 ++++++ tests/golden/lipschitz_linear_ren.json | 38 ++ tests/golden/lipschitz_ren.json | 78 +++++ tests/golden/monlipnet.json | 93 +++++ tests/golden/plnet.json | 177 ++++++++++ tests/golden/r2dn.json | 108 ++++++ tests/golden/ren_composition.json | 326 ++++++++++++++++++ tests/golden/unitary.json | 60 ++++ tests/jax_tests/test_bilipnet.py | 76 ++++ tests/jax_tests/test_bilipschitz_ren.py | 55 +++ tests/jax_tests/test_contracting_ren.py | 91 +++++ tests/jax_tests/test_dyn_orthogonal.py | 50 +++ tests/jax_tests/test_general_ren.py | 45 +++ .../{test_plnet_lbdn.py => test_lbdn.py} | 59 +--- tests/jax_tests/test_linear_ren.py | 69 ++++ tests/jax_tests/test_lipnets.py | 130 ------- tests/jax_tests/test_lipschitz_ren.py | 45 +++ tests/jax_tests/test_monlipnet.py | 73 ++++ tests/jax_tests/test_orthogonal.py | 8 + tests/jax_tests/test_plnet.py | 67 ++++ tests/jax_tests/test_r2dn.py | 48 +++ tests/jax_tests/test_ren.py | 294 ---------------- ...composition.py => test_ren_composition.py} | 42 +-- tests/jax_utils.py | 177 ++++++++++ tests/test_parity.py | 61 ---- tests/torch_tests/test_bilipnet.py | 57 +++ tests/torch_tests/test_bilipschitz_ren.py | 70 ++++ tests/torch_tests/test_dyn_orthogonal.py | 40 +++ tests/torch_tests/test_lipnets.py | 169 --------- tests/torch_tests/test_monlipnet.py | 58 ++++ tests/torch_tests/test_orthogonal.py | 50 +++ tests/torch_tests/test_plnet.py | 48 +++ tests/torch_tests/test_ren.py | 176 ---------- tests/torch_tests/test_ren_composition.py | 94 +++++ tests/torch_utils.py | 219 ++++++++++++ 45 files changed, 3490 insertions(+), 915 deletions(-) create mode 100644 tests/generate_golden.py create mode 100644 tests/golden.py create mode 100644 tests/golden/bilipnet.json create mode 100644 tests/golden/bilipschitz_ren.json create mode 100644 tests/golden/contracting_linear_ren.json create mode 100644 tests/golden/contracting_ren.json create mode 100644 tests/golden/dyn_unitary.json create mode 100644 tests/golden/general_linear_ren.json create mode 100644 tests/golden/general_ren.json create mode 100644 tests/golden/lbdn.json create mode 100644 tests/golden/lipschitz_linear_ren.json create mode 100644 tests/golden/lipschitz_ren.json create mode 100644 tests/golden/monlipnet.json create mode 100644 tests/golden/plnet.json create mode 100644 tests/golden/r2dn.json create mode 100644 tests/golden/ren_composition.json create mode 100644 tests/golden/unitary.json create mode 100644 tests/jax_tests/test_bilipnet.py create mode 100644 tests/jax_tests/test_bilipschitz_ren.py create mode 100644 tests/jax_tests/test_contracting_ren.py create mode 100644 tests/jax_tests/test_dyn_orthogonal.py create mode 100644 tests/jax_tests/test_general_ren.py rename tests/jax_tests/{test_plnet_lbdn.py => test_lbdn.py} (55%) create mode 100644 tests/jax_tests/test_linear_ren.py delete mode 100644 tests/jax_tests/test_lipnets.py create mode 100644 tests/jax_tests/test_lipschitz_ren.py create mode 100644 tests/jax_tests/test_monlipnet.py create mode 100644 tests/jax_tests/test_plnet.py create mode 100644 tests/jax_tests/test_r2dn.py delete mode 100644 tests/jax_tests/test_ren.py rename tests/jax_tests/{test_composition.py => test_ren_composition.py} (76%) create mode 100644 tests/jax_utils.py delete mode 100644 tests/test_parity.py create mode 100644 tests/torch_tests/test_bilipnet.py create mode 100644 tests/torch_tests/test_bilipschitz_ren.py create mode 100644 tests/torch_tests/test_dyn_orthogonal.py delete mode 100644 tests/torch_tests/test_lipnets.py create mode 100644 tests/torch_tests/test_monlipnet.py create mode 100644 tests/torch_tests/test_orthogonal.py create mode 100644 tests/torch_tests/test_plnet.py delete mode 100644 tests/torch_tests/test_ren.py create mode 100644 tests/torch_tests/test_ren_composition.py create mode 100644 tests/torch_utils.py diff --git a/README.md b/README.md index d7039d5..601ae05 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,19 @@ This will create a Python virtual environment at `./.venv`, install the pinned P ### Testing -Tests are written with [pytest](https://pytest.org) and live in `tests/`: `tests/jax_tests/` for the JAX/Flax implementations, `tests/torch_tests/` for the PyTorch implementations, and `tests/test_parity.py` for JAX-vs-PyTorch checks. `./run_tests.sh` is a thin wrapper around `uv run pytest`, so any pytest arguments work: +Tests are written with [pytest](https://pytest.org) and live in `tests/`. Each network has its own test file, in `tests/jax_tests/` for the JAX/Flax implementations and `tests/torch_tests/` for the PyTorch implementations (e.g. `test_plnet.py`, `test_bilipschitz_ren.py`). `./run_tests.sh` is a thin wrapper around `uv run pytest`, so any pytest arguments work: ```bash ./run_tests.sh # everything ./run_tests.sh -m torch # PyTorch tests only (or -m jax) ./run_tests.sh -n auto # run in parallel -./run_tests.sh -k bilip # select by name +./run_tests.sh -k plnet # select by name ``` +Besides property tests (output shapes, inverse round-trips, Lipschitz bounds, gradients, error handling), every network is checked against **golden input/output files** in `tests/golden/.json`. Each case stores the constructor arguments, the (unconstrained) parameters, some inputs, and the expected outputs. The JAX and PyTorch tests both load the same file, so if both pass, the two implementations agree with each other. Networks that only exist in JAX (e.g. `LBDN`, the contracting/Lipschitz/general RENs, the linear RENs and the R2DN) are checked against their golden files too, which guards against regressions when refactoring. + +Golden files are generated from the JAX implementation with `uv run python tests/generate_golden.py [network ...]`. Only regenerate them deliberately (a new case, or an intended change to a network) and review the diff. + The same suite runs on every push to `main` and every pull request through GitHub Actions (`.github/workflows/tests.yml`, CPU only). ### A Note on Dependencies diff --git a/tests/generate_golden.py b/tests/generate_golden.py new file mode 100644 index 0000000..a37a276 --- /dev/null +++ b/tests/generate_golden.py @@ -0,0 +1,310 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Regenerate the golden input/output files in `tests/golden/`. + + uv run python tests/generate_golden.py # all networks + uv run python tests/generate_golden.py plnet lbdn # just these + +The JAX implementation is used as the reference: random (seeded) parameters and +inputs are generated, the outputs are computed with JAX, and everything is +written to JSON. The JAX *and* PyTorch tests then check against these files. + +Only regenerate on purpose (e.g. when a network is deliberately changed or a case +is added) and review the diff: regenerating after an accidental behaviour change +would hide it from the JAX tests (the PyTorch tests would still flag it). +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +import jax +import jax.numpy as jnp +import numpy as np + +import golden +from jax_utils import build_model, run_model + +BATCH = 4 + +# Parameters that act as (positive) scales are perturbed multiplicatively so +# they stay positive; everything else gets additive noise (so biases are non-zero). +_SCALE_KEYS = ("a", "fq", "p", "alpha") + + +def _is_scale(name): + return name in _SCALE_KEYS or name.startswith("fab") + + +def perturb(params, rng, noise=0.3): + def go(tree): + out = {} + for k, v in tree.items(): + if isinstance(v, dict): + out[k] = go(v) + continue + v = np.asarray(v, dtype=np.float32) + if _is_scale(k): + v = v * np.exp(0.2 * rng.standard_normal(v.shape)).astype(np.float32) + else: + v = v + noise * rng.standard_normal(v.shape).astype(np.float32) + out[k] = v.astype(np.float32) + return out + return go(params) + + +def _randn(rng, *shape): + return rng.standard_normal(shape).astype(np.float32) + + +def _make_case(network, name, config, rng, inputs, init_args): + model = build_model(network, config) + variables = model.init(jax.random.key(0), *init_args(inputs)) + params = perturb(jax.tree.map(np.asarray, variables["params"]), rng) + jparams = {"params": jax.tree.map(jnp.asarray, params)} + outputs = run_model(network, model, jparams, inputs) + return { + "name": name, + "config": config, + "params": params, + "inputs": inputs, + "outputs": outputs, + } + + +# --- init-argument adapters --------------------------------------------------- +def _x(i): + return (jnp.asarray(i["x"]),) + + +def _state_u(i): + return (jnp.asarray(i["state"]), jnp.asarray(i["u"])) + + +def _carry_u(i): + c = i["carry"] + carry = { + "rens": [jnp.asarray(r) for r in c["rens"]], + "dyn_in": None if c["dyn_in"] is None else jnp.asarray(c["dyn_in"]), + "dyn_out": None if c["dyn_out"] is None else jnp.asarray(c["dyn_out"]), + } + return (carry, jnp.asarray(i["u"])) + + +# --- the cases --------------------------------------------------------------- +def static_cases(network, specs, size, seed): + rng = np.random.default_rng(seed) + return [ + _make_case(network, name, cfg, rng, {"x": _randn(rng, BATCH, size(cfg))}, _x) + for name, cfg in specs + ] + + +def ren_cases(network, specs, seed): + rng = np.random.default_rng(seed) + out = [] + for name, cfg in specs: + inputs = { + "state": _randn(rng, BATCH, cfg["state_size"]), + "u": _randn(rng, BATCH, cfg["input_size"]), + } + out.append(_make_case(network, name, cfg, rng, inputs, _state_u)) + return out + + +def make_qsr(rng, nu, ny): + X, Y, S = _randn(rng, ny, ny), _randn(rng, nu, nu), _randn(rng, nu, ny) + Q = -X.T @ X + R = S @ np.linalg.solve(Q, S.T) + Y.T @ Y + return Q.astype(np.float32), S, R.astype(np.float32) + + +def gen_unitary(): + specs = [ + ("size3_bias", {"input_size": 3, "use_bias": True}), + ("size1_bias", {"input_size": 1, "use_bias": True}), + ("size5_bias", {"input_size": 5, "use_bias": True}), + ] + return static_cases("unitary", specs, lambda c: c["input_size"], 1) + + +def gen_monlipnet(): + base = {"input_size": 3, "mu": 0.5, "nu": 4.0, "tau": 8.0} + specs = [ + ("units4x4_mu_nu_fixed", base | {"units": [4, 4], "is_mu_fixed": True, "is_nu_fixed": True}), + ("units3_mu_tau_fixed", base | {"units": [3], "nu": 4.0, "tau": 8.0, + "is_mu_fixed": True, "is_tau_fixed": True}), + ("units3x3x3_nu_tau_fixed", base | {"units": [3, 3, 3], "is_nu_fixed": True, + "is_tau_fixed": True}), + ] + return static_cases("monlipnet", specs, lambda c: c["input_size"], 2) + + +def gen_bilipnet(): + base = {"input_size": 3, "mu": 0.5, "nu": 4.0, "tau": 8.0, + "is_mu_fixed": True, "is_nu_fixed": True} + specs = [ + ("depth1_units4", base | {"units": [4], "depth": 1}), + ("depth2_units4x4", base | {"units": [4, 4], "depth": 2}), + ("depth3_units3", base | {"units": [3], "depth": 3}), + ] + return static_cases("bilipnet", specs, lambda c: c["input_size"], 3) + + +def gen_plnet(): + bilip = {"input_size": 3, "units": [4], "mu": 0.5, "nu": 4.0, "tau": 8.0, + "is_mu_fixed": True, "is_nu_fixed": True, "depth": 2} + specs = [ + ("default", {"BiLipBlock": bilip}), + ("add_constant", {"BiLipBlock": bilip, "add_constant": True}), + ("optimal_point", {"BiLipBlock": bilip, "optimal_point": [[1.0, -0.5, 0.3]]}), + ] + return static_cases("plnet", specs, lambda c: c["BiLipBlock"]["input_size"], 4) + + +def gen_lbdn(): + base = {"input_size": 5, "hidden_sizes": [8], "output_size": 2, "gamma": 3.0} + specs = [ + ("relu", base | {"activation": "relu"}), + ("tanh", base | {"activation": "tanh"}), + ("deep_sigmoid_nobias", base | {"hidden_sizes": [6, 6], "activation": "sigmoid", + "use_bias": False}), + ] + return static_cases("lbdn", specs, lambda c: c["input_size"], 5) + + +def gen_dyn_unitary(): + specs = [ + ("io3_nx4", {"input_size": 3, "state_size": 4}), + ("io2_nx6", {"input_size": 2, "state_size": 6}), + ] + return ren_cases("dyn_unitary", specs, 6) + + +_REN = {"input_size": 3, "state_size": 4, "features": 5, "output_size": 2, + "init_method": "long_memory"} + + +def gen_contracting_ren(): + specs = [ + ("relu", _REN | {"activation": "relu"}), + ("tanh", _REN | {"activation": "tanh"}), + ("tanh_no_polar", _REN | {"activation": "tanh", "do_polar_param": False}), + ] + return ren_cases("contracting_ren", specs, 7) + + +def gen_lipschitz_ren(): + specs = [ + ("relu_gamma2", _REN | {"activation": "relu", "gamma": 2.0}), + ("tanh_gamma0.5", _REN | {"activation": "tanh", "gamma": 0.5}), + ] + return ren_cases("lipschitz_ren", specs, 8) + + +def gen_general_ren(): + rng = np.random.default_rng(90) + Q, S, R = make_qsr(rng, 3, 2) + qsr = {"Q": Q.tolist(), "S": S.tolist(), "R": R.tolist()} + specs = [ + ("tanh", _REN | {"activation": "tanh"} | qsr), + ("relu", _REN | {"activation": "relu"} | qsr), + ] + return ren_cases("general_ren", specs, 9) + + +def gen_bilipschitz_ren(): + base = {"input_size": 3, "state_size": 4, "features": 5, "output_size": 3, + "mu": 0.5, "nu": 4.0, "init_method": "long_memory"} + specs = [ + ("relu", base | {"activation": "relu"}), + ("tanh", base | {"activation": "tanh"}), + ("io1_relu", base | {"input_size": 1, "output_size": 1, "activation": "relu"}), + ] + return ren_cases("bilipschitz_ren", specs, 10) + + +def gen_linear_ren(): + lin = _REN | {"features": 0, "init_method": "long_memory"} + lin.pop("activation", None) + rng = np.random.default_rng(110) + Q, S, R = make_qsr(rng, 3, 2) + qsr = {"Q": Q.tolist(), "S": S.tolist(), "R": R.tolist()} + return { + "contracting_linear_ren": ren_cases("contracting_linear_ren", [("default", lin)], 11), + "lipschitz_linear_ren": ren_cases("lipschitz_linear_ren", [("gamma2", lin | {"gamma": 2.0})], 12), + "general_linear_ren": ren_cases("general_linear_ren", [("default", lin | qsr)], 13), + } + + +def gen_r2dn(): + base = {"input_size": 3, "state_size": 4, "features": 5, "output_size": 2, + "init_method": "long_memory"} + specs = [ + ("hidden2x2", base | {"hidden": [2, 2]}), + ("hidden4", base | {"hidden": [4]}), + ] + return ren_cases("r2dn", specs, 14) + + +def gen_ren_composition(): + rng = np.random.default_rng(15) + base = {"input_size": 2, "state_size": 3, "features": 5, "num_layers": 2, + "mu": 0.5, "nu": 5.0, "init_method": "long_memory", "dyn_state_multiplier": 2} + specs = [ + ("plain", base), + ("single_layer_tanh", base | {"num_layers": 1, "activation": "tanh"}), + ("dyn_in", base | {"dyn_orth_at_input": True}), + ("dyn_in_out", base | {"dyn_orth_at_input": True, "dyn_orth_at_output": True}), + ] + out = [] + for name, cfg in specs: + nx, nio, L = cfg["state_size"], cfg["input_size"], cfg["num_layers"] + dyn = cfg["dyn_state_multiplier"] * nx + inputs = { + "carry": { + "rens": _randn(rng, L, BATCH, nx), + "dyn_in": _randn(rng, BATCH, dyn) if cfg.get("dyn_orth_at_input") else None, + "dyn_out": _randn(rng, BATCH, dyn) if cfg.get("dyn_orth_at_output") else None, + }, + "u": _randn(rng, BATCH, nio), + } + out.append(_make_case("ren_composition", name, cfg, rng, inputs, _carry_u)) + return out + + +GENERATORS = { + "unitary": ("Cayley-parameterised orthogonal layer.", gen_unitary), + "monlipnet": ("Monotone Lipschitz network.", gen_monlipnet), + "bilipnet": ("Bi-Lipschitz network (orthogonal + monotone layers).", gen_bilipnet), + "plnet": ("Polyak-Lojasiewicz network (quadratic potential of a BiLipNet).", gen_plnet), + "lbdn": ("Lipschitz-bounded deep network (JAX only).", gen_lbdn), + "dyn_unitary": ("Dynamic (stateful) orthogonal layer.", gen_dyn_unitary), + "contracting_ren": ("Contracting REN (JAX only).", gen_contracting_ren), + "lipschitz_ren": ("Lipschitz-bounded REN (JAX only).", gen_lipschitz_ren), + "general_ren": ("QSR-constrained REN (JAX only).", gen_general_ren), + "bilipschitz_ren": ("Bi-Lipschitz REN.", gen_bilipschitz_ren), + "r2dn": ("Contracting R2DN (JAX only).", gen_r2dn), + "ren_composition": ("Composition of bi-Lipschitz RENs and orthogonal layers.", gen_ren_composition), +} +LINEAR = ("contracting_linear_ren", "lipschitz_linear_ren", "general_linear_ren") + + +def main(names): + todo = names or [*GENERATORS, *LINEAR] + linear = None + for name in todo: + if name in LINEAR: + linear = linear or gen_linear_ren() + golden.dump(name, f"Linear REN ({name}); JAX only.", linear[name]) + elif name in GENERATORS: + description, fn = GENERATORS[name] + golden.dump(name, description, fn()) + else: + raise SystemExit(f"unknown network '{name}'") + print(f"wrote tests/golden/{name}.json") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/golden.py b/tests/golden.py new file mode 100644 index 0000000..1140b6f --- /dev/null +++ b/tests/golden.py @@ -0,0 +1,131 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Backend-agnostic loader for the golden input/output files in `tests/golden/`. + +Each `tests/golden/.json` holds a list of cases: + + { + "name": human readable case id, + "config": constructor arguments, using the JAX argument names, + "params": direct (unconstrained) parameters, in the JAX/Flax layout, + "inputs": model inputs, + "outputs": expected outputs + } + +The JAX tests load `params` as-is; the PyTorch tests translate them to +a `state_dict` (see `torch_utils.py`). Because both backends are checked against +the same file, passing both implies they agree with each other. + +Regenerate the files with `uv run python tests/generate_golden.py`. +""" + +import json +from pathlib import Path + +import numpy as np + +GOLDEN_DIR = Path(__file__).parent / "golden" + +# Tolerances used when comparing against the stored outputs (float32). +ATOL = 1e-5 +RTOL = 1e-4 + + +def to_arrays(obj): + """Recursively turn (nested) lists of numbers into float32 arrays. + + Zero-size arrays are stored as `{"shape": [...]}` since an empty list + can't carry its shape. + """ + if isinstance(obj, dict): + if set(obj) == {"shape"}: + return np.zeros(obj["shape"], dtype=np.float32) + return {k: to_arrays(v) for k, v in obj.items()} + if obj is None: + return None + return np.asarray(obj, dtype=np.float32) + + +def load(network): + """Return the list of golden cases for `network`.""" + with open(GOLDEN_DIR / f"{network}.json") as f: + data = json.load(f) + cases = data["cases"] + for case in cases: + for key in ("params", "inputs", "outputs"): + case[key] = to_arrays(case[key]) + return cases + + +def cases(network): + """Cases for `pytest.mark.parametrize`, e.g. + + @pytest.mark.parametrize("case", golden.cases("plnet"), ids=golden.case_id) + """ + return load(network) + + +def case_id(case): + return case["name"] + + +def assert_close(actual, expected, atol=ATOL, rtol=RTOL, path="outputs"): + """Compare (nested) outputs; `None` entries must match `None`.""" + if isinstance(expected, dict): + assert set(actual) == set(expected), f"{path}: keys {set(actual)} != {set(expected)}" + for k in expected: + assert_close(actual[k], expected[k], atol, rtol, f"{path}.{k}") + elif expected is None: + assert actual is None, f"{path}: expected None" + else: + assert actual is not None, f"{path}: got None" + np.testing.assert_allclose( + np.asarray(actual), expected, atol=atol, rtol=rtol, err_msg=path + ) + + +# --------------------------------------------------------------------------- +# Writing (used by generate_golden.py) +# --------------------------------------------------------------------------- +def _fmt_array(a): + a = np.asarray(a, dtype=np.float32) + if a.size == 0: + return json.dumps({"shape": list(a.shape)}) + if a.ndim == 0: + return np.format_float_positional(np.float32(a), unique=True, trim="-") + return "[" + ", ".join(_fmt_array(x) for x in a) + "]" + + +def _dump(obj, indent=0, key_is_data=False): + pad = " " * indent + if isinstance(obj, dict): + if not obj: + return "{}" + items = [ + f'{pad} {json.dumps(k)}: {_dump(v, indent + 1, key_is_data or k in DATA_KEYS)}' + for k, v in obj.items() + ] + return "{\n" + ",\n".join(items) + f"\n{pad}}}" + if obj is None: + return "null" + if isinstance(obj, (np.ndarray, list, tuple)) and key_is_data: + arr = np.asarray(obj) + if arr.dtype == object: # ragged / mixed list, recurse + return "[" + ", ".join(_dump(x, indent + 1, True) for x in obj) + "]" + return _fmt_array(arr) + if isinstance(obj, list) and obj and isinstance(obj[0], dict): + items = [f"{pad} {_dump(v, indent + 1)}" for v in obj] + return "[\n" + ",\n".join(items) + f"\n{pad}]" + return json.dumps(obj) + + +# top-level fields whose values are numeric data (written on one line per array) +DATA_KEYS = {"params", "inputs", "outputs"} + + +def dump(network, description, cases_): + """Serialise cases to `tests/golden/.json` (arrays inline).""" + GOLDEN_DIR.mkdir(exist_ok=True) + text = _dump({"network": network, "description": description, "cases": cases_}) + json.loads(text) # sanity check: must be valid JSON + (GOLDEN_DIR / f"{network}.json").write_text(text + "\n") diff --git a/tests/golden/bilipnet.json b/tests/golden/bilipnet.json new file mode 100644 index 0000000..fc1e7a0 --- /dev/null +++ b/tests/golden/bilipnet.json @@ -0,0 +1,168 @@ +{ + "network": "bilipnet", + "description": "Bi-Lipschitz network (orthogonal + monotone layers).", + "cases": [ + { + "name": "depth1_units4", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "units": [4], + "depth": 1 + }, + "params": { + "mon_0": { + "Fab0": [[0.03734774, 0.35051015, -0.59332395, -0.8656727], [0.063919045, -0.24245343, 0.30934945, 0.7966014], [-0.085690714, -0.26382276, -0.06793377, -0.7730607], [-0.23357339, 0.43705392, 1.190525, -0.37213483]], + "Fq": [[-0.40009505, -0.043084443, 0.50212497, 0.65005666], [0.54830086, 0.16142258, 0.42619026, 0.5167632], [-0.8620872, 0.20599416, -0.7405474, 0.02526337]], + "b0": [0.082933724, 0.21016347, -0.13343024, -0.32292178], + "by": [0.00783745, -0.015824193, 0.42167947], + "fab0": [2.098978], + "fq": [1.5383468] + }, + "uni_0": { + "W": [[-0.0101876855, -0.08466499, -0.62108517], [0.90952295, -0.85749674, 0.8287516], [-0.1007932, 0.36972108, 0.3318373]], + "a": [2.2712052], + "b": [0.14741048, -0.49165717, 0.018406054] + }, + "uni_1": { + "W": [[-0.45433268, 0.66909695, -0.89638275], [0.04498601, 0.27763692, 1.2582662], [-1.8616441, -0.9845546, -0.34991968]], + "a": [1.6829766], + "b": [0.47719207, -0.35736802, 0.106359586] + } + }, + "inputs": { + "x": [[2.040919, -2.555665, 0.41809884], [-0.5677696, -0.4526493, -0.21559717], [-2.0199862, -0.23193237, -0.8652131], [3.3229995, 0.22578661, -0.3526308]] + }, + "outputs": { + "y": [[-0.5477836, -0.94041896, -2.6743288], [-0.30573723, -2.2601619, -0.50969166], [-2.2564716, -6.9312577, 3.339264], [1.0604169, 0.89001167, -0.34130967]] + } + }, + { + "name": "depth2_units4x4", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "units": [4, 4], + "depth": 2 + }, + "params": { + "mon_0": { + "Fab0": [[-0.11832603, 0.9619458, -0.7148931, -0.92734325], [-0.17703776, -0.103501625, 0.19462664, 0.48181295], [-0.61197263, -0.72889334, 0.13260373, -0.3943746], [-0.11402646, 0.1797497, 0.69796884, -0.36424926]], + "Fab1": [[0.10391255, -0.7194061, -0.55272615, -0.60223687], [0.36771655, -0.7433944, 0.9624788, 1.0834225], [0.2859063, -0.63121974, -0.7630262, -0.5497146], [-0.31197348, -1.0738126, 0.7782326, -0.5686178], [-0.31757167, -0.8598043, -0.21064267, 0.22524142], [0.16724172, 0.99079937, -0.61713153, 0.5285913], [0.7636514, 0.2860642, 0.41291073, -0.18226475], [-0.01630795, 0.18242814, -0.8954761, -0.19414672]], + "Fq": [[-0.06358568, -0.36578375, 0.47696194, 0.73377895, 0.015737325, -0.28929377, 0.46246308, 0.9908066], [-0.08471983, -0.24684957, -0.7124401, 0.0189794, 0.5231277, 0.76135033, 0.29230115, 0.38058466], [0.49536788, 0.34737983, 0.06444332, -0.22113007, -0.78516287, 0.04595408, -0.6130222, -0.45826665]], + "b0": [-0.100049004, 0.2559325, -0.12197411, -0.04616121], + "b1": [0.24411552, 0.19343106, 0.5085623, -0.6271455], + "by": [0.2570577, -0.14468524, 0.040407546], + "fab0": [2.1372347], + "fab1": [3.2572901], + "fq": [2.3184273] + }, + "mon_1": { + "Fab0": [[1.1088806, 0.4584955, 0.16637021, -0.04776133], [-0.086203724, -0.46156436, 0.73293847, 0.32708842], [0.80245227, 0.43018842, 0.37042537, -0.6518352], [0.30787614, -0.2929175, -0.052875444, 0.027601553]], + "Fab1": [[0.42629042, -0.55861586, 0.5904022, 0.27652812], [0.5986296, -0.4897812, -0.34210926, 0.5256482], [0.33085844, 0.21440887, -0.21610601, -0.20282272], [0.35419315, 0.26519468, -0.4792626, -0.5622867], [0.8419069, 0.13965338, 1.019371, -0.12515694], [1.1538424, -0.6023085, -0.827274, 0.28301603], [0.54299587, 0.14828715, 0.34779707, -0.40420556], [-0.0946526, -0.3903088, -0.29351997, -0.5422246]], + "Fq": [[-0.7724052, -0.1337035, -0.22767025, 1.2388294, -0.23707464, 0.29930466, -0.53288233, 0.05826536], [-0.5557458, 0.04553713, -0.56052756, 0.0122357905, -0.17539337, 0.6251269, -0.2987721, 0.82156885], [-0.07576601, -0.6137148, 0.44639874, -0.18214777, -0.07741984, -0.03566289, 0.12614462, -0.60910285]], + "b0": [-0.23643269, -0.017241077, 0.68873364, -0.0534793], + "b1": [0.03824293, 0.1542136, -0.012037781, 0.68419826], + "by": [-0.15945749, 0.22326095, 0.048131395], + "fab0": [1.7493129], + "fab1": [1.9743465], + "fq": [3.039397] + }, + "uni_0": { + "W": [[-0.3801359, -0.012759099, -0.62380815], [0.51539683, -0.86298645, 1.1891503], [0.36002582, 0.663666, 1.2943523]], + "a": [1.9896883], + "b": [0.25095782, 0.17808098, -0.03017258] + }, + "uni_1": { + "W": [[0.05271904, 0.8276398, -0.21576113], [0.21247286, 0.28037632, 1.4815625], [-1.2774111, -0.8628355, -0.489245]], + "a": [2.4278612], + "b": [-0.5511595, 0.109753616, 0.3576792] + }, + "uni_2": { + "W": [[-1.0283253, 0.8941809, 0.13675909], [0.4093182, 0.21593204, -1.4330845], [0.877007, 0.09208387, -1.527992]], + "a": [3.2357554], + "b": [-0.024472078, 0.29839888, 0.013169107] + } + }, + "inputs": { + "x": [[-1.0484055, 1.405963, -0.021651229], [-0.37225056, -1.718185, 1.6818255], [0.7527786, 0.7535638, 1.1378813], [0.34922656, -0.6392466, -0.80024123]] + }, + "outputs": { + "y": [[1.5219538, 3.6983445, 0.54177845], [0.39646232, -0.28535262, -1.067722], [1.5371224, 3.2749834, -2.712789], [-1.2318777, 2.7357776, 0.18252575]] + } + }, + { + "name": "depth3_units3", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "units": [3], + "depth": 3 + }, + "params": { + "mon_0": { + "Fab0": [[0.59874815, 0.64590687, -0.65850157], [-1.2546848, -0.10101959, 0.06699303], [0.10613321, 1.1532266, -0.2403534]], + "Fq": [[-0.2631032, -0.5127159, 0.7319198], [0.40899566, -0.16540664, -0.1582856], [0.098403454, 0.053405643, 0.18658297]], + "b0": [-0.032124277, 0.87423784, 0.27790263], + "by": [-0.18995629, 0.23340221, 0.10704117], + "fab0": [1.3255183], + "fq": [1.754988] + }, + "mon_1": { + "Fab0": [[0.99409586, -0.94569296, 0.5283523], [-0.1914902, 0.06936842, -0.5368692], [-0.25664178, 0.48916417, -0.14969882]], + "Fq": [[-0.9209149, 0.7405354, -0.72897106], [1.1373606, 0.50223684, 0.59811103], [-0.91224045, 0.6872619, -0.7953192]], + "b0": [0.6998136, -0.10095766, -0.27444896], + "by": [0.22070612, -0.11914718, 0.08673485], + "fab0": [1.4111592], + "fq": [2.8170104] + }, + "mon_2": { + "Fab0": [[-0.036704004, 0.57774544, -1.0912789], [-0.3510343, 0.6226966, -1.434139], [-0.025765494, -0.61274546, -0.4720061]], + "Fq": [[0.843291, 0.14311019, 0.5204179], [0.7979484, 0.15494868, -0.17970096], [0.57906944, 0.95331675, -0.3346155]], + "b0": [-0.15565029, -0.07344822, 0.25763762], + "by": [-0.08069001, 0.32344317, 0.17420286], + "fab0": [2.0929291], + "fq": [1.7776256] + }, + "uni_0": { + "W": [[-0.3625594, -0.59581006, -0.25374407], [0.97102827, -0.98339057, 1.0601298], [-0.007436648, 0.18683162, 0.7972889]], + "a": [2.5978022], + "b": [0.04751828, -0.14774583, -0.39955577] + }, + "uni_1": { + "W": [[-0.21109496, 0.87472, -0.14755602], [0.1019779, 0.4936114, 1.3398991], [-0.9200096, -0.5984975, -0.10163018]], + "a": [2.1192431], + "b": [0.23250026, -0.23400761, 0.44986737] + }, + "uni_2": { + "W": [[-0.7149563, 1.0547512, -0.63650805], [0.7856516, 0.44912857, -1.0756166], [1.1824117, -0.23148935, -1.0866363]], + "a": [2.0085564], + "b": [-0.042581353, 0.30571187, 0.38319603] + }, + "uni_3": { + "W": [[0.2792319, 0.39444157, 0.5825593], [0.14791554, -0.7306355, -0.43871948], [0.11990547, -0.82521254, 0.2627818]], + "a": [1.0747428], + "b": [0.14475149, -0.32162756, 0.24048164] + } + }, + "inputs": { + "x": [[-2.2414815, -0.5726513, 0.1898536], [-1.1509497, -1.3474107, 0.36618584], [-0.649415, -1.6958272, -0.6841152], [0.86073756, -0.47374606, 0.93477255]] + }, + "outputs": { + "y": [[5.314756, 0.27612114, 0.9253231], [0.8685178, 0.24675316, 1.7983363], [1.3744571, 2.0771189, 2.818355], [-0.63558716, 0.6194762, 0.5988545]] + } + } + ] +} diff --git a/tests/golden/bilipschitz_ren.json b/tests/golden/bilipschitz_ren.json new file mode 100644 index 0000000..f960492 --- /dev/null +++ b/tests/golden/bilipschitz_ren.json @@ -0,0 +1,117 @@ +{ + "network": "bilipschitz_ren", + "description": "Bi-Lipschitz REN.", + "cases": [ + { + "name": "relu", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 3, + "mu": 0.5, + "nu": 4.0, + "init_method": "long_memory", + "activation": "relu" + }, + "params": { + "B2": [[0.33642292, -0.6356889, -0.9250491], [-0.16997755, 0.65901625, 0.4063617], [0.11434073, 0.92538327, 0.5147123], [-0.04214201, 0.795136, 0.2694033]], + "C2": [[-0.8858165, -1.1214674, -0.5358851, 0.8640333], [-0.049063653, 0.828202, 0.3139376, -0.97430146], [0.7091633, -0.49504158, -0.5421673, 1.1567049]], + "D12": [[0.18690905, 0.006079331, 0.30963027], [-1.5717698, -0.6653327, -0.06528831], [0.30951107, -0.15948302, -0.50243366], [-0.08908862, -0.6436414, 0.35391343], [-0.5991871, -0.80273414, 0.066374786]], + "D21": [[-0.016081974, -0.037310466, -0.09472743, -0.22030658, 0.41773918], [0.06349921, 0.6117089, 0.6655634, 0.0852406, 0.32417113], [0.16925898, -0.4305414, -0.36678427, 0.62448543, -0.7713495]], + "D22": [[-0.03744573, -0.19020487, -0.30193707], [0.516321, 0.3319826, -0.45109585], [-0.31573063, 0.17520362, -0.17650945]], + "X": [[1.004549, 0.24420348, 0.03863673, -0.038764913, -0.14207977, -0.025926784, 0.11600529, 0.52043, 0.31950095, 0.91231126, 0.27496943, 0.03514272, -0.44441494], [-0.031995453, 1.2984061, -0.17890884, 0.28079715, -0.09200488, 0.007946987, -0.3573449, 0.6210536, 0.13603117, 0.30252895, 0.81731653, -0.4390214, 0.36348256], [-0.2135594, 0.49561578, 0.9486694, 0.48735923, -0.21870805, -0.5291967, 0.20942716, -0.02041464, -0.0008161994, 0.31461233, -0.26464587, 0.568427, -0.350531], [-0.50465, -0.5425867, -0.38557172, 0.7077989, -0.13768055, -0.36430827, -0.0448998, 0.21937408, -0.07552179, 0.113891095, -0.28720227, 0.071143545, 1.0980191], [-0.21546103, -0.48457247, -0.08493748, -0.035310935, 1.7657384, -0.3090945, 1.0694495, -0.5883796, -0.07406379, -0.19704452, -0.16639875, -0.21211737, -0.069729745], [0.4318052, -0.10199454, 0.29640752, -0.08595441, -0.025629645, 0.2622682, 0.047537014, -0.89827746, 0.16309902, -0.24009551, -0.31254235, 0.5685443, -0.35075825], [-0.27406096, -0.86470443, -0.5288279, -0.00047255468, -0.4407434, -0.8803004, 0.6287138, -1.0061613, -0.8505858, 0.044757, -0.24444093, 0.76463133, -0.2292652], [0.0661629, -0.44067967, -0.24865612, -0.51669776, -0.0009797509, -0.18278083, -0.36125147, -0.6294763, -0.1851879, 0.17854276, 0.18420708, -0.08847854, 0.1434734], [0.20904416, 0.09124984, 0.36590165, -0.10533622, 0.0048559997, -0.5049754, 0.56633466, -0.27398047, 1.360977, 0.13829474, -0.2880798, -0.058622766, 0.15940583], [-0.07132421, -0.13246402, -0.13571149, -0.25123453, -0.0902289, 0.06952399, 0.09671915, 0.2869286, -0.02108549, 0.04676205, 0.21499097, 0.06979206, 0.26401713], [0.023260595, 0.12669623, 0.085421406, 0.186312, -0.31894425, -0.06050914, 0.0242865, 0.4801511, -0.049844366, 0.10783979, 0.22151294, -0.64038694, 0.34143424], [0.1730272, 0.34231043, -0.43145287, 0.05610576, 0.024243787, -0.49254304, 0.11054701, -0.20797363, 0.118102655, -0.5203945, -0.28920886, 0.052634344, -0.48004168], [-0.01111387, 0.3610281, -0.033254605, 0.15953639, 0.40944925, 0.13727383, -0.13769431, -0.5963906, 0.07764493, -0.123959064, -0.16223198, 0.18955374, 0.3507381]], + "X3": [[0.508197, 0.04614816, -0.12192626], [0.22820495, 0.9682465, 0.24967898], [-0.22212431, -0.14364375, 0.8137002]], + "Y1": [[0.9208988, -0.096973225, 0.32845968, -0.31713396], [-0.000080691876, 0.6280283, -0.45377877, -0.20185788], [-0.4731248, 0.026987059, 0.7570297, -0.18350463], [0.10938678, 0.43725124, -0.18270858, 0.39344835]], + "Y3": [[-0.14280355, -0.02598261, 0.31375706], [-0.03576233, -0.28525802, 0.25283068], [0.30128, -0.077108145, 0.066299774]], + "Z3": {"shape": [0, 3]}, + "bv": [0.08836336, 0.7508707, -0.19551636, -0.13912256, -0.04490513], + "bx": [0.19888057, -0.1029575, 0.13859338, 0.25427043], + "by": [0.57467556, -0.05835531, -0.10437452], + "p": [3.814292] + }, + "inputs": { + "state": [[-1.1033385, -0.72502464, -0.7818053, 0.26697585], [-0.24858072, 0.12648305, 0.84304255, 0.85793656], [0.47518364, -0.4507686, -0.7549323, -0.8148141], [-0.34385484, -0.051380094, -0.97227365, -1.1344875]], + "u": [[0.30570522, -1.851685, -0.17705351], [0.42582566, -0.9853556, -1.1129541], [-0.760626, 0.6480246, -0.12983136], [-1.8695972, -0.4233491, 1.0138968]] + }, + "outputs": { + "next_state": [[0.48254257, 0.01879488, 0.17731479, 0.7567346], [0.8357279, -0.7657709, 0.757183, 0.14599678], [-0.2976691, 0.35451245, 0.32662135, 0.50147605], [-0.84627044, 1.0595843, -0.7108051, 0.5409535]], + "y": [[4.243295, -3.1461217, -1.483656], [2.100142, -0.9225408, -3.562724], [-2.3815508, 2.5274243, -0.86837626], [-5.172107, 0.9620795, 0.6402812]] + } + }, + { + "name": "tanh", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 3, + "mu": 0.5, + "nu": 4.0, + "init_method": "long_memory", + "activation": "tanh" + }, + "params": { + "B2": [[-0.15923275, -1.2448946, -0.50660646], [0.4654666, 0.024811804, 0.66210836], [-0.21324928, 0.42649037, 0.44972873], [-0.5317499, 0.9386661, 0.18485472]], + "C2": [[-0.20692888, -0.761351, -0.61827916, 0.7872253], [0.68030477, 0.17221466, 0.020827383, -0.77432144], [0.8658, -0.14422044, -0.12218434, 1.077774]], + "D12": [[-0.2053124, 0.56320345, 0.062796034], [-0.8814967, -0.37566966, 0.25413576], [0.16741207, 0.07306096, -0.17951772], [0.07224625, -0.79364306, 0.26683474], [-0.05140513, 0.0395301, 0.5879085]], + "D21": [[-0.27238712, -0.3169273, -0.40094092, -0.08234471, 0.5723325], [-0.37919438, 0.9162915, 0.9432763, 0.05754128, 0.09462343], [0.46913216, -0.6952463, -0.25670055, 0.83539534, -0.9265443]], + "D22": [[0.41583702, -0.05809552, 0.44555202], [-0.1759091, -0.35025555, -0.6283351], [-0.26063195, 0.026565177, 0.28574413]], + "X": [[-0.029256105, -0.013570499, -0.29249483, 0.2102756, -0.15524879, 0.088164516, 0.5381171, 0.031285305, -0.052460354, 1.2641196, 0.08565176, -0.06206839, -0.48230922], [-0.108406864, 0.49933422, 0.05747584, 0.16824347, 0.0150857465, 0.54739183, -0.3447678, 0.118485995, 0.0076108146, -0.35647655, 1.0981588, 0.24304911, 0.33073363], [-0.054285638, 0.18582134, 0.79877347, 0.44015598, 0.53213775, 0.044013843, 0.065402865, -0.319576, 0.0055937185, -0.11100976, 0.06716002, 0.5361903, 0.52825075], [0.0567239, 0.3238828, 0.14813142, 0.91506577, 0.5748002, 0.33435538, -0.44153443, -0.0049231877, -0.067509, -0.23760994, 0.64665425, -0.36211482, 1.433231], [-0.5107922, 0.41088822, -0.15525635, -0.66760206, 1.4592524, 0.0002618283, 0.35537016, 0.50059116, -0.43271407, 0.34082067, 0.1800575, 0.53974426, -0.1137248], [0.26827496, -0.16854787, 0.25144485, 0.0803217, 0.24602892, 0.649294, 0.233186, -0.902166, 1.015939, -0.09370884, -0.046770923, 0.19986315, 0.16473679], [-0.7455783, -0.07705642, 0.26317278, -0.09936274, -0.5475217, -0.098033644, 0.77532756, -1.2565377, -0.8376671, -0.32515365, -0.055808682, 0.025922887, 0.5313025], [0.10637922, 0.3805658, 0.06596986, -0.33838457, -0.3443647, 0.23618893, -0.47758204, -0.44803348, -0.014930755, -0.24398725, -0.34408644, 0.491172, -0.2616297], [0.18061978, -0.0118677085, 0.03795291, 0.011211561, 0.29871714, 0.43989956, -0.2805449, 0.45317373, 1.2479409, -0.053369362, 0.20155768, -0.1646489, -0.27828726], [0.4488361, -0.17494501, -0.6346922, -0.9842672, 0.09262396, -0.047969725, 0.5122138, -0.3515649, -0.36510062, 0.24169147, 0.39529815, 0.4179119, 0.007974362], [-0.19402806, 0.038425744, 0.4523066, -0.024801446, -0.13412967, 0.13270566, 0.059605565, 0.03364162, 0.62000006, 0.200442, 0.338941, -0.102121115, -0.07800592], [0.15381971, 0.21886833, -0.18495059, -0.15284728, -0.07736097, -0.27565148, 0.32996508, -0.26410577, 0.04956555, 0.29767162, -0.28909042, 0.040633798, -0.037873454], [0.15439957, 0.059019476, 0.01974469, 0.37730157, -0.16567, -0.19724591, 0.19876765, 0.039932683, 0.2870513, 0.32754865, -0.3215675, -0.6174398, 0.38467038]], + "X3": [[0.9919995, 0.08899368, -0.26387846], [0.2433736, 0.9372391, 0.16930282], [0.27676687, 0.0381645, 0.97872686]], + "Y1": [[0.7995301, 0.80754715, -0.4016546, 0.21582492], [-0.4427978, 0.7598375, 0.4245942, -0.21289065], [0.32135025, -0.108878456, 1.2669175, -0.4171845], [-0.3342259, 0.43853143, 0.10256239, 0.6937722]], + "Y3": [[-0.14188135, -0.82848763, -0.095553465], [-0.39669597, -0.2904001, 0.44670326], [-0.16553967, -0.18975173, -0.33556253]], + "Z3": {"shape": [0, 3]}, + "bv": [0.19599296, 0.1600993, -0.25657877, -0.5012412, -0.069349214], + "bx": [0.031458907, -0.59993076, 0.33473545, -0.28041357], + "by": [0.021145955, -0.24198928, -0.23757194], + "p": [4.909608] + }, + "inputs": { + "state": [[-1.8766865, -1.2054423, 0.11256507, 0.11437992], [2.5298088, 1.1936291, 0.73095095, 1.4226058], [-0.67687297, -0.59703577, -0.7762026, -0.19285369], [2.101367, -0.2943777, -0.66988707, 0.00999973]], + "u": [[0.79965657, 0.44604862, 0.81994605], [-1.2211956, -0.88413507, -0.25311726], [0.07021187, 1.3314166, -1.2784076], [0.61818826, -0.24731757, 0.59048164]] + }, + "outputs": { + "next_state": [[-0.45128277, -0.3157692, 0.6161296, -0.27661633], [0.10133015, 0.12007588, -0.37957692, 0.3331908], [0.2526547, -1.1104926, 0.81127256, -0.031602636], [0.24328065, -0.18196738, 0.25801378, -0.73999625]], + "y": [[2.6874168, -1.8215699, 0.053436533], [-4.0338063, 0.9349375, 2.107602], [1.1876007, 2.4555357, -3.2716138], [1.6442709, -0.7144594, 2.3642614]] + } + }, + { + "name": "io1_relu", + "config": { + "input_size": 1, + "state_size": 4, + "features": 5, + "output_size": 1, + "mu": 0.5, + "nu": 4.0, + "init_method": "long_memory", + "activation": "relu" + }, + "params": { + "B2": [[0.22411086], [-0.65865886], [-1.0866686], [0.43844408]], + "C2": [[-1.082657, -1.1127398, -1.265932, 1.2006236]], + "D12": [[-0.04298705], [-0.29715043], [0.39837253], [-0.90347165], [-0.42632997]], + "D21": [[0.45172292, 0.5421226, -0.5239831, -0.59459263, 0.884521]], + "D22": [[0.20540148]], + "X": [[1.1051707, -0.13680178, -0.1254268, -0.22811076, 0.259565, -0.047862154, -0.3129946, 0.20700078, 0.13619035, 0.96779877, 0.055851128, -0.118134126, -0.18442738], [0.2474465, 0.6547263, -0.46831658, -0.40860635, 0.24663487, -0.33144712, -0.09553581, -0.22930995, -0.3184541, -0.065382615, 0.90084076, 0.5678788, -0.11592095], [0.062540464, 0.58838797, 1.4099855, 0.17403227, 0.25910214, -0.018433273, 0.24022953, 0.07380333, -0.25144494, 0.51027673, -0.0069224215, 1.1223762, -0.15594292], [-0.36543354, -0.5304301, -0.989247, 1.1740116, -0.52677166, -0.16621688, -0.008581795, -0.4524839, -0.16384539, 0.102781765, -0.08125088, 0.1609451, 0.17484885], [0.16554952, 0.24557142, -0.4105951, 0.21904142, 1.2479674, -0.2303774, 0.7799593, -0.27926308, 0.3025186, -0.22570106, 0.22337013, -0.031975426, 0.07338116], [-0.25529504, -0.020973215, -0.35677457, -0.21517223, -0.4299781, 0.9947998, -0.22730589, -0.33059442, 0.28504428, -0.6587106, 0.32002664, -0.12244283, 0.13426352], [0.0047224984, -0.24823903, -0.19166948, -0.33630106, 0.40918496, 0.18190488, 0.3708152, -0.6952417, -0.70620155, -0.502009, -0.2607891, 0.10332481, 0.06764066], [0.59586465, -0.018189175, 0.266038, -0.16262679, -0.0518689, -0.016101716, -0.036630146, 0.0706096, -0.27797523, -0.5063302, 0.18914145, -0.35929853, 0.22672325], [0.17358127, 0.18847665, -0.19589011, 0.068328336, 0.19807328, -0.17598668, -0.45842558, 0.49869353, 1.2284808, -0.16336678, -0.3139192, -0.38425907, -0.8176758], [-0.33720577, -0.33955956, -0.27580008, 0.56620276, -0.047682796, -0.13010429, 0.011216956, -0.24248469, 0.37965298, 0.59692717, -0.23321639, 0.29133016, 0.22258118], [0.790482, 0.37468648, -0.41962436, -0.12220858, 0.24248606, 0.17274836, 0.37576988, -0.04049644, 0.07864887, 0.4420692, 0.48365444, -0.006035709, 0.71902275], [-0.019925231, 0.05363747, -0.2526023, 0.08548441, -0.5121126, 0.4667109, -0.6587206, 0.2659066, -0.25134304, 0.1096159, 0.45548496, 0.2944134, 0.15119189], [-0.19129752, -0.092169404, 0.36581457, -0.21985756, 0.26349562, 0.057851117, -0.12040915, 0.28730196, -0.28428528, 0.09903725, -0.39717335, -0.076756984, -0.17168556]], + "X3": [[1.0535522]], + "Y1": [[1.2682655, 0.30046114, 0.12868124, 0.06879859], [-0.13800275, 0.88416356, -0.28700563, 0.61347073], [-0.030698372, -0.046331853, 1.0367707, -0.25099635], [0.48490658, -0.14046456, -0.29935038, 1.5281669]], + "Y3": [[0.35465032]], + "Z3": {"shape": [0, 1]}, + "bv": [0.19567138, -0.14578049, 0.42985752, -0.0014573426, 0.075901456], + "bx": [0.33962196, 0.04664512, -0.14280598, -0.6520513], + "by": [0.33346117], + "p": [2.600787] + }, + "inputs": { + "state": [[0.06294964, -1.542722, 0.30980766, 0.79004025], [-0.20270541, 1.0401038, -0.955193, 0.9132055], [0.18542276, 0.13308544, 1.333892, 0.66369325], [-0.34611946, 0.47260106, -0.7047358, 0.47426066]], + "u": [[1.279131], [1.179546], [-0.7334139], [-0.2683357]] + }, + "outputs": { + "next_state": [[1.4879332, -1.4933126, -0.41845787, -1.0794257], [1.221838, -0.2927904, -0.37366688, -0.012672007], [0.35562873, -0.41552073, 0.13825195, -0.8982253], [0.96459246, -0.07096848, -0.96055216, -0.9484402]], + "y": [[2.477827], [2.362155], [-1.0120703], [-0.7325181]] + } + } + ] +} diff --git a/tests/golden/contracting_linear_ren.json b/tests/golden/contracting_linear_ren.json new file mode 100644 index 0000000..df36243 --- /dev/null +++ b/tests/golden/contracting_linear_ren.json @@ -0,0 +1,37 @@ +{ + "network": "contracting_linear_ren", + "description": "Linear REN (contracting_linear_ren); JAX only.", + "cases": [ + { + "name": "default", + "config": { + "input_size": 3, + "state_size": 4, + "features": 0, + "output_size": 2, + "init_method": "long_memory" + }, + "params": { + "B": [[-0.40643084, -0.81371015, -0.5844567], [0.31356484, 0.3850641, 0.57465315], [0.30647153, 0.37268883, 0.62300754], [0.18782051, 0.55151975, 0.1023193]], + "C": [[0.17301393, 0.4734859, 1.2520707, -1.1678638], [-0.7325173, -1.0047017, 0.14742345, -0.4863457]], + "D": [[0.15834126, -0.22163703, 0.41569415], [0.24657731, 0.18821296, 0.12051213]], + "X": [[1.2867008, -0.39959398, 0.18417889, 0.18083306, 0.43904376, 0.104109034, -0.07512641, 0.23445682], [-0.13171867, 0.99452776, 0.10285546, -0.26287854, 0.17957899, 0.92562264, 0.14774479, -0.15655127], [0.32586047, 0.18156059, 0.9465925, 0.18958716, 0.37792656, 0.5373527, 0.49304035, 0.26493955], [0.13952056, -0.028158236, -0.3019995, 1.3771566, -0.3785214, 0.17008366, 0.39056042, 0.497488], [-0.09075535, -0.39276507, 0.07321624, 0.45431253, 0.8527139, -0.53343433, -0.17248471, 0.21106353], [0.4738118, 0.1263632, -0.2238456, 0.089139454, -0.004985761, 0.22859731, -0.22034134, 0.11617793], [0.092363894, -0.027895238, -0.066506386, -0.38547498, -0.1458527, 0.3619349, 0.20466524, -0.4319092], [0.40033287, 0.15907963, 0.63242614, 0.018753652, -0.13841538, -0.43429342, 0.39715153, 0.9823035]], + "X3": [[0.7537195, -0.1941263], [0.17885664, 0.75089633]], + "Y1": [[0.9188325, -0.104951315, 0.05758767, 0.32844555], [0.006620277, 1.275673, -0.12596667, 0.098339565], [-0.6414677, -0.43498445, 1.2387741, -0.17704482], [0.17397448, 0.16270329, 0.39668366, 1.2435577]], + "Y3": [[0.30509743, -0.0335014], [-0.20948556, -0.21946765]], + "Z3": [[-0.14641319, -0.33894876]], + "bx": [-0.16423307, -0.027770717, 0.07548359, -0.10166724], + "by": [-0.57710487, -0.021684887], + "p": [2.9588182] + }, + "inputs": { + "state": [[0.034192767, 1.3597475, 1.2247211, -0.5103071], [-0.29796952, -0.5273842, 0.56972635, -0.056064438], [0.7468856, -1.8473248, 1.5665488, -0.09643216], [0.68037844, -0.13656634, -0.37909856, 0.46311015]], + "u": [[0.82451355, -0.20252988, -0.15278618], [0.6856986, -0.87034065, -1.5143836], [0.39498186, -0.67056584, -1.9203405], [-0.81405365, -0.46759754, -1.1932025]] + }, + "outputs": { + "next_state": [[-0.28419036, 0.92614347, 1.036893, -0.004936211], [1.4216729, -0.9111456, -0.8285185, -0.4710917], [1.9802758, -1.6291466, -0.52655864, 0.115429714], [1.4209849, -0.61480063, -0.96084964, -0.05471222]], + "y": [[2.3139691, -0.8373592], [-0.42759976, 0.6604712], [0.16437393, 1.3048244], [-2.0608287, -1.0965155]] + } + } + ] +} diff --git a/tests/golden/contracting_ren.json b/tests/golden/contracting_ren.json new file mode 100644 index 0000000..120d838 --- /dev/null +++ b/tests/golden/contracting_ren.json @@ -0,0 +1,112 @@ +{ + "network": "contracting_ren", + "description": "Contracting REN (JAX only).", + "cases": [ + { + "name": "relu", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "relu" + }, + "params": { + "B2": [[0.026758019, -0.7907088, -1.3126723], [0.24017847, 0.31458718, 0.21650384], [0.4095704, 0.22043158, 0.44985068], [0.14027481, 0.43852654, 0.31286338]], + "C2": [[-0.80765986, -1.0834486, -1.3056408, 0.89567786], [0.52742743, 0.1266655, 0.5745806, -0.6650368]], + "D12": [[0.09791289, 0.72390705, 0.3613394], [-1.2738074, -0.2597877, 0.23507464], [0.17812735, 0.24492225, -0.22337373], [0.4268126, -0.17689666, -0.051291242], [-0.10324488, -0.3176127, 0.19376817]], + "D21": [[-0.23549226, 0.11449447, -0.39964437, -0.29583067, 1.0624789], [-0.1927285, 0.30135256, 1.1669192, 0.022220194, 0.28290212]], + "D22": [[-0.02918608, 0.37710452, 0.20682117], [-0.09816403, -0.11057278, -0.075058624]], + "X": [[1.4570589, -0.1284075, -0.09110412, 0.10577673, -0.036231134, -0.05918527, -0.33422017, -0.0034564405, -0.13307437, 1.3219837, 0.19592656, -0.0072430843, 0.20051432], [-0.101960875, 1.315638, -0.0016198682, 0.17501472, -0.38726798, 0.10400402, -0.50646126, -0.6105987, -0.09134307, -0.26997828, 1.0023451, 0.67342705, -0.24951695], [-0.18718308, 0.061621185, 1.147904, -0.052921824, -0.061779104, 0.2107389, 0.1559723, -0.31010276, -0.023754397, 0.010586055, -0.3163454, 1.0347679, -0.25738695], [0.29162002, 0.057823773, 0.026791947, 0.8226915, -0.03558295, -0.5993239, -0.33942226, 0.10885194, -0.63857013, 0.25398257, -0.523829, 0.22702156, 0.72312903], [0.23369734, 0.039285365, -0.4610505, 0.37474462, 1.8076574, -0.1660647, 0.541448, 0.12588635, -0.22653273, 0.32957605, -0.16286758, -0.015357125, -0.23798893], [-0.18782194, -0.38331756, 0.37712082, -0.046226274, 0.2897765, 1.0293235, -0.12829718, -1.1174896, 0.391082, 0.00238773, -0.11258006, -0.08997652, -0.41357243], [-0.24205378, 0.49621728, -0.20136999, -0.31622815, 0.1011979, 0.4221817, 0.3418869, -1.0352534, -1.0551485, -0.5283059, 0.22047801, -0.007033176, 0.021432554], [-0.22569345, 0.13643526, -0.16178921, -0.042870965, -0.33247823, -0.36483082, 0.40065956, -0.124120064, -0.19679487, -0.010137131, -0.13234358, -0.1523883, 0.18902479], [-0.09056029, -0.045433097, 0.0066664675, 0.3529525, 0.20415331, 0.11478009, -0.16907142, -0.41459063, 1.5996192, 0.28993416, -0.042212524, 0.16256516, 0.23443292], [0.24935535, 0.27641505, -0.1366853, 0.45449188, -0.37397796, 0.25851694, 0.14817962, 0.26208577, 0.5637025, 0.6797126, -0.34355307, -0.5066015, 0.24506672], [-0.30450374, -0.003721617, 0.25191826, -0.49313924, -0.6329941, 0.07778971, 0.013315768, -0.07374092, 0.01156043, -0.2581547, -0.15148482, -0.049996458, -0.29151264], [-0.49304444, 0.15170434, -0.01841959, 0.12195856, -0.29678848, -0.19741763, -0.29971293, -0.26599258, 0.058622375, -0.23489238, 0.10681989, 0.39262077, 0.60754836], [-0.41783673, 0.2663707, -0.02684639, -0.004208919, -0.43495914, -0.13805816, 0.22295919, -0.024743514, 0.02431631, -0.08721501, 0.34637094, -0.0064417706, -0.44587117]], + "X3": [[0.7923782, -0.590639], [-0.97543156, 0.8409654]], + "Y1": [[1.400068, 0.014135972, -0.35176373, -0.28220996], [0.339184, 1.047288, 0.0143997725, -0.016038537], [0.011520079, 0.24162172, 1.1657702, 0.064711414], [-0.31286052, 0.15333264, -0.20527413, 1.3281537]], + "Y3": [[-0.38131526, -0.041286293], [-0.002207486, -0.39739367]], + "Z3": [[0.5165915, 0.43812206]], + "bv": [-0.13907513, 0.23151636, 0.113602825, -0.78406787, 0.075119406], + "bx": [-0.018403225, 0.024965206, -0.3230625, -0.08080412], + "by": [-0.05347763, 0.3564283], + "p": [4.3896728] + }, + "inputs": { + "state": [[0.0012301534, 0.29874554, -0.27413785, -0.89059186], [-0.4546708, -0.9916465, 0.0601436, 1.3402152], [-0.4922065, -0.6204749, 0.48984206, 0.356887], [0.10541425, -0.930468, -0.029251823, 0.6953032]], + "u": [[-1.3442146, -0.45761576, -1.9012227], [-1.2895378, -1.841735, -0.23509113], [-1.2674465, 0.27126434, 0.15675108], [-0.18693094, -2.5167596, -0.5386929]] + }, + "outputs": { + "next_state": [[1.4985882, -0.98900515, -2.2132018, -3.5303466], [1.2074958, -2.730378, -2.4138913, -3.2000234], [-0.57029957, -1.1968597, -1.0995083, -1.2361028], [2.0064394, -1.8976061, -1.4963938, -2.9225063]], + "y": [[-1.5327226, 1.8797711], [1.4041296, 0.8570286], [0.7452614, 0.9633362], [0.4138596, 0.7614733]] + } + }, + { + "name": "tanh", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "tanh" + }, + "params": { + "B2": [[-0.025002006, -0.79182315, -1.3315347], [0.31288493, 0.35182425, 0.7245306], [-0.13987884, 0.635806, 0.9169384], [-0.21912104, 0.4331338, 0.40380365]], + "C2": [[-0.8414078, -1.4006681, -0.7998482, 1.4774907], [0.04234682, 0.5299461, 0.0032861829, -0.6051165]], + "D12": [[-0.08373886, -0.20829718, 0.5165617], [-1.1856568, 0.0422647, 0.51937515], [0.31255993, 0.20606661, 0.38225678], [0.1676198, -0.7863552, -0.2545619], [-0.15167435, 0.2651229, 0.44346628]], + "D21": [[-0.16196126, 0.03167233, -0.49203697, -0.47777942, 0.6573189], [0.26866606, 0.6287671, 0.8832161, 0.607894, 0.4838308]], + "D22": [[-0.025190912, 0.15105626, 0.5612627], [0.1775917, 0.016743165, -0.5058356]], + "X": [[1.1163871, -0.58400357, -0.42271027, 0.2563918, 0.21187052, -0.044981703, -0.51300335, -0.11140456, -0.20362149, 1.1631976, 0.67731917, 0.06507915, -0.23379335], [-0.35116547, 0.9831717, -0.05303796, -0.34545577, 0.03490657, -0.34527406, 0.3336322, 0.31879517, 0.3254254, -0.14221515, 1.1074855, -0.03962085, -0.11664362], [-0.101743706, -0.3899146, 0.5668409, 0.23829445, -0.057370625, 0.06492746, 0.30051315, -0.5199398, -0.23523904, 0.052600455, 0.11762844, 0.8436938, 0.3087538], [0.06311848, -0.36401466, -0.27922913, 1.2416413, 0.13914944, -0.56971836, 0.4043137, 0.17940812, 0.40300292, -0.11511027, -0.088711865, -0.33793715, 1.7378571], [-0.05263627, 0.47625998, -0.19418775, 0.04915236, 0.8737394, -0.26118338, 0.9187494, -0.20167671, 0.38768128, 0.101171345, -0.31318527, -0.1504792, -0.1377197], [-0.014855802, -0.1608432, -0.24818638, -0.09137631, -0.3080669, 0.63846815, 0.06557095, -0.75462174, 0.10033974, 0.0010524455, -0.19498686, -0.29314363, 0.2560313], [-0.15545091, 0.44949052, -0.2339518, 0.11595061, -0.068184905, -0.22620656, 0.9543967, -1.0191916, -0.6845696, -0.014187452, -0.3257449, -0.030620774, 0.015586552], [0.2875389, -0.27188072, -0.011798956, -0.51656365, 0.19544794, -0.32444492, -0.5419079, 0.0102462955, 0.047406524, -0.4573341, -0.32641068, -0.22298247, -0.338849], [0.11382837, -0.24221024, -0.21645443, 0.17499164, -0.2266544, 0.12983403, -0.29141557, -0.36364156, 0.7641254, 0.5583609, -0.09607772, 0.07318001, -0.009314853], [0.04798853, 0.0149059445, 0.5724651, -0.311678, -0.46723863, -0.3035879, -0.40041274, 0.22408862, 0.24611346, -0.05401574, -0.41713077, -0.106439, 0.41733718], [-0.84587085, 0.15798804, -0.32272658, 0.31211016, -0.3233772, -0.085624754, -0.45188376, -0.29317737, 0.41575176, 0.24617362, 0.18206307, -0.2610517, -0.5681422], [-0.118097015, -0.009270976, -0.02521631, -0.028137578, -0.3365429, -0.019882165, -0.011602014, 0.38716862, 0.56002, -0.04109671, -0.22989179, 0.27119935, -0.18228638], [-0.22273014, -0.017595045, -0.31299135, 0.18183203, -0.031174665, 0.0750025, -0.054882012, -0.21817635, -0.2843879, -0.0711828, -0.16462812, 0.0701704, 0.21292402]], + "X3": [[0.591308, 0.02013639], [-0.40285817, 0.8150567]], + "Y1": [[0.91169393, -0.622582, 0.027452236, 0.045295008], [-0.04740726, 0.8727048, -0.1120811, -0.29296187], [-0.08091012, -0.16578355, 1.0275056, -0.36124086], [0.070676975, 0.04296567, -0.042468853, 0.86823857]], + "Y3": [[0.16570199, -0.49938053], [0.138136, 0.07291124]], + "Z3": [[0.0850129, 0.11499378]], + "bv": [-0.19609332, -0.07783802, 0.19111751, 0.12919177, 0.06201383], + "bx": [-0.45429608, 0.16136613, 0.3508413, 0.30290145], + "by": [0.070163414, -0.4673036], + "p": [4.957384] + }, + "inputs": { + "state": [[-0.0055550304, 1.52897, -0.5552479, -0.3894303], [-1.816755, 1.5691059, 0.96433234, 0.91684806], [0.66889846, 0.11014859, 0.21548894, -0.2520066], [-0.20360042, 0.054303613, 1.5118295, 0.5556879]], + "u": [[-0.058460128, -0.5793921, -0.6349985], [1.602705, 0.5066873, 0.067550555], [-0.34618184, -1.1090534, -0.066861555], [0.87365806, -0.39253724, -0.22724259]] + }, + "outputs": { + "next_state": [[-0.29385263, -0.09952763, -1.6582649, -0.9779785], [-1.80084, 0.5741693, 0.18227041, 0.0817572], [-0.1297926, -0.021969363, -0.16755894, 0.05525598], [-0.4554864, 0.03028597, 0.51934636, 0.5566797]], + "y": [[-2.68624, -0.63646924], [0.32280135, -2.1773343], [-2.0535784, 0.6050209], [-0.24707706, -1.7721435]] + } + }, + { + "name": "tanh_no_polar", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "tanh", + "do_polar_param": false + }, + "params": { + "B2": [[-0.45052952, -0.65049994, -0.8701736], [0.47602734, 0.098837554, 0.34962082], [-0.088656366, 0.20342836, -0.21689731], [-0.22550419, 0.88279796, 0.4606733]], + "C2": [[-1.0210831, -1.107046, -0.711084, 0.044710398], [0.082419455, 0.753769, 0.5214044, -0.19062394]], + "D12": [[0.6308677, 0.21755096, 0.22325428], [-0.67793715, -0.44394067, 0.050011322], [0.5066337, 0.6272828, -0.2511987], [0.2121181, -0.54890937, 0.55436856], [-0.17328045, 0.2621873, -0.13440955]], + "D21": [[0.06485522, 0.2288354, -0.10483387, -0.25188172, 0.26607898], [-0.070205435, 0.640801, 0.77524006, 0.16321781, 0.7333971]], + "D22": [[0.14819103, 0.14795326, -0.14263372], [0.30868155, -0.071988, 0.32893062]], + "X": [[0.72650045, -0.25624064, 0.061398968, -0.21069965, 0.20263532, 0.07802633, -0.27689138, 0.02185968, -0.10538331, 1.2468952, -0.1897659, -0.13175194, 0.3633711], [0.67157805, 1.5997026, 0.018966904, 0.065656416, 0.46003762, -0.037330445, -0.29290473, 0.035062682, 0.13545504, -0.24874489, 0.45925993, -0.4310189, 0.1996288], [-0.22751297, -0.04240413, 1.0637677, 0.18574226, -0.10047084, 0.14961472, -0.2670181, -0.10851421, -0.3073247, 0.3395302, -0.008124969, 0.73502386, -0.10571155], [-0.066518545, 0.21015595, -0.47869086, 0.68883824, -0.11343218, 0.7596918, 0.28670368, -0.033435505, 0.21359989, 0.6172309, -0.07012318, -0.10992401, 1.3403612], [0.14825265, 0.20139995, -0.15246673, 0.577261, 1.8880224, 0.023461118, 0.8289099, -0.43433183, 0.25732595, -0.05823258, 0.13017172, 0.20473783, -0.10238753], [-0.5071446, 0.11036237, -0.2224151, -0.09905823, -0.18135296, 0.9228538, -0.6128324, -0.6543999, 0.63515514, 0.3334198, 0.5939526, 0.006777937, -0.5407309], [-0.26816607, -0.36207137, -0.15056239, 0.023894396, -0.6006562, 0.10274266, 0.3249872, -0.88348734, -0.8983801, -0.094089374, -0.021912917, -0.1619334, -0.18374145], [-0.50482696, -0.008888195, 0.55366015, 0.5941486, 0.39654532, 0.2117405, -0.20294404, 0.46083662, -0.30113727, -0.020662652, -0.087368146, 0.02758857, -0.13056448], [-0.025134651, -0.32537976, -0.11209911, 0.68416697, -0.020878209, -0.07159817, 0.15971228, 0.21243656, 0.98062956, -0.06214764, 0.27548277, 0.08095129, 0.036586214], [0.46620533, -0.20333184, 0.02482377, -0.15601762, 0.4470925, -0.5855582, -0.20104252, -0.1587471, 0.1990619, 0.4162616, 0.4185583, -0.47220984, 0.22514251], [-0.08798396, -0.19968757, 0.16187057, -0.2760419, -0.62307405, -0.11106555, -0.44933128, -0.19467476, 0.11146643, 0.09359992, 0.77863336, -0.059896424, -0.4599995], [-0.22682907, -0.27581528, -0.36534756, 0.13061017, -0.1935697, -0.5932118, 0.20907684, -0.033427972, 0.107051186, 0.031716466, 0.18949643, 0.30209962, 0.37086076], [0.12749484, 0.11762029, 0.12296008, -0.43699133, -0.049390726, -0.077794954, 0.06226402, -0.40760827, 0.49023443, 0.0313166, -0.36339527, -0.5126661, 0.12986735]], + "X3": [[0.9730974, -0.21548763], [0.02763415, 0.8076473]], + "Y1": [[1.1654975, -0.21736257, -0.011550067, 0.29365176], [0.7715007, 0.69770706, -0.13935234, -0.2519507], [0.23529851, -0.3444298, 0.8546929, -0.008880385], [-0.29360357, -0.28719753, -0.14268687, 0.36986792]], + "Y3": [[-0.4336435, -0.123910226], [0.044469193, -0.055728]], + "Z3": [[-0.5321906, -0.1391359]], + "bv": [0.23953131, 0.16675967, -0.023624359, -0.26621097, 0.18936491], + "bx": [-0.17367871, -0.35077456, -0.24065588, 0.43450361], + "by": [0.06605402, 0.34777412], + "p": [3.7303498] + }, + "inputs": { + "state": [[-0.14725463, -2.5325196, 0.3772074, -1.4921718], [-1.2964406, -0.63495886, 1.2725912, -0.37084535], [0.2709916, 1.747968, 1.5940298, -0.10335342], [-0.24152102, -1.2608706, -0.69445807, 0.42535836]], + "u": [[0.39573076, 0.11023823, 0.99480164], [-0.772368, -0.05607652, 0.731243], [0.5841459, 1.0709457, 0.3970204], [-0.30940256, 0.36219227, -1.0025952]] + }, + "outputs": { + "next_state": [[0.5530293, -0.96796894, 0.3034351, -0.17446661], [0.21751799, -0.5216813, 0.72977597, -0.28488886], [-1.5931062, 0.85106134, -0.58930266, 0.85466367], [1.0132471, -0.854463, 0.2270356, 0.15109885]], + "y": [[2.8379736, 0.30182728], [1.3087995, 0.90651894], [-2.9533815, 3.5327277], [2.7288127, -2.2757971]] + } + } + ] +} diff --git a/tests/golden/dyn_unitary.json b/tests/golden/dyn_unitary.json new file mode 100644 index 0000000..ebda283 --- /dev/null +++ b/tests/golden/dyn_unitary.json @@ -0,0 +1,46 @@ +{ + "network": "dyn_unitary", + "description": "Dynamic (stateful) orthogonal layer.", + "cases": [ + { + "name": "io3_nx4", + "config": { + "input_size": 3, + "state_size": 4 + }, + "params": { + "X": [[0.39373326, -0.50505185, -0.5531293, 0.3524349, 0.72835535, 0.3011747, 0.39470622], [0.19198796, 0.27993107, -0.29867455, 0.6431416, 0.27259937, -0.61207885, 0.80388784], [0.7063238, -0.76507795, 0.28256634, 0.7708328, 0.58943015, -0.46604055, 0.23297901], [-0.7299789, 0.2215861, -0.06341612, 0.42369628, -0.4694703, -0.63670826, -0.0050984025], [0.36433977, 0.36942697, -0.5127543, 0.26382568, 0.71618605, -0.2068526, 0.11735915], [1.0137616, -0.7942073, -0.3136886, 0.24764055, -0.18754366, 0.5843144, -0.22735517], [-0.83704877, 0.31796017, 0.45483637, 0.16172801, 0.16314834, -0.22094613, 0.29020068]], + "bx": [-0.09127059, -0.117596425, 0.38036627, -0.44870815], + "by": [-0.12420715, 0.3547382, -0.19750237] + }, + "inputs": { + "state": [[1.0531157, 1.7764913, -2.5532918, -0.13796507], [1.0137194, 1.3521419, 0.6537884, 1.4971179], [0.28995758, 0.55126715, 0.17873769, -1.0738587], [-0.84662896, 0.37958425, -0.5801952, 1.2715514]], + "u": [[1.2923867, 1.7987863, -0.026073838], [1.3837098, -0.90584314, -0.81631476], [0.08130305, 0.28143084, -1.5989883], [-1.7307242, 0.3550784, -0.8621881]] + }, + "outputs": { + "next_state": [[-1.0538156, 0.74098283, 0.28301632, 0.83291984], [-1.1200949, 1.4277985, -0.5328393, -0.07401991], [1.1852947, 0.22852525, -0.6795257, 0.028841168], [1.3147019, -0.45008406, 1.627536, -0.9113271]], + "y": [[-2.666546, 1.9968336, 1.6008909], [-0.76470107, -1.6460655, 0.1054548], [-0.11321492, 0.3396525, 0.8581854], [-1.5750057, -0.44309863, -0.48223856]] + } + }, + { + "name": "io2_nx6", + "config": { + "input_size": 2, + "state_size": 6 + }, + "params": { + "X": [[0.073493026, -0.9193951, -0.54167736, 0.06892595, 0.5529869, 0.22177991, 0.37468666, 0.57515496], [0.69667476, -0.5700069, -0.1119968, 0.36634606, -0.5037223, 0.3095407, 0.8524367, -0.6289638], [0.42948043, 0.39759967, 0.73473525, -0.90141726, 0.70760715, -0.23407263, 0.31714153, -0.24761638], [0.8444872, -0.38875684, -0.108996995, 0.46395224, 0.23807487, -0.0016897768, -0.39194834, 0.12974316], [0.10354328, 0.48660398, -0.34584147, 0.45017907, 0.3403391, -0.7001407, 0.99031633, -0.09829527], [0.4272046, -0.48887706, -0.94657445, 0.31240064, 0.6668283, -0.5229482, 0.453217, -0.5560013], [0.83070385, 0.63510877, -0.53755677, -0.1836647, 0.11522816, 0.17988907, 0.93195254, -0.20889544], [-0.1569833, -0.14653315, 0.36321607, 0.46691427, 0.42553174, 0.2855024, -0.2720545, 0.5945095]], + "bx": [0.2983038, 0.05037208, 0.48242202, 0.31120643, -0.010422866, 0.55073947], + "by": [-0.3078576, 0.38127017] + }, + "inputs": { + "state": [[-1.2394711, 0.28284147, -0.29003042, 0.83475494, -0.9203901, 0.19091922], [0.6661273, 1.0357219, -1.1061453, 1.4237888, -0.90069664, -1.5125506], [-0.9545692, 0.15484929, -0.3217841, 0.44045562, 0.49560326, 0.3181445], [-0.2545279, 1.814078, -2.2160335, -1.734349, -0.50688595, -1.2552173]], + "u": [[-2.5559254, -0.7950685], [-0.7492611, -2.4485493], [0.85869294, 1.2991121], [1.2343011, -0.5083407]] + }, + "outputs": { + "next_state": [[1.3291188, 0.5213513, 1.5140901, -0.48109785, 1.0002521, 0.40816638], [1.0836813, -0.8636055, 1.8258196, -1.6555908, -0.45323998, 0.32775718], [0.73510486, -0.26462173, 0.9877031, 1.8452501, -0.30396876, 0.59994406], [-0.08660132, -0.70044935, 0.47505376, 1.2773885, -2.05618, 2.7330701]], + "y": [[-1.7036645, -1.7351239], [-2.9066138, 0.9276235], [0.69780624, 0.30351532], [-2.2798045, 1.0193428]] + } + } + ] +} diff --git a/tests/golden/general_linear_ren.json b/tests/golden/general_linear_ren.json new file mode 100644 index 0000000..64b7e22 --- /dev/null +++ b/tests/golden/general_linear_ren.json @@ -0,0 +1,40 @@ +{ + "network": "general_linear_ren", + "description": "Linear REN (general_linear_ren); JAX only.", + "cases": [ + { + "name": "default", + "config": { + "input_size": 3, + "state_size": 4, + "features": 0, + "output_size": 2, + "init_method": "long_memory", + "Q": [[-1.5975236892700195, 0.005036603659391403], [0.005036603659391403, -0.019526256248354912]], + "S": [[-0.1812286376953125, -0.6080449223518372], [-0.2231849879026413, -1.2062805891036987], [-0.6264480948448181, 0.39023715257644653]], + "R": [[-15.483796119689941, -35.3070182800293, 9.207743644714355], [-35.3070182800293, -71.1535873413086, 21.24281883239746], [9.207743644714355, 21.24281883239746, -5.065634727478027]] + }, + "params": { + "B": [[-0.40281877, -0.7217306, -0.53416854], [0.4505941, 0.49800166, 0.21748699], [-0.011539243, 0.77802956, 0.72685903], [-0.20368159, 0.23980537, 0.54857314]], + "C": [[-0.36624342, 0.23996569, 0.70197827, -0.55951], [-0.4632485, -0.61469525, 0.7493503, -0.7276551]], + "D": [[0.1606428, 0.42373812, -0.0110313725], [0.19007835, -0.037770957, 0.30856663]], + "X": [[1.199991, 0.26274586, 0.10452691, 0.4920012, 0.8610141, -0.10024983, -0.177558, 0.18328881], [-0.18672009, 0.8066402, 0.21824245, 0.34862876, 0.17230545, 0.1521805, -0.28608975, -0.32204685], [-0.36225337, -0.1223303, 1.545263, -0.09221957, -0.22063375, 0.20964684, 0.9858648, 0.03620434], [-0.20600225, -0.0004513401, 0.13036115, 0.42660797, -0.48306733, 0.32536817, 0.25290138, 0.82539177], [-0.3060612, -0.055663988, -0.3580957, -0.1020291, 0.7409772, 0.039621554, 0.34988657, -0.06678616], [0.11082467, 0.41574916, -0.40078127, -0.104243405, -0.047069553, 0.3743139, 0.46610004, 0.20565812], [-0.4493, -0.081291616, -0.26809305, -0.27571413, 0.60744834, -0.3188912, 0.33275127, -0.21456368], [0.25829428, 0.106110975, 0.038228314, -0.02285014, 0.3413151, -0.27950996, -0.2180815, -0.12353177]], + "X3": [[1.4003639, -0.088069916], [-0.31122023, 0.87427795]], + "Y1": [[1.7498941, -0.03224189, 0.2550062, 0.13955456], [-0.0037149847, 0.7921885, -0.10028858, -0.10633572], [-0.6627829, 0.7770397, 1.2774509, 0.034885954], [-0.061004747, -0.23689331, 0.28488654, 0.9439432]], + "Y3": [[0.14378177, -0.41624856], [0.11354297, 0.6000904]], + "Z3": [[-0.08688866, 0.2399865]], + "bx": [0.010600092, 0.33155715, 0.32022738, 0.018493913], + "by": [-0.16787565, 0.08602625], + "p": [2.852154] + }, + "inputs": { + "state": [[1.8267566, -3.078332, 0.95806396, 0.069637224], [1.3182501, 0.38562924, 1.8272586, 0.031743757], [-0.51622945, 0.5804849, 0.43210685, -0.35683936], [-0.24730383, 0.7194407, 0.704316, -0.49393424]], + "u": [[-0.36771372, -1.8067904, 1.6792074], [-0.22429088, 1.3372774, 0.4174656], [1.9439628, 1.53711, 0.31829843], [1.4807634, -0.95012355, 1.2586181]] + }, + "outputs": { + "next_state": [[-0.19522381, -2.4605918, 1.3163987, 1.5431118], [-0.042069234, 0.8848173, 0.4161079, 0.20878488], [-0.36199534, 2.5644546, -0.37216347, -0.9236564], [-0.1430192, 1.3706954, -0.041852087, -0.44360745]], + "y": [[-0.17668486, 156.46951], [0.41203856, -66.60908], [-1.3120186, -149.60841], [-0.0057578683, 35.997276]] + } + } + ] +} diff --git a/tests/golden/general_ren.json b/tests/golden/general_ren.json new file mode 100644 index 0000000..6abe4ae --- /dev/null +++ b/tests/golden/general_ren.json @@ -0,0 +1,82 @@ +{ + "network": "general_ren", + "description": "QSR-constrained REN (JAX only).", + "cases": [ + { + "name": "tanh", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "tanh", + "Q": [[-3.304182529449463, 0.924851655960083], [0.924851655960083, -0.5022961497306824]], + "S": [[0.6362590789794922, 0.6433106660842896], [-0.602710485458374, 1.3340758085250854], [-0.7983688712120056, 0.13768619298934937]], + "R": [[-1.340086579322815, -2.851837635040283, -0.09609854221343994], [-2.851837396621704, -2.7442080974578857, 0.3692953288555145], [-0.09609848260879517, 0.36929526925086975, 2.867051601409912]] + }, + "params": { + "B2": [[0.2715905, -1.1760737, -0.69005895], [0.070266545, 0.057022452, 0.28102475], [-0.34787357, 0.62867546, 0.46610853], [0.027791798, 0.6409975, 0.24021626]], + "C2": [[-0.83236027, -0.78683645, -0.99306333, 0.64422166], [-0.17119008, 0.5242911, 0.09184399, -0.1366533]], + "D12": [[-0.063215315, -0.20069776, 0.04089378], [-0.7190167, -0.362572, -0.22530472], [-0.0073081404, -0.06518844, 0.088590324], [-0.06675628, -0.6358518, -0.03250873], [-0.066800445, -0.05947967, -0.28398603]], + "D21": [[0.8367248, 0.34516597, -0.49358526, -0.833657, 0.6410592], [0.21387488, 0.71829283, 0.9745345, 0.7651309, -0.06113231]], + "D22": [[-0.22571523, -0.3019673, 0.04937604], [0.13739973, -0.4398259, -0.050378602]], + "X": [[1.2176383, 0.2745234, -0.33461747, 0.16887689, 0.10151377, -0.13077761, 0.58890724, -0.3356155, -0.083212115, 1.1085846, 0.072934605, -0.051372807, 0.41306683], [0.4336035, 1.2738019, 0.3004937, -0.016479474, -0.34943867, 0.65633434, -0.2872566, 0.66878223, -0.03813179, 0.0695437, 0.9023833, 0.2874348, -0.15242347], [0.5529611, 0.16050853, 1.0523877, -0.17102157, 0.08416608, -0.00097406196, 0.5699947, 0.28083467, -0.46799538, 0.4587292, -0.7119535, 1.4190669, -0.5657926], [0.00023034407, 0.24370594, -0.0023188884, 0.6705801, 0.369981, -0.53387225, -0.43805742, 0.41517106, -0.25948378, 0.073995546, 0.24295853, -0.031944763, 1.0597277], [0.036191065, 0.064243376, 0.356115, 0.28026545, 1.4721893, 0.12891738, 0.46109363, -0.055451736, -0.23449677, 0.2352402, 0.4874199, 0.025500087, -0.5935835], [-0.15261053, 0.14039986, 0.08910631, 0.028114364, 0.29664388, 1.3033154, -0.32220885, -0.8890759, 0.33726078, 0.069849536, 0.37317598, 0.5494125, 0.20999932], [-0.15168592, -0.105377726, 0.15992175, -0.29638273, -0.02029794, 0.18147975, 1.053644, -1.0618159, -0.75459516, 0.408757, 0.091575466, -0.51491237, 0.06429758], [0.07542875, -0.12642847, 0.012894515, -0.4428913, 0.37382814, 0.12646766, -0.16305746, 0.09869525, -0.2445578, -0.25911263, -0.800155, -0.85423267, 0.6065436], [-0.33576393, -0.0883242, 0.17146073, -0.1085324, 0.35750905, -0.28460088, 0.16886543, -0.6261289, 1.5291971, 0.20631148, 0.11008826, 0.09035386, 0.08048732], [0.2592326, 0.2887358, -0.28331476, 0.31724942, -0.28427047, 0.049384262, -0.07581194, 0.14417061, -0.099585354, 0.35584232, -0.14365572, -0.28762388, 0.32033506], [0.1717646, 0.15418416, 0.2869538, -0.31792024, 0.2743283, -0.31697202, 0.11422378, -0.053633343, 0.08154827, 0.20921306, 0.3089676, -0.27618724, 0.27890483], [0.35313952, 0.20555188, 0.31101677, -0.5289275, 0.021634921, -0.034231223, -0.5862588, 0.032618858, 0.0009925949, -0.4519139, -0.2519914, 0.31621733, 0.18055521], [0.21226992, -0.3316133, -0.32308295, -0.1516456, -0.01497749, -0.17251775, -0.05757756, 0.21291928, 0.20533772, 0.041432984, -0.34237897, -0.14730464, 0.38338888]], + "X3": [[1.2610595, 0.31867978], [-0.31834045, 0.86071336]], + "Y1": [[0.6369884, -0.88243055, -0.3275933, -0.5312816], [0.4181338, 1.1422526, -0.19238874, 0.020257235], [0.124701776, 0.52550024, 1.2335287, 0.45840237], [0.14658737, 0.39184412, 0.43432865, 1.2808608]], + "Y3": [[0.011270734, 0.33018345], [-0.11622835, 0.43033922]], + "Z3": [[-0.08023598, -0.20215002]], + "bv": [0.5674286, -0.12116408, 0.051106047, -0.316925, -0.2077152], + "bx": [-0.09961026, -0.29446462, 0.36843875, -0.21508361], + "by": [-0.37450743, 0.3366258], + "p": [4.7369547] + }, + "inputs": { + "state": [[-0.80283695, 0.2428499, -1.6563455, 0.65610486], [1.143453, -0.452611, 0.43048576, 0.25093257], [-0.39435205, -0.8624049, -2.0325525, 1.4104235], [-0.047632236, 2.5223274, 0.82621825, 0.27778053]], + "u": [[-0.65743107, 1.3924692, -0.5062891], [1.5699492, -0.3983613, 0.18595393], [-1.5226704, 2.3432324, -0.09402583], [-0.3851136, 0.81084764, -0.89137185]] + }, + "outputs": { + "next_state": [[-0.38127217, 0.10859561, 0.15537235, 0.32492614], [0.7747818, -1.1220391, 0.51940155, -0.018674955], [-0.8693878, 0.34181592, 0.8678472, 1.3981798], [-0.05111628, 0.6955228, 0.60896456, 0.53042996]], + "y": [[0.55742425, 5.3850574], [0.5192508, 2.8514528], [1.1205169, 6.7631645], [-0.19214332, 2.661691]] + } + }, + { + "name": "relu", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "relu", + "Q": [[-3.304182529449463, 0.924851655960083], [0.924851655960083, -0.5022961497306824]], + "S": [[0.6362590789794922, 0.6433106660842896], [-0.602710485458374, 1.3340758085250854], [-0.7983688712120056, 0.13768619298934937]], + "R": [[-1.340086579322815, -2.851837635040283, -0.09609854221343994], [-2.851837396621704, -2.7442080974578857, 0.3692953288555145], [-0.09609848260879517, 0.36929526925086975, 2.867051601409912]] + }, + "params": { + "B2": [[0.046742972, -0.8003876, -0.7933947], [0.3912886, 0.9464419, 0.7110123], [0.5884269, 1.2307273, 0.70240587], [0.12660937, 1.1956939, 0.097237155]], + "C2": [[-0.92529947, -1.012545, -1.3917172, 1.1762455], [0.60709447, 0.22651094, 0.08564845, -0.41777623]], + "D12": [[0.70826465, -0.36212063, 0.30546498], [-0.9671798, -0.82912904, -0.058703788], [0.06919436, 0.07606474, -0.1887461], [0.14709157, -0.64616454, 0.10646334], [-0.008410513, 0.11263503, 0.4449801]], + "D21": [[-0.14586842, 0.051120132, -0.54174864, -0.13859886, 0.8624424], [-0.025743425, 0.19237712, 0.6707179, 0.6284669, 0.09059298]], + "D22": [[-0.0023867935, 0.40423667, -0.076977685], [0.256323, -0.45654073, 0.106789365]], + "X": [[1.0873055, -0.2770187, 0.4136169, -0.0020815919, -0.20673425, -0.088232875, 0.28565273, -0.4121513, 0.04458248, 0.68357337, 0.06794873, -0.15876326, -0.022192333], [0.22806293, 1.0126169, 0.16546318, 0.20960535, 0.12206109, 0.06781291, 0.012740224, 0.19123459, 0.3613328, 0.5525054, 1.4811974, -0.2979139, 0.08812758], [0.1467539, 0.36471486, 1.3190869, -0.42955568, 0.08268079, -0.5131965, -0.14239883, -0.27811182, -0.18371119, 0.47446018, -0.16294548, 0.80664784, 0.036786687], [0.32121143, -0.5108207, 0.4874717, 0.7704617, -0.29863837, 0.03788632, 0.20138912, 0.81159574, 0.26842198, -0.17894116, -0.06296956, -0.28861606, 1.1719664], [0.24674304, -0.10204763, 0.33780068, -0.5089318, 1.2290461, -0.42639673, 0.4437463, -0.32015055, 0.34943682, -0.28941426, -0.0432996, 0.35158652, 0.47923934], [0.35111845, -0.67371786, -0.09492469, -0.10925523, -0.2093206, 0.7857344, -0.090021186, -0.84697515, 0.41636935, -0.40831292, -0.4782499, 0.03930448, -0.52143127], [0.20699276, -0.15259655, -0.15524247, 0.06568333, -0.111221254, 0.21049707, 0.3892137, -1.5208719, -0.27720898, 0.049200103, -0.7778186, 0.50932765, -0.42708537], [0.61689156, 0.5454927, 0.26426536, 0.00050293526, -0.2628623, 0.28721085, -0.1203256, 0.35534313, -0.3682607, -0.47450757, -0.21098404, 0.018995425, -0.14753154], [0.042978365, -0.27743167, 0.4878127, -0.15658982, 0.15865415, -0.50247663, 0.25203162, 0.46877512, 1.4993215, -0.0702913, -0.27624437, 0.18656316, 0.1788235], [-0.3885446, 0.009306486, 0.16796458, -0.23039816, 0.18772309, -0.07665152, 0.44256938, -0.37812647, 0.16260949, 0.27506095, -0.5604915, -0.27562237, -0.25329298], [0.2851313, -0.10356728, 0.0019037555, 0.049155354, 0.78180236, -0.0678239, 0.5869344, -0.052418735, -0.4715814, -0.24239734, -0.06875461, -0.108587764, -0.43769658], [-0.2726136, 0.5784425, 0.07482319, 0.103380136, 0.5501019, -0.49469122, 0.17291355, -0.13950731, -0.0072933487, -0.2031744, 0.0380959, 0.66473126, 0.36475897], [0.076975435, 0.24978061, -0.35668224, -0.8776465, 0.31595504, 0.6140892, 0.14311296, -0.18296307, 0.016933788, 0.36987168, -0.3490397, 0.024312105, 0.40120992]], + "X3": [[0.69307566, 0.631663], [0.24725494, 1.502111]], + "Y1": [[0.6777103, 0.06742727, 0.4630693, 0.13753562], [0.3815761, 0.44535196, -0.024612824, -0.5986802], [-0.14893617, 0.0887625, 0.9675851, -0.37755305], [0.09075388, 0.19786121, -0.028089236, 1.245144]], + "Y3": [[0.34436318, 0.37415743], [-0.100722365, 0.1558669]], + "Z3": [[-0.19074604, -0.0253688]], + "bv": [0.3227564, -0.0315493, -0.1356114, -0.17830043, 0.15570031], + "bx": [0.4849305, 0.2781357, -0.06708594, 0.2408539], + "by": [0.28568855, 0.44024497], + "p": [3.5869982] + }, + "inputs": { + "state": [[-2.3180752, -0.5615534, 1.0030477, -0.7606695], [-0.2702305, -0.23171212, -2.2494705, -0.87876993], [0.77856463, 1.4597834, -0.06399139, -0.09288893], [0.5698699, 0.18409064, 0.5273473, -0.8559692]], + "u": [[0.04198794, 0.5244702, 0.6162741], [-1.5374323, 0.84131557, 0.13582507], [0.17607851, -1.7495761, 0.05958362], [-1.5602274, -2.5031295, 0.11653887]] + }, + "outputs": { + "next_state": [[-0.74712443, 0.4377111, 1.0364678, 0.5836489], [-0.90921175, 0.6716645, 1.0089943, 0.85499585], [1.0612028, 0.9470632, 0.15678768, 1.6996092], [1.2713315, 0.050089747, 0.031301662, 1.1374193]], + "y": [[0.9420475, 2.0704577], [1.1217057, 2.8385456], [0.75849557, -3.1199214], [-0.9774822, -10.530242]] + } + } + ] +} diff --git a/tests/golden/lbdn.json b/tests/golden/lbdn.json new file mode 100644 index 0000000..41682c6 --- /dev/null +++ b/tests/golden/lbdn.json @@ -0,0 +1,103 @@ +{ + "network": "lbdn", + "description": "Lipschitz-bounded deep network (JAX only).", + "cases": [ + { + "name": "relu", + "config": { + "input_size": 5, + "hidden_sizes": [8], + "output_size": 2, + "gamma": 3.0, + "activation": "relu" + }, + "params": { + "layers_0": { + "XY": [[0.31264108, 0.63436234, -0.1427789, -0.49710825, 0.04236971, -0.06134312, -0.59568727, -0.33192092], [0.20999396, 0.02256408, -0.60066354, -0.19129054, -0.3037694, -0.05249413, -0.03527096, -0.13951385], [0.20422393, -0.1252599, -0.27537176, -0.37673026, 0.31184456, 0.683691, -0.23882854, 0.1605333], [-0.24267885, -0.09723239, 0.20726903, -0.7489575, 0.54468024, 0.23412874, 0.33507013, -0.37267488], [0.5662478, -0.6967473, 0.10409111, -0.041546702, 0.6964553, 0.024814188, 0.3407347, 0.0064623505], [0.5833281, 0.13622427, -0.43505907, 0.20600422, 0.1964487, 0.34089863, -0.15617278, 0.28140295], [0.5258589, -0.31990045, 0.31714278, 0.08361265, 0.37484056, 0.15017593, 0.2873417, -0.20438194], [0.003579408, -0.82280254, -0.057804674, -0.12002256, -0.724784, -0.6812637, 0.40421534, -0.19909507], [-0.5040365, -0.03468384, -0.3732182, -0.45727196, -0.08342501, 0.18593161, 0.027149824, 0.35345683], [-0.19323878, -0.05807218, -0.35089302, 0.43943307, -0.08675214, -0.32884628, -0.10917825, 0.41013363], [-0.44308272, -0.09970911, -0.09739575, -0.22494437, 0.28520757, 0.4926642, 0.66830987, -0.78303933], [0.6698703, -0.06606252, -0.5292069, 0.19101277, -0.1379706, 0.09509154, -0.40386713, -0.011655986], [-0.3900975, 0.5550625, -0.24125546, -0.26355726, 0.2214747, 0.0036651492, -0.018822268, 0.20086625]], + "a": [2.3138251], + "b": [-0.30775416, -0.004585543, 0.13015902, -0.15947695, 0.016317165, 0.21666262, 0.088968374, 0.26060107], + "d": [0.13841869, 0.1464017, 0.54799026, 0.18695156, 0.038841344, 0.21298316, -0.2757371, -0.10131144] + }, + "layers_1": { + "XY": [[-0.13605699, 0.060297757], [0.12263757, 0.68416035], [-0.25149345, 0.60214615], [0.45615855, -0.031769633], [1.045799, 0.44492278], [-0.96159583, 0.055849716], [-0.12422004, 0.42326066], [-0.44526622, 0.12388565], [0.11779692, -0.25307992], [-0.35476127, -0.108742796]], + "a": [1.6877941], + "b": [0.1871933, -0.27824974], + "d": [-0.34494475, 0.035692066] + } + }, + "inputs": { + "x": [[-0.80193144, -1.3243589, -0.24836162, 0.42044523, 1.1360465], [0.1097064, -0.5526473, -0.7847804, 0.7487458, 1.634783], [0.27276877, -1.2333287, -0.9582652, 1.6000191, 0.20288244], [-1.7321348, -0.083696194, -1.163226, -0.6292881, -0.48800582]] + }, + "outputs": { + "y": [[-0.24834289, -0.055386137], [0, 0], [0, 0], [-2.030851, -0.32956636]] + } + }, + { + "name": "tanh", + "config": { + "input_size": 5, + "hidden_sizes": [8], + "output_size": 2, + "gamma": 3.0, + "activation": "tanh" + }, + "params": { + "layers_0": { + "XY": [[0.2327117, 0.29632816, -0.112878665, -0.17530498, 0.22818905, -0.19300655, -0.36449522, -0.10298717], [0.5791934, 0.63758636, -0.21833785, -0.59858954, -0.6059097, 0.47385314, 0.117067635, -0.65199655], [0.3574127, 0.28130728, 0.013841748, -0.34826574, 0.16076075, 0.7073045, 0.15069999, 0.022617716], [0.61867124, 0.40899593, 0.45083606, -0.05284465, 0.9561999, 0.26866257, 0.01794202, -0.2763489], [0.3465043, -0.39166042, 1.0355679, 0.1527423, 0.6430813, 0.25049943, -0.44472864, 0.16470659], [-0.32321382, -0.087502435, 0.004102826, -0.49234688, -0.5564194, 0.77668595, -0.28991586, -0.041689634], [0.23101524, -0.37320432, 0.58018553, 0.14706399, 0.2598651, 0.21006495, 0.0024440717, 0.58270925], [0.46862194, -0.2463713, -0.39310086, -0.39938495, -0.64672613, -0.1450863, 0.3088408, -0.15379485], [-0.60777414, -0.18567719, -0.11606519, -0.46569958, -0.18023154, 0.40470374, 0.13171233, 0.24046336], [0.24035595, 0.71934795, 0.4420396, 0.4001805, 0.0026442818, -0.260109, -0.06299255, 0.7185568], [0.11939488, -0.18415497, -0.676579, 0.24725798, 0.26040298, 0.13490897, 0.12488279, 0.20968544], [0.3713974, -0.20822787, -0.35237926, -0.04693316, -0.19328669, 0.19605005, -0.5496944, 0.6652164], [-0.4164065, 0.25680816, -0.11586504, -1.0972697, -0.18307668, 0.26929587, -0.1828475, 0.3137337]], + "a": [2.3543346], + "b": [0.25301427, -0.2087594, -0.27109742, -0.2586569, 0.04762581, 0.22373384, -0.17851599, -0.453056], + "d": [0.31874722, -0.3195251, 0.35844913, -0.016288763, -0.20950498, -0.19201198, 0.13855986, 0.26681155] + }, + "layers_1": { + "XY": [[-0.43347433, 0.15514205], [0.28422257, 0.618526], [-0.15762907, -0.29834807], [-0.2888099, 0.093951955], [0.41591808, 0.03767094], [-0.25836188, -0.6014459], [-0.3677886, 0.55058175], [0.103695154, 0.35680836], [0.3380029, 0.3918931], [0.08014984, -0.15119575]], + "a": [1.0639937], + "b": [-0.022100046, -0.22016521], + "d": [-0.2118526, -0.2142717] + } + }, + "inputs": { + "x": [[-0.70658016, -0.63016564, -1.6796945, 1.9504915, 0.9166192], [-0.97390735, 0.9082103, 1.3424871, -2.3895533, -0.5489448], [-0.38797322, 0.64825445, -0.12146542, -0.23043133, -0.058371387], [1.8533257, 2.1599805, -0.5248264, -0.926244, 2.692553]] + }, + "outputs": { + "y": [[0.58458513, -0.81557024], [-0.2275641, 0.6958203], [0.3837088, -0.32784155], [0.89745575, -0.7032506]] + } + }, + { + "name": "deep_sigmoid_nobias", + "config": { + "input_size": 5, + "hidden_sizes": [6, 6], + "output_size": 2, + "gamma": 3.0, + "activation": "sigmoid", + "use_bias": false + }, + "params": { + "layers_0": { + "XY": [[0.36616826, 0.22397286, -0.29702455, -0.4664069, -0.20996934, -0.3531437], [-0.24239615, -0.16101593, 0.7050042, 0.18836252, -0.43139738, -0.39965445], [-0.18741708, -0.01692497, 0.2904758, -0.7700549, -0.23209655, 0.40695277], [-0.2738034, 0.3702778, 0.17368592, 0.5760683, 0.39683145, -0.34225464], [0.36044073, -0.29755965, 0.3618406, -0.10569482, 0.74988604, 0.60574925], [-0.096441686, -0.13600045, 0.118117005, -0.51217693, 0.72098655, 0.11994557], [0.64033055, 0.08946547, -0.20386934, 0.038578838, 0.05428034, -0.01404988], [0.20140144, 0.038050085, -0.24097814, 0.29317695, -0.13624828, -0.010720104], [0.85588634, -0.34180957, -0.06833747, 0.5490849, 0.6650522, 0.60090923], [0.18000396, 0.09395341, 0.32862765, -0.2590102, -0.102915704, 0.26691225], [-0.4074696, -0.51003253, 0.24445684, 0.46046057, 0.29889852, -0.34034327]], + "a": [2.3072834], + "b": [0.3742649, 0.21709472, 0.20917746, -0.39370012, -0.11212724, -0.15449867], + "d": [0.0623714, -0.5054826, 0.38152587, 0.090039544, 0.13045111, -0.3552421] + }, + "layers_1": { + "XY": [[-0.34936106, 0.2826911, -0.7829927, 0.71341234, 0.38053542, -0.19117798], [0.36153525, 0.1619249, 0.94682467, 0.55553675, -0.31373823, 0.768777], [0.27554116, 0.34330136, -0.54640955, -0.011434555, 0.43429816, 0.14444025], [-0.09024377, 0.07465041, -0.3701573, -0.10820323, -0.6835327, -0.4648807], [0.20347089, 0.29839253, -0.3346607, 0.96956146, -0.42679152, 0.055832148], [0.4767121, 0.09329556, 0.33129877, -0.3974824, -0.08817283, 0.43383083], [0.02954948, 0.68323976, 0.006381601, 0.03875372, -0.58724797, -0.23091286], [0.30308944, -0.050766185, -0.0033309758, 0.11543915, -0.2780324, 0.23741889], [-0.8984565, -0.058674656, -0.4570927, 0.47813246, 0.4129262, -0.45745757], [0.34060818, 0.6231352, 0.59153193, 0.16786833, -0.429992, -0.75592923], [0.40902948, 0.34272367, 0.71558255, 0.195393, 0.15214726, -0.42458022], [0.10848121, 0.3679328, -0.48076755, -0.37270114, -0.08028162, 0.42985523]], + "a": [1.3357009], + "b": [0.021091655, 0.06925635, -0.05305091, 0.08004747, 0.13677187, -0.3034914], + "d": [0.12485405, -0.07539216, -0.28376365, 0.06274224, 0.44629326, -0.43803468] + }, + "layers_2": { + "XY": [[-0.802737, -0.11191741], [0.14973205, -0.5427996], [0.32400262, -0.96096134], [-0.156469, -0.7562258], [-0.1086746, 0.46406412], [-1.1376059, -0.15161476], [-0.0016548485, 0.3584992], [-0.2981413, 0.3048444]], + "a": [1.9292567], + "b": [0.32053638, 0.40245858], + "d": [-0.349925, -0.16634753] + } + }, + "inputs": { + "x": [[-0.044646863, -0.3297108, 0.8711105, -1.3261814, -0.16947019], [1.8594975, 0.038897336, 1.1986152, 1.1368635, 1.3166478], [0.7381192, -1.3804461, -0.26738772, -0.38033557, -1.5488553], [-0.4638816, -1.2500799, -1.6655514, 0.21089625, -0.043591272]] + }, + "outputs": { + "y": [[0.0020629345, -0.46660802], [-0.0030833206, -0.45690617], [0.000066249566, -0.46176744], [0.0010324412, -0.4635964]] + } + } + ] +} diff --git a/tests/golden/lipschitz_linear_ren.json b/tests/golden/lipschitz_linear_ren.json new file mode 100644 index 0000000..db932c5 --- /dev/null +++ b/tests/golden/lipschitz_linear_ren.json @@ -0,0 +1,38 @@ +{ + "network": "lipschitz_linear_ren", + "description": "Linear REN (lipschitz_linear_ren); JAX only.", + "cases": [ + { + "name": "gamma2", + "config": { + "input_size": 3, + "state_size": 4, + "features": 0, + "output_size": 2, + "init_method": "long_memory", + "gamma": 2.0 + }, + "params": { + "B": [[0.27120665, -1.3035274, -0.60656285], [0.1958345, 0.4443609, 0.053900868], [0.048028134, 0.38839352, 0.51704395], [-0.28517446, 0.6417335, 0.8922816]], + "C": [[0.19141865, 0.22711514, 1.2083597, -0.82112986], [-0.35587108, -0.56942743, 0.5503082, -1.1033623]], + "D": [[-0.104296096, 0.16543664, -0.114033125], [0.1316908, 0.2938621, -0.16323403]], + "X": [[1.3694055, 0.48654136, 0.32371378, 0.34963888, 1.2983983, 0.67636174, 0.05575827, 0.0006252249], [0.18093221, 0.72754145, -0.46595305, -0.2646024, 0.11176961, 1.0990276, -0.46090764, -0.56503916], [-0.09484267, -0.056416955, 0.98523974, 0.20140874, 0.36865026, 0.069268934, 1.1489187, -0.32856053], [-0.35289243, 0.0708715, 0.19939893, 1.2178731, 0.17782259, 0.23525213, 0.24294843, 0.45173317], [-0.22024974, 0.13654794, 0.17900158, -0.45362774, 0.59756553, -0.13164958, -0.06972743, 0.082145594], [0.22489057, -0.42993513, 0.2969738, 0.023188042, -0.23124407, 0.20713758, -0.6557516, -0.19428557], [0.0545729, -0.21271515, -0.09840386, -0.044028714, 0.014514268, 0.30619103, 0.25436398, -0.07799038], [0.16159357, 0.6229851, -0.10282431, 0.15386018, -0.4731915, 0.4036482, 0.19781508, 0.6112899]], + "X3": [[0.9822819, -0.3128447], [0.26582968, 1.22606]], + "Y1": [[0.7398883, -0.44188055, 0.13473964, -0.0514176], [0.7836498, 0.81572604, -0.15856868, -0.03322928], [-0.4995172, -0.2667261, 1.4933088, -0.03941323], [-0.18251659, -0.38156942, 0.17414495, 1.1974893]], + "Y3": [[0.076936305, -0.11676039], [0.0916614, 0.114476755]], + "Z3": [[-0.2042654, 0.18669976]], + "bx": [-0.6346069, -0.38670772, -0.09117516, -0.032549728], + "by": [-0.34928262, 0.09861659], + "p": [3.4932828] + }, + "inputs": { + "state": [[-0.00682678, 1.0461433, 0.7415884, 0.7239565], [1.6187762, -1.2055582, -0.62695545, -1.3206632], [-0.10775251, 0.9987637, -0.021947887, 0.49588007], [-1.9107686, 0.14706416, -0.90694326, 1.7753894]], + "u": [[0.8868491, 0.94934946, -0.05785496], [0.6128623, 0.65789014, -0.34440267], [-0.49737203, -0.11477278, -0.605452], [-0.59433943, -0.28337538, -0.72841775]] + }, + "outputs": { + "next_state": [[-0.4694875, 0.5658406, 0.71906465, 0.2265441], [-0.6535689, -0.023697674, -0.78743553, -0.4349953], [-0.20108217, -0.20175917, -0.10969555, -0.019720454], [-1.2338533, -0.88171697, -0.17672223, 0.19825685]], + "y": [[0.24539852, -1.4582114], [-0.05626571, 1.0184644], [-0.76887685, -0.67900586], [-3.4908586, -1.3165333]] + } + } + ] +} diff --git a/tests/golden/lipschitz_ren.json b/tests/golden/lipschitz_ren.json new file mode 100644 index 0000000..ff0e862 --- /dev/null +++ b/tests/golden/lipschitz_ren.json @@ -0,0 +1,78 @@ +{ + "network": "lipschitz_ren", + "description": "Lipschitz-bounded REN (JAX only).", + "cases": [ + { + "name": "relu_gamma2", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "relu", + "gamma": 2.0 + }, + "params": { + "B2": [[-0.50361043, -0.35896152, -1.112064], [-0.28890592, 0.5835506, 0.89639914], [-0.06427954, 0.9280747, 0.9266898], [-0.38386172, -0.1259296, -0.02418086]], + "C2": [[-0.48456943, -1.3476148, -1.3913274, 0.4715274], [0.11983447, 0.5829764, 0.5783793, -0.40412804]], + "D12": [[0.010679305, 0.0767545, -0.20751892], [-0.89245355, -0.62732494, -0.29800773], [0.8717886, 0.04976584, -0.010464922], [0.98806787, -0.3727018, 0.11716467], [-0.14780948, -0.401968, 0.4838892]], + "D21": [[0.0215825, 0.81710863, -0.44421315, -0.67154676, 0.5930991], [0.39203805, 0.5958261, 1.3842779, 0.4539413, 0.50892454]], + "D22": [[0.58138853, -0.07211451, 0.15978384], [0.47156468, 0.16603373, -0.19035156]], + "X": [[0.38129497, 0.71794915, 0.12909429, 0.3781587, -0.14289407, 0.5455445, 0.15586181, 0.014939233, -0.27054518, 1.0826008, -0.14595902, -0.06683076, -0.48675326], [-0.14716618, 1.0479637, -0.37524188, 0.022168955, 0.19898818, -0.09202799, -0.18623525, -0.18603726, 0.2823427, -0.46035153, 0.86299384, -0.12582153, 0.043199115], [-0.020871026, -0.33225185, 1.6120268, 0.05328353, -0.4026115, -0.26743007, 0.029275464, 0.18389098, -0.40264654, 0.21340106, -0.41198763, 0.7184995, 0.2559304], [0.5102992, -0.025412362, -0.092861585, 1.3803698, -0.3215937, 0.08337655, 0.06997367, 0.30279654, 0.08957353, -0.07991831, 0.18436638, -0.33796948, 1.2210011], [-0.20180295, -0.27310976, -0.110293925, 0.013800433, 1.588321, -0.21012443, 0.8443397, 0.46637118, 0.034285333, -0.29446414, 0.27592534, 0.08263447, -0.6345581], [-0.29648662, 0.042948507, -0.41157168, -0.20442605, -0.5138767, 1.1460053, 0.15247923, -0.9666464, 0.99339354, -0.105780505, -0.08675406, 0.605977, -0.39025894], [0.21874888, -0.39452896, 0.07087674, 0.63473487, 0.08140681, -0.23154683, 1.4933591, -1.1358433, -0.9273917, 0.091311626, -0.40276957, 0.3345295, 0.48564753], [0.22762975, 0.099108316, -0.044789307, 0.2660207, -0.94502354, -0.03743108, 0.07683772, -0.56764776, -0.15644878, -0.038251497, -0.010369792, 0.030417372, 0.17254111], [0.13738877, -0.3112056, -0.035088595, -0.0546295, 0.011526732, -0.1773274, 0.2909562, 0.39136454, 1.7712702, 0.5716957, 0.44798422, -0.13797949, -0.18431306], [-0.119291686, 0.16753156, -0.038949195, 0.29732808, 0.49511746, -0.12907839, 0.29391232, -0.11788344, -0.37994927, -0.14627413, -0.095738895, -0.5697303, 0.14979315], [0.45320433, -0.34951863, -0.100294225, 0.46306753, 0.1041696, 0.05869999, -0.078460194, 0.30559763, 0.0987727, -0.32525635, -0.4270645, -0.80450773, 0.19730145], [-0.07982535, 0.43443894, 0.15299836, -0.38161114, 0.08711669, 0.3575585, -0.27052703, -0.15262273, 0.5222962, -0.5239176, -0.3270861, 0.605706, 0.059698723], [-0.34274727, -0.43657362, 0.13903643, 0.36428088, 0.24797536, 0.123355605, 0.448642, 0.09334284, -0.2497432, 0.6095641, -0.37021625, -0.040361557, 0.19233334]], + "X3": [[1.4293408, -0.22911097], [-0.083016045, 0.465878]], + "Y1": [[1.1012604, 0.13354875, 0.1952032, -0.27802017], [0.070908785, 0.7347565, -0.14373566, -0.023573983], [0.38779393, -0.13208443, 1.0521919, -0.2542303], [-0.17198534, -0.21155651, -0.04891446, 1.2285612]], + "Y3": [[-0.33339083, 0.21382014], [-0.2876807, 0.09810226]], + "Z3": [[-0.16826129, -0.093672544]], + "bv": [-0.120650776, 0.3261184, 0.07183985, 0.31679618, -0.11502009], + "bx": [-0.07747811, 0.47771063, -0.50113845, -0.2704405], + "by": [-0.06044389, 0.53931683], + "p": [3.6171608] + }, + "inputs": { + "state": [[-1.7382663, -1.3366427, -1.3611068, -0.35161713], [-2.3125815, -0.18889719, -0.95722926, 0.89360017], [0.95684725, 1.3922583, 0.7674701, -0.05302978], [0.85979396, 1.5054811, -0.65359455, 0.61035115]], + "u": [[-0.042673826, 1.4400167, -0.83689505], [-0.3015466, 0.3623386, 0.25811026], [-1.6394479, 0.36015522, -0.1184977], [-0.23974785, -0.15530166, 0.2189717]] + }, + "outputs": { + "next_state": [[1.4012337, 0.2841835, -0.7398934, -0.8770298], [0.7323088, 0.86641836, -0.6920468, 0.21595284], [0.7934779, 0.35057676, 0.17441034, -0.5245844], [0.10639705, 1.2949253, -1.0200082, 0.2306523]], + "y": [[2.9773197, 3.2558205], [2.4153428, 1.8797104], [0.45639944, 4.313595], [-0.5787045, 1.8442125]] + } + }, + { + "name": "tanh_gamma0.5", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "activation": "tanh", + "gamma": 0.5 + }, + "params": { + "B2": [[-0.24471526, -1.0989863, -1.1631268], [0.11551279, 0.12370214, 0.09123504], [0.114458345, 0.18538564, 1.4297779], [-0.3830461, 0.73841363, -0.11927116]], + "C2": [[-0.93456954, -1.2799611, -0.97461426, 1.261198], [0.057298418, 0.215749, 0.3418068, -1.0019875]], + "D12": [[0.2444673, 0.5956985, 0.4117443], [-0.5938672, -0.1257526, 0.05259244], [-0.23261458, -0.105347686, -0.36562586], [0.4294678, -0.31010008, 0.29496503], [-0.40406185, 0.37651944, 0.581541]], + "D21": [[0.055371836, 0.34741932, -0.14870162, -0.53429246, 0.6893516], [0.3526558, 0.5428671, 1.0341415, 0.1365411, -0.060423702]], + "D22": [[0.52232605, -0.45671085, 0.14616866], [0.09611309, 0.06675263, -0.20808142]], + "X": [[1.3047333, 0.10530544, -0.10264834, -0.42216402, -0.068912275, -0.17968695, -0.37749544, 0.11433616, 0.17484844, 1.2227823, 0.03726308, -0.07150864, 0.34472728], [0.23498347, 1.6962247, 0.1731445, -0.27533904, 0.29653898, 0.7071081, 0.08981331, -0.046830125, -0.17061421, 0.5829976, 0.9070094, -0.06337786, 0.10254315], [-0.41572455, -0.06991387, 0.6314802, -0.02982606, 0.16278127, 0.29227486, -0.11153762, 0.26132154, -0.1030369, -0.4821077, 0.18776406, 0.91072863, 0.049423173], [-0.7707522, 0.08206344, 0.09332755, 1.0934315, 0.3326895, -0.0879961, 0.6671172, 0.24093105, 0.18720187, -0.72644484, -0.5679327, 0.47533664, 1.1197112], [0.30603352, -0.08721252, -0.008111942, 0.20554434, 1.1674653, 0.28244245, 0.3222971, 0.27338478, 0.018940743, 0.14197324, 0.4113767, -0.2988123, 0.17532532], [0.07639391, 0.29404482, 0.034487095, -0.10559029, 0.19333012, 1.4687864, 0.3688639, -0.9285367, 0.7313127, -0.41813257, -0.11752169, -0.030764533, -0.11611786], [-0.16517656, 0.030818397, 0.16540658, 0.62716866, -0.16537794, -0.22792353, 1.1219366, -0.8813247, -1.0874648, 0.5041735, -0.018024758, -0.053562768, -0.14027387], [-0.23270543, 0.073570125, -0.24438435, 0.7789907, 0.14164713, -0.0337028, -0.19231884, 0.14356901, 0.052112132, 0.109103344, -0.30683625, -0.034236457, -0.47723612], [0.4639048, 0.23805478, 0.095116176, 0.12044033, -0.13766041, -0.011085069, 0.3210013, 0.01947347, 1.5240942, 0.09876612, 0.10240009, 0.12854502, 0.34524193], [-0.054004144, 0.23759493, -0.20307252, -0.1400561, -0.18028887, 0.29083377, -0.055177942, 0.4035998, -0.26980752, 0.3864299, -0.30994457, 0.37369925, -0.17077611], [0.17404214, -0.16908018, -0.19232933, 0.10958807, -0.47749248, -0.320603, 0.49025923, -0.20933668, 0.28149235, -0.2211978, -0.15270287, -0.087339684, 0.29611492], [0.028583188, -0.052401997, 0.29242933, -0.19210121, 0.25232643, 0.12882969, -0.2732617, 0.31158984, 0.3705582, -0.039712273, 0.0688475, 0.23922423, -0.30441487], [0.26888376, -0.27139223, 0.18840002, -0.1294527, -0.27853668, 0.21324828, 0.09901982, 0.58232385, 0.007538666, 0.073474474, 0.06338247, -0.046129923, 0.6380363]], + "X3": [[1.4801518, 0.23344024], [-0.2726513, 1.1654605]], + "Y1": [[1.0481036, -0.10395694, -0.43428734, 0.2963407], [0.3911633, 1.1247548, 0.16804108, 0.558823], [0.6119902, 0.09819255, 0.17587316, 0.24526712], [0.07725093, 0.2369168, -0.123656355, 0.7309134]], + "Y3": [[0.18380848, -0.059007194], [0.06123773, 0.15497701]], + "Z3": [[0.060395606, -0.40207317]], + "bv": [0.20054196, -0.11868081, -0.036223236, 0.15340668, -0.40996328], + "bx": [-0.055052113, -0.5240359, -0.14455846, 0.122900106], + "by": [-0.023071608, -0.06668474], + "p": [4.653902] + }, + "inputs": { + "state": [[0.7699411, 1.5583218, -0.26087454, -0.19837509], [1.6461709, -0.57878524, -1.0786495, 0.802963], [-0.5418882, 1.2721843, 0.5135475, -1.7799098], [0.2533247, -0.014213847, -0.68933, 1.0785818]], + "u": [[-1.7978252, -1.2800208, -0.5234267], [0.76621413, -1.5694802, -0.6661272], [-0.7451258, -1.5424984, -0.41986233], [-1.1857109, -0.48898765, 0.2486883]] + }, + "outputs": { + "next_state": [[0.48044837, -0.12104195, -0.35036504, 0.28446075], [0.7744948, -0.8482312, 0.040346593, 0.59763765], [0.06232645, 0.11033106, -1.1228688, -0.24748647], [0.46913135, -0.9318199, 0.6193843, 0.8181429]], + "y": [[-1.4667065, -0.14453456], [-0.13550213, 0.09674661], [-2.4307013, 0.9491092], [0.90662175, -0.067761324]] + } + } + ] +} diff --git a/tests/golden/monlipnet.json b/tests/golden/monlipnet.json new file mode 100644 index 0000000..280fe29 --- /dev/null +++ b/tests/golden/monlipnet.json @@ -0,0 +1,93 @@ +{ + "network": "monlipnet", + "description": "Monotone Lipschitz network.", + "cases": [ + { + "name": "units4x4_mu_nu_fixed", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "units": [4, 4], + "is_mu_fixed": true, + "is_nu_fixed": true + }, + "params": { + "Fab0": [[0.22049716, 0.4890259, 0.08617562, 0.9401134], [0.2558039, -0.24591777, 0.42528033, -0.055981442], [0.37202895, 0.33750203, 0.96850985, -0.2578378], [-0.30975127, 0.09036125, 0.8622034, -0.13251013]], + "Fab1": [[0.30771154, -1.3730597, 0.25824666, 0.65716416], [-0.06309289, 0.22834669, -0.18546653, 0.20268704], [-0.56297714, -0.5468919, -0.74452305, 0.42454094], [0.03211825, 1.418281, -0.39650577, -0.71913886], [-0.5583456, -0.022512138, -0.17751339, 0.21692134], [-0.50571257, 0.7928748, 0.26723272, 0.013224155], [0.3570255, 0.3613974, 0.22304393, 0.7011218], [0.8410313, 0.10980892, -0.66274, -0.28723818]], + "Fq": [[0.47126108, 0.0590026, 0.07286385, -0.90306264, -0.074051514, -0.2607226, -0.23512617, -0.6919776], [0.1656493, 0.23822919, -0.12710181, 0.14167446, -0.3791923, -0.02692911, 0.12537047, -0.86478186], [0.10138321, 0.25987884, 0.6432996, -0.73375404, 0.48840746, -0.09875933, -0.35699064, -0.57334673]], + "b0": [-0.22773245, -0.04023856, -0.27168182, 0.057013754], + "b1": [0.3387683, -0.25082555, 0.42856324, -0.20028599], + "by": [0.046023652, -0.25091597, -0.06667055], + "fab0": [1.7440581], + "fab1": [2.2132332], + "fq": [1.7386944] + }, + "inputs": { + "x": [[0.18905339, -0.5227485, -0.41306356], [-2.4414673, 1.7997074, 1.1441659], [-0.32542282, 0.7738066, 0.28121066], [-0.5538228, 0.97756743, -0.31055656]] + }, + "outputs": { + "y": [[0.050054744, -0.5502404, -0.45121258], [-9.059955, 6.10448, 2.5598006], [-1.950451, 1.8542109, 1.0736405], [-2.637277, 2.246213, 0.15481871]] + } + }, + { + "name": "units3_mu_tau_fixed", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "units": [3], + "is_mu_fixed": true, + "is_tau_fixed": true + }, + "params": { + "Fab0": [[0.6590594, 0.5525426, 0.048138443], [0.52939355, 0.37626547, -0.121100344], [0.15683249, 0.7479767, 0.3676964]], + "Fq": [[0.38858777, -0.06383102, 0.15825646], [-1.2291744, -0.14681156, 0.31955168], [0.1027495, -0.11310792, -0.4222318]], + "b0": [-0.40479517, -0.17747247, -0.027722422], + "by": [0.2072856, 0.39611387, -0.24250376], + "fab0": [1.7271314], + "fq": [1.2579598] + }, + "inputs": { + "x": [[-0.6778862, -0.82115996, -1.57008], [-0.2629696, 0.4011582, 0.9084019], [0.6471027, 2.4573364, 0.31868163], [-0.4563339, 1.8716346, -1.0472181]] + }, + "outputs": { + "y": [[-2.0283127, 1.184828, -5.328971], [-1.4395599, 2.0751557, 0.98695767], [1.205656, 9.697035, 1.551538], [-1.5743196, 7.305057, -4.343689]] + } + }, + { + "name": "units3x3x3_nu_tau_fixed", + "config": { + "input_size": 3, + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "units": [3, 3, 3], + "is_nu_fixed": true, + "is_tau_fixed": true + }, + "params": { + "Fab0": [[0.28197592, 1.1682656, -0.47659007], [1.2485965, -0.15992555, -0.21969278], [0.45451584, 0.1493285, 0.2173667]], + "Fab1": [[1.178697, -0.8683492, -0.019536812], [0.3052102, -0.687796, -0.04531223], [-0.92365026, 0.3612733, -0.6631546], [-1.0077356, -0.18384314, 0.83672696], [0.019230776, 0.71325916, -0.09990075], [-0.3527392, -0.71502554, -0.04876697]], + "Fab2": [[0.23281023, -0.39869738, -0.19060555], [-0.7563827, 0.82016945, 0.3783283], [0.028752953, 0.38310367, 0.44385344], [0.20959215, 0.8172512, -0.20892042], [-0.26046443, 0.72944164, -1.3667178], [0.4294544, 0.6176383, 0.38309118]], + "Fq": [[0.051991478, 0.28274715, 0.70868725, -0.7729224, -0.5937631, 0.49040133, -0.3492568, 0.1875212, -0.3602813], [0.4860711, -0.8102819, 0.5353698, -0.059799135, -0.6811096, -0.4501125, -0.5625565, 0.73954654, -0.2205447], [0.5342692, -0.8928851, 0.7684181, 0.21675006, -0.5137974, -0.814303, 0.58889294, 0.761267, -0.43621555]], + "b0": [0.6904485, 0.21675198, -0.14640181], + "b1": [-0.57130677, 0.19705392, 0.16725342], + "b2": [-0.17868955, -0.537166, 0.33734304], + "by": [-0.17649569, -0.14152353, 0.8536936], + "fab0": [2.3675628], + "fab1": [3.0387745], + "fab2": [1.5669823], + "fq": [2.2346296] + }, + "inputs": { + "x": [[2.0751877, -0.048723545, 0.502127], [-0.9361622, -0.81068945, 0.20169531], [-0.38405445, 0.3532886, -1.5618241], [0.66200495, -0.89948034, 1.7224234]] + }, + "outputs": { + "y": [[1.720689, -0.37076974, 2.3401375], [-2.1512191, -0.669056, 1.0969003], [-1.710139, 0.8709239, -0.4710834], [1.2938443, -1.8146315, 3.3683333]] + } + } + ] +} diff --git a/tests/golden/plnet.json b/tests/golden/plnet.json new file mode 100644 index 0000000..246b5bc --- /dev/null +++ b/tests/golden/plnet.json @@ -0,0 +1,177 @@ +{ + "network": "plnet", + "description": "Polyak-Lojasiewicz network (quadratic potential of a BiLipNet).", + "cases": [ + { + "name": "default", + "config": { + "BiLipBlock": { + "input_size": 3, + "units": [4], + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "depth": 2 + } + }, + "params": { + "BiLipBlock": { + "mon_0": { + "Fab0": [[0.23012203, -0.7422156, -0.7046257, 0.9706961], [-0.3361004, 0.82568413, -0.6507746, -0.69468373], [0.12917234, -0.76369256, -0.5237465, 0.16865611], [0.17223021, 0.28311586, -0.17716584, 0.1512922]], + "Fq": [[0.07251868, -0.95747554, 0.46799228, 0.016376354], [-0.28603792, 1.1014916, -0.018092368, -0.58799255], [1.1432192, -0.86752224, -0.3111256, 0.99362373]], + "b0": [0.22438793, 0.2552902, -0.21290547, -0.18215916], + "by": [-0.23934688, -0.17527148, -0.07137702], + "fab0": [1.8499174], + "fq": [3.0590537] + }, + "mon_1": { + "Fab0": [[-0.65893847, 0.02946277, -0.8406506, -0.4362114], [0.41977862, 0.38578373, -0.3717922, -0.7226405], [0.030087292, -0.30321878, -0.32012594, -0.013504356], [0.88920647, 0.6203633, 0.2701181, -0.57623684]], + "Fq": [[0.5103037, 0.36834192, -0.85060215, -0.43355685], [-1.0046659, 1.4018228, -0.22689211, -0.5929904], [0.24823338, 0.15697503, -0.06868833, 0.017911613]], + "b0": [-0.07463401, -0.50655884, -0.57268184, -0.2929304], + "by": [-0.023411706, 0.23939572, 0.7278662], + "fab0": [1.5187087], + "fq": [2.3266125] + }, + "uni_0": { + "W": [[0.055772305, -0.036343373, -0.40892243], [-0.854671, -0.28612936, 0.10096559], [0.07197407, -1.4857996, 0.91913426]], + "a": [1.5862284], + "b": [0.04957146, 0.34123057, 0.15543368] + }, + "uni_1": { + "W": [[-0.5476791, 0.3922141, -0.027115185], [-0.5334221, 0.18482393, -0.6574999], [-0.9427035, 0.33872902, -0.12911046]], + "a": [1.4141481], + "b": [0.4041812, 0.027014479, 0.055017903] + }, + "uni_2": { + "W": [[0.24431977, 0.6493766, 1.1058724], [1.5385989, 0.23798352, 0.7904861], [0.18490338, -0.5922145, 1.6556091]], + "a": [2.3176134], + "b": [0.013765156, 0.6773312, 0.64552236] + } + } + }, + "inputs": { + "x": [[-0.65179116, -0.17471729, 1.663724], [0.65914774, -1.6413972, -0.0052032643], [-0.62346375, 0.14863153, -1.6081878], [0.24177188, 0.23538092, 1.575626]] + }, + "outputs": { + "y": [1.2134815, 2.3925376, 0.7281305, 1.4379151] + } + }, + { + "name": "add_constant", + "config": { + "BiLipBlock": { + "input_size": 3, + "units": [4], + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "depth": 2 + }, + "add_constant": true + }, + "params": { + "BiLipBlock": { + "mon_0": { + "Fab0": [[-0.6572732, -0.9734158, 0.14496052, -0.03788477], [0.60873437, 0.79500747, -0.568256, -0.43277323], [0.3509078, -0.8204142, -0.72250235, 0.18770443], [-0.08895975, 1.069541, -0.399063, 0.6666229]], + "Fq": [[-0.39360407, -0.37273246, 0.6819522, -0.053471547], [0.12115441, 0.8867592, -0.23754612, -1.0858495], [0.7942295, -0.28518447, -0.102287285, 1.6259799]], + "b0": [0.06427599, -0.8923196, -0.13186611, 0.49951023], + "by": [0.0019037506, -0.1313501, -0.26209792], + "fab0": [2.4750574], + "fq": [2.4769988] + }, + "mon_1": { + "Fab0": [[-0.30540693, 0.24451068, -0.545952, 0.09607132], [0.16142663, 0.61232126, -0.5531481, -0.360932], [-0.15233713, -0.1889656, -1.2510173, 0.19958559], [1.0022845, 0.26581693, 0.10643975, -0.5023583]], + "Fq": [[0.030041963, 0.75897825, -1.2839175, -0.30682623], [-0.9035282, 0.58130926, 0.14593032, 0.16125342], [-0.15659612, 0.24222784, -0.6506821, 0.48208433]], + "b0": [0.48328087, -0.48435122, 0.35227782, -0.64970094], + "by": [-0.102002725, 0.11538739, -0.40949616], + "fab0": [1.378278], + "fq": [1.9733646] + }, + "uni_0": { + "W": [[0.5000891, -0.13322051, 0.4386027], [-0.6449519, 0.47371426, 0.4950203], [0.14363138, -1.0905645, 0.64023316]], + "a": [2.2253435], + "b": [0.661945, 0.16383535, -0.049711432] + }, + "uni_1": { + "W": [[-0.88930947, 0.19311866, 0.011943864], [-0.88684034, -0.06320386, -0.7657607], [-1.0336903, 0.09688285, -0.59006]], + "a": [2.347564], + "b": [0.25148776, -0.16073446, 0.3886002] + }, + "uni_2": { + "W": [[-0.13841277, 0.5767518, 0.6344635], [1.4358002, 0.20063299, 1.3861825], [0.2658151, -0.05083703, 1.2087485]], + "a": [3.0501113], + "b": [-0.04644361, -0.04268364, 0.024056258] + } + }, + "c": [0.2456209] + }, + "inputs": { + "x": [[0.11580668, 0.62486637, 1.0901799], [1.682021, -0.1310929, 0.13526021], [1.2246342, 0.2695503, 0.75449985], [0.6098595, -1.1349119, -0.858941]] + }, + "outputs": { + "y": [5.663268, 8.172804, 8.787078, 2.129326] + } + }, + { + "name": "optimal_point", + "config": { + "BiLipBlock": { + "input_size": 3, + "units": [4], + "mu": 0.5, + "nu": 4.0, + "tau": 8.0, + "is_mu_fixed": true, + "is_nu_fixed": true, + "depth": 2 + }, + "optimal_point": [[1.0, -0.5, 0.3]] + }, + "params": { + "BiLipBlock": { + "mon_0": { + "Fab0": [[0.05345335, -0.97009647, -0.5259979, 1.136177], [0.29346368, 0.80193955, -0.88690054, -0.74154925], [0.2948956, -0.5918696, -0.38214228, -0.084444225], [-0.09551519, 1.3016632, -0.4353353, 0.7070476]], + "Fq": [[-0.43321353, -0.20296873, 0.69535637, 0.48981085], [-0.03270568, 0.87169486, 0.25664836, -1.2264178], [0.8679678, -0.81938887, -0.26721203, 1.2743553]], + "b0": [0.22529145, 0.7884319, 0.42823663, 0.40181872], + "by": [-0.035029143, -0.07527537, 0.05580877], + "fab0": [1.2167127], + "fq": [2.8503344] + }, + "mon_1": { + "Fab0": [[-0.6944036, -0.18846649, -1.1026404, 0.12477584], [0.9153998, 0.2802888, -0.4621269, -0.33432207], [-0.6700005, -0.13667977, -0.35321057, -0.719777], [1.0299062, -0.16035381, 0.29654065, -0.8774072]], + "Fq": [[0.17884116, 0.46383202, -0.9960884, -0.57014173], [-0.67690253, 1.2571636, -0.38833046, -0.24260983], [-0.10219319, 0.08677898, -0.2391959, 0.073850244]], + "b0": [0.2151732, 0.023827033, -0.27586475, 0.42769125], + "by": [-0.023573818, -0.079116605, -0.272655], + "fab0": [1.7047043], + "fq": [1.870958] + }, + "uni_0": { + "W": [[0.29326513, 0.3434308, -0.35553575], [-0.5114434, 0.31054834, 0.5176394], [0.1996401, -0.79304755, 0.6492167]], + "a": [1.6933026], + "b": [-0.00831568, 0.53419733, 0.31883433] + }, + "uni_1": { + "W": [[-0.74914616, 0.5875616, -0.3318174], [-0.5269426, 0.15451854, -0.81232584], [-1.3002256, -0.03395784, -0.05112131]], + "a": [1.8347353], + "b": [-0.09170246, 0.21854566, -0.19807413] + }, + "uni_2": { + "W": [[-0.11042119, 0.44484526, 0.7767471], [1.9048834, 0.14502896, 0.96009284], [0.32293016, -1.0259672, 1.1489645]], + "a": [2.4027328], + "b": [-0.29290465, 0.13594766, -0.6105243] + } + } + }, + "inputs": { + "x": [[0.9381846, 1.9344972, -0.058941938], [1.2884654, -0.62583244, 0.6350175], [0.15966722, 1.7719189, 0.64833856], [-0.40744478, 0.62048936, 0.38984683]] + }, + "outputs": { + "y": [8.515611, 0.388256, 7.9926753, 5.6960783] + } + } + ] +} diff --git a/tests/golden/r2dn.json b/tests/golden/r2dn.json new file mode 100644 index 0000000..a1c4b75 --- /dev/null +++ b/tests/golden/r2dn.json @@ -0,0 +1,108 @@ +{ + "network": "r2dn", + "description": "Contracting R2DN (JAX only).", + "cases": [ + { + "name": "hidden2x2", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "hidden": [2, 2] + }, + "params": { + "B1": [[0.1773766, -0.20205803, 0.23574081, -0.0063740094, -0.29945487], [-0.021639299, -0.30212408, 0.30381954, -0.008579479, 0.25592753], [-0.33451682, 0.4858252, -0.0903184, 0.40957978, 0.25621542], [-0.33028355, 0.09694635, -0.1627351, 0.12280935, 0.4297436]], + "B2": [[0.24790955, -0.5043207, -1.2051023], [0.44468802, 0.7158644, 0.78582436], [-0.09452623, 0.63332784, 0.5884486], [-0.5823377, 0.30168965, 0.47977522]], + "C1": [[-0.35297042, -0.36807495, 0.268572, 0.08566989], [-0.07054687, 0.031648077, -0.12135775, -0.16107884], [-0.13817653, 0.115818575, 0.39639112, -0.14122707], [-0.020597184, -0.06548343, -0.37800834, 0.075582415], [0.2600296, -0.2531055, 0.2499292, -0.1715003]], + "C2": [[-0.6836059, 1.366511, 0.18710741, -1.7032497], [-0.35765478, 0.51117295, -1.4193239, 0.0473414]], + "D12": [[0.698489, 0.09615015, 0.06678743], [-1.4691706, -0.46397993, -0.35645723], [0.39086032, -0.1184245, -0.21659133], [0.22162579, -0.6647859, 0.4371778], [-0.01814872, -0.2745086, 0.20171538]], + "D21": [[0.84866655, 0.41111082, -1.1492224, 0.3773034, -0.9842266], [0.12228057, -0.6957388, 0.76565605, -1.3219662, 0.1779738]], + "D22": [[0.25262824, -0.1597976, -0.22593443], [0.2859137, -0.055290088, 0.27610156]], + "X": [[0.8791952, 0.21799979, 0.32356235, 0.02468756, 1.4940274, -0.11884216, -0.0635191, 0.023811301], [0.30165616, 0.9137219, -0.13097745, 0.07612121, 0.37495416, 0.8623137, -0.38329569, -0.03925542], [0.18712895, -0.16283756, 1.0372741, -0.5606961, 0.0063832933, 0.03082139, 1.1735622, 0.09278229], [0.2315333, -0.2983127, 0.1980866, 0.50462294, -0.17905627, 0.33175725, 0.25535503, 0.4382025], [0.1765253, -0.12924069, -0.23337637, 0.061335962, 0.33743483, 0.41829666, -0.43259266, -0.12851286], [-0.042763565, -0.5521894, 0.35192823, 0.5268258, -0.18753599, 0.2734364, -0.08073189, 0.08112037], [-0.20265731, 0.24946614, -0.34968296, 0.40901673, 0.106908366, 0.7274215, -0.33352613, -0.028655574], [-0.38566503, -0.041977238, 0.13053426, -0.2758543, -0.4305232, -0.5201304, 0.066255875, 0.49141312]], + "Y": [[1.3515646, 0.01012608, -0.28849962, -0.10898908], [-0.45236722, 0.5270961, 0.3197834, -0.08344463], [0.09433246, -0.08205171, 1.3974762, 0.4312577], [-0.23284733, -0.10982814, -0.20256566, 1.3953283]], + "bv": [0.37398052, -0.08708635, 0.68981785, 0.007922215, -0.36478022], + "bx": [-0.60999775, -0.043125022, 0.09639601, -0.17598106], + "by": [0.10892866, 0.35852543], + "network": { + "layers_0": { + "XY": [[0.5745754, 0.4309712], [1.0631208, 0.7834252], [0.33462834, 0.49034262], [0.95990264, 0.051912963], [-0.53545994, -0.44371206], [0.46453792, -0.29438478], [-0.21894448, 0.17704219]], + "a": [1.8289042], + "b": [0.36330286, -0.25053298], + "d": [-0.21073468, -0.00676942] + }, + "layers_1": { + "XY": [[0.14029512, 0.06191531], [-0.54320186, 1.0741305], [0.35935575, -0.070177436], [0.849095, 1.2216016]], + "a": [1.7343105], + "b": [-0.042095568, 0.0022068582], + "d": [-0.3272755, -0.10347981] + }, + "layers_2": { + "XY": [[0.06195046, 0.6797937, -0.07217559, 1.0908726, -1.0443761], [0.01065363, 0.40555507, 0.80447876, -0.4255162, 0.11247209], [0.7566825, 0.46551764, 0.4476699, -0.3723629, 0.27267793], [0.17611736, 0.14710239, 0.65267056, -0.16213208, -0.550812], [-0.7030909, 0.3038161, -0.5443954, -0.19139773, -0.81702024], [-0.8501901, -0.21552862, -0.1940229, 0.30334812, -0.27751637], [0.09072742, -0.330393, -0.39647946, 0.05497977, 0.3970092]], + "a": [2.1455698], + "b": [0.5096497, 0.38019773, -0.0500777, 0.3328637, -0.13144295], + "d": [-0.19693755, -0.18637931, 0.5163629, 0.07335614, 0.12705101] + } + }, + "p": [2.9304311] + }, + "inputs": { + "state": [[0.69551975, -0.9794742, -1.5734904, -2.9249706], [-0.35323218, 1.2476064, 0.033072624, 0.51184404], [1.0232382, -0.8821458, 2.6522868, -0.87690824], [0.37355307, 2.7395809, -0.11425241, 0.114294514]], + "u": [[-0.51187956, 0.33452648, -2.1266837], [-0.64680874, 1.6940932, 0.21132614], [-0.24039851, 0.3729285, 0.4146068], [0.15210326, 0.5785879, -1.6595968]] + }, + "outputs": { + "next_state": [[1.9647884, -2.9332001, -0.86908776, -1.2907408], [-1.8738557, 1.5284448, 1.2902621, 0.7065413], [-0.021606982, -0.15919553, 2.2353396, 2.1842916], [1.9826846, 0.028442394, -0.72717637, -1.041223]], + "y": [[3.0000129, 0.702162], [0.47598162, 0.62243783], [-0.1484341, -4.660478], [3.2753425, 1.2154456]] + } + }, + { + "name": "hidden4", + "config": { + "input_size": 3, + "state_size": 4, + "features": 5, + "output_size": 2, + "init_method": "long_memory", + "hidden": [4] + }, + "params": { + "B1": [[0.09764526, 0.38152823, -0.008821533, -0.5033277, -0.020694807], [-0.28073484, 0.2690167, -0.16214453, -0.05476699, 0.011791424], [-0.439656, -0.101026416, 0.27548802, -0.15787695, -0.037544146], [0.06070301, -0.19151564, -0.07848486, 0.14045958, -0.105044045]], + "B2": [[0.0649881, -0.86014825, -1.1602981], [0.0017789602, 1.2071329, 0.8773027], [0.060291864, 0.7519996, 0.7957585], [0.16760351, 0.509394, -0.07810506]], + "C1": [[-0.3727333, -0.066656865, 0.40903065, 0.47246012], [-0.06617911, -0.094097644, 0.36610287, 0.622875], [-0.2700966, 0.14740029, 0.050435577, -0.088875465], [-0.08785498, -0.092051655, 0.0064566224, -0.4527679], [0.007390957, 0.24376252, 0.23423612, 0.24544507]], + "C2": [[-0.9213795, 1.0687544, 0.8930746, -1.1470265], [-0.19589068, 1.3111302, -1.0037909, -0.2540472]], + "D12": [[0.13503292, 0.004396595, 0.095859125], [-1.0391248, 0.02388072, 0.28316107], [0.38978118, -0.069838144, 0.11352673], [0.5332985, 0.057132185, -0.019009903], [-0.18500304, 0.2467428, -0.15526581]], + "D21": [[0.77465403, 0.07849286, -1.1584561, -0.3257726, -0.7805643], [-0.23803759, 0.119731516, 0.3867677, -0.95279735, 0.49559262]], + "D22": [[0.067961425, 0.048225995, 0.13716802], [-0.09806482, 0.019369837, -0.32131392]], + "X": [[0.6969669, -0.2452424, -0.29355833, -0.40186638, 0.7742758, 0.19406895, 0.12988177, -0.11771295], [0.100447305, 1.1159565, 0.8049442, -0.46690866, 0.08215277, 0.6042062, 0.38289443, -0.055786956], [0.14113235, 0.06962689, 1.2181151, -0.85921395, -0.33389345, 0.035342783, 0.60172355, 0.1755331], [0.37883848, -0.3749713, -0.11667172, 1.198742, 0.55255735, 0.31255823, 0.22875969, 0.74588937], [0.020636411, 0.049621325, -0.16842286, 0.13515493, 0.2175687, -0.120124824, 0.645591, -0.03838687], [0.5691592, -0.103785194, -0.012095112, -0.36119512, -0.16212466, 0.6403376, 0.012780928, -0.5880356], [0.43086874, -0.2602018, -0.12099161, -0.008321112, -0.5578326, 0.4632574, 0.54795474, -0.096882835], [0.107314534, -0.084514566, -0.17687045, -0.06906295, -0.42971364, -0.12096443, -0.16700415, 0.6794503]], + "Y": [[0.5627811, 0.37738946, 0.21340717, 0.04423282], [-0.17955427, 1.5394981, 0.43517083, -0.012878272], [-0.0589835, 0.056329247, 1.54509, -0.2713933], [0.024822583, 0.33944523, -0.41485915, 0.9755173]], + "bv": [0.33285758, -0.13055955, -0.03339873, -0.4619882, 0.113580875], + "bx": [-0.35197884, -0.116500996, 0.10063728, -0.16713458], + "by": [0.25946948, 0.21325345], + "network": { + "layers_0": { + "XY": [[0.6668899, 0.45021144, 0.4478243, -0.085288465], [0.70571476, 0.54901606, 0.5624716, -0.77074754], [-0.012501538, -0.1233606, 0.21453384, -0.48076499], [0.6125616, -0.21080354, -0.7839485, 0.21017605], [-0.69865036, -0.8150569, -0.20429157, -0.10316605], [-0.20885624, -0.7714671, -0.04003805, 0.108066306], [-0.36253396, -0.040508166, 0.524986, -0.18322569], [-0.45858833, -0.36924252, 0.08627805, 0.9868925], [-0.45208353, 0.22239332, 0.2879066, -0.526748]], + "a": [2.0357568], + "b": [-0.03215566, -0.07911688, 0.06464072, -0.264083], + "d": [-0.34855446, -0.06429869, 0.0049017454, -0.15998586] + }, + "layers_1": { + "XY": [[0.27392668, -0.1550857, -0.018431157, 0.24383, 0.47922093], [-0.8890384, -0.20823157, 0.68107194, -0.6082294, 0.11968091], [-0.70476156, -0.42062655, -0.34937054, -0.5455848, 0.4282784], [-0.13080424, -0.10168231, -0.06744944, -0.22831437, -0.37113768], [-0.5492073, 0.26372045, 0.7035707, 0.58741224, 0.5073646], [0.25070298, -0.22071391, -1.2361772, -0.15988883, -0.6498689], [0.24275415, -0.6139609, -0.035212547, 0.014066875, -0.95321345], [-0.21731849, -0.08044971, -0.7543318, -0.51999116, -0.99443424], [-0.842175, -0.5112335, -0.44206345, -0.007782027, 0.15160304]], + "a": [2.0391707], + "b": [0.14042422, -0.48050532, -0.0060329945, -0.16369078, -0.034961153], + "d": [-0.27795535, -0.028673155, 0.3499635, -0.13983193, -0.20110732] + } + }, + "p": [1.638456] + }, + "inputs": { + "state": [[-0.28985086, 0.599883, 0.795565, 2.0819159], [-0.7232159, 1.6581454, 0.79662055, -1.5933194], [1.1074729, 1.7491307, 1.2299939, 0.3835589], [0.29981375, -1.8399895, 0.46211445, 0.3205261]], + "u": [[-0.78944135, 0.33786207, -0.5229359], [2.3901725, -1.0838655, -0.77111965], [-2.0358775, -0.28489777, 0.22182651], [0.11025763, -0.54545397, 0.96558595]] + }, + "outputs": { + "next_state": [[-0.5868579, 0.06263294, -0.28201422, 0.29812276], [1.0349193, -3.5763528, -0.60445917, -0.45957893], [-1.3622618, -0.35185865, 0.1384506, -0.76823217], [-1.0976853, 0.16344956, 0.26977372, -0.35589382]], + "y": [[-1.6254721, -0.49311382], [5.722006, 1.7519004], [0.8368622, 0.6320712], [-2.2226102, -3.2936835]] + } + } + ] +} diff --git a/tests/golden/ren_composition.json b/tests/golden/ren_composition.json new file mode 100644 index 0000000..278b914 --- /dev/null +++ b/tests/golden/ren_composition.json @@ -0,0 +1,326 @@ +{ + "network": "ren_composition", + "description": "Composition of bi-Lipschitz RENs and orthogonal layers.", + "cases": [ + { + "name": "plain", + "config": { + "input_size": 2, + "state_size": 3, + "features": 5, + "num_layers": 2, + "mu": 0.5, + "nu": 5.0, + "init_method": "long_memory", + "dyn_state_multiplier": 2 + }, + "params": { + "input_orth": { + "W": [[0.90548766, 0.22652519], [0.944981, 0.03288859]], + "a": [1.7983469], + "b": [0.22719422, 0.1500405] + }, + "orths_0": { + "W": [[-0.8386086, -0.177042], [-0.87648565, 0.6195766]], + "a": [1.0921131], + "b": [-0.21100816, 0.54883355] + }, + "orths_1": { + "W": [[0.35754085, -0.37856895], [-0.261814, -1.140745]], + "a": [1.4880507], + "b": [0.13506074, -0.38058895] + }, + "output_orth": { + "W": [[0.91753477, 0.19102], [0.9066955, 1.7214587]], + "a": [1.2757778], + "b": [-0.074249014, -0.16371515] + }, + "rens_0": { + "B2": [[-0.034037985, -0.24842916], [-0.23530996, 0.8239466], [-0.70430136, -0.5716756]], + "C2": [[-0.12123865, 0.56769466, 0.10714576], [-0.43680114, -0.88005215, -0.852255]], + "D12": [[-0.41372624, 0.0582048], [0.56492, -0.5560934], [-0.12912926, -0.24729201], [-0.37011427, -0.11273101], [-0.6725003, -0.005728662]], + "D21": [[-0.7396132, -1.0472865, -0.19229569, 0.5030453, 0.87108976], [1.0781027, 0.67002004, -0.4632571, 0.2990358, 1.2797451]], + "D22": [[-0.102697425, 0.12394374], [-0.38604668, -0.30786514]], + "X": [[1.1697491, -0.34908038, 0.064461924, 0.13881485, 0.042018756, 0.2934899, 0.5871854, 0.058665473, 1.2940228, 0.4542619, -0.13113886], [-0.19477202, 0.27596754, 0.13561493, 0.3230431, 0.13826023, 0.6381703, 0.4673285, 0.25363728, -0.0026771545, 0.72654915, 0.041812085], [0.15951638, -0.010948188, 0.91747177, -0.10652833, 0.21922094, -0.5040101, -0.086423315, 0.030564502, 0.2179263, 0.2200958, 0.9323657], [-0.16639537, -0.34265915, 0.35955393, 1.7610881, 0.54896736, -0.7344734, -0.1862135, 1.0843279, 0.006125121, 0.15154693, 0.5507796], [-0.45054597, -0.2231615, 0.12031513, 0.19741637, 0.6210078, -0.9205322, 0.33042228, 0.33715242, 0.12502138, 0.06870286, 0.36332962], [0.19157703, 0.45861492, -0.23711668, 0.13217397, -0.4200869, 0.20372021, -0.081808895, 0.1609186, 0.0638153, -0.48937818, 0.3952381], [0.29242033, -0.054049723, -0.07263355, -0.0035585577, -0.3670408, 0.70536435, 0.8374003, -0.43432632, -0.09857292, -0.20211895, -0.27408758], [0.09933368, -0.3732356, -0.31583396, -0.24507234, 0.24012648, -0.30146685, -0.30167776, 0.115550786, -0.34326783, -0.100739665, 0.26505813], [0.120957464, 0.00639327, 0.45558542, -0.1289352, -0.18597583, -0.53156674, 0.006414397, 0.18612623, -0.04632491, 0.56201494, 0.302758], [0.2739058, -0.36641425, 0.08419429, 0.106461585, -0.61799335, -0.011295421, -0.07667058, -0.5265496, 0.14381874, 0.80225265, 0.22167696], [-0.2602703, -0.08150001, 0.3338234, -0.25441703, 0.10333663, 0.0066488883, 0.051187262, 0.4951107, -0.46115896, 0.3497981, 0.20679344]], + "X3": [[0.9699762, -0.50335497], [0.09061443, 0.8746746]], + "Y1": [[0.11275661, -0.41536182, -0.3639689], [-0.30772588, 0.81260824, -0.2537632], [0.46653262, 0.319002, 0.97303957]], + "Y3": [[0.41607738, 0.64105827], [0.14561404, -0.39940602]], + "Z3": {"shape": [0, 2]}, + "bv": [0.11159396, 0.4292179, -0.30207583, -0.0035077909, 0.2810954], + "bx": [0.35574573, 0.25535285, 0.11173168], + "by": [-0.04277467, -0.17192885], + "p": [4.4773235] + }, + "rens_1": { + "B2": [[0.1939389, 0.35167173], [-1.127878, -0.02871418], [0.05721098, 0.9643136]], + "C2": [[-0.12166709, -0.22818643, 0.73306006], [-0.041508526, 0.7896854, -1.7680959]], + "D12": [[-0.4240115, 0.5203207], [0.2325173, 0.68210137], [0.29511812, 0.413097], [0.4403558, -0.10003999], [0.5722208, -0.46520823]], + "D21": [[0.077451535, -0.5949715, 0.01990825, -0.121279776, -0.4765614], [1.1770583, -0.76109356, -0.8878472, -1.1296653, 0.3489201]], + "D22": [[0.4571665, -0.198015], [-0.42145696, -0.5865956]], + "X": [[0.79898465, -0.028373113, 0.06833729, -0.48347685, -0.17778432, 0.29626563, 0.011117029, -0.11015107, 0.8647144, -0.4825532, -0.4301998], [-0.16636133, 1.3596146, 0.33779427, -0.054944664, 0.17932989, 0.11857205, 0.3402022, -0.21851477, 0.072340816, 1.2607348, -0.38543457], [-0.027777987, 0.06906553, 1.1198466, 0.15109624, 0.1726166, 0.29648218, 0.41411996, 0.10975582, 0.6569963, -0.14478193, 0.91939026], [0.09636517, -0.23526327, -0.44360855, 0.6907003, 1.6148925, -0.47124383, -1.3062283, 0.07116091, 0.111543804, 0.057569787, -0.28957888], [0.08038982, -0.15487458, 0.07106339, -0.010508324, 1.3715721, -0.45144442, 0.17724717, -0.29758728, -0.24268739, 0.11130745, 0.2070476], [0.23564303, -0.5841737, 0.512555, -0.07125304, -0.04234626, 1.4583793, -0.09739743, -0.60717773, 0.03294667, 0.083903834, 0.27911532], [0.38908708, -0.3092175, -0.37045428, -0.581252, -0.57563865, 0.08772481, 0.85557956, -0.98589206, 0.09785304, 0.08841103, 0.35611027], [0.50939775, -0.24152482, 0.44828537, 0.3203634, -0.23769389, 0.056988996, -0.12583433, -0.47143286, 0.015958356, -0.48325935, 0.017437343], [-0.30787995, -0.09523834, 0.24533851, 0.03936164, -0.23583907, 0.07253195, 0.21057203, 0.29373747, -0.06335813, 0.3423903, -0.047135916], [-0.16030253, 0.35687092, 0.19161667, 0.20840445, 0.044916384, -0.532428, 0.14384435, 0.1735868, -0.028575255, 0.6995434, 0.045772243], [0.5159933, 0.36349678, -0.19189367, -0.23789565, -0.083910376, -0.0033905103, 0.21814601, -0.16809238, -0.30193532, 0.6540398, 0.22626548]], + "X3": [[1.1009126, 0.5386337], [0.050961766, 0.32255203]], + "Y1": [[1.1604618, -0.16855448, 0.31599057], [-0.5206347, 1.0749557, 0.2598288], [0.44348508, 0.31062293, 1.3008468]], + "Y3": [[0.019003594, 0.01402658], [-0.27907598, 0.30908507]], + "Z3": {"shape": [0, 2]}, + "bv": [0.48536313, 0.20340338, -0.23897639, -0.39850327, -0.1335764], + "bx": [0.49133703, 0.07207829, -0.26821706], + "by": [-0.084342234, 0.26740295], + "p": [3.7036328] + } + }, + "inputs": { + "carry": { + "rens": [[[-1.430873, -0.9365477, 0.39393836], [-0.52408665, 0.5256162, 0.80732363], [-1.4435314, 1.0170637, -0.59564865], [2.094085, 0.72868896, -0.5389894]], [[0.2014878, 0.21497792, 0.3295181], [1.3932605, 0.532844, 1.6606375], [0.6342201, -0.17774108, -1.5244079], [-1.5605397, 0.20478748, 0.6909786]]], + "dyn_in": null, + "dyn_out": null + }, + "u": [[1.4433056, -0.6133017], [0.07777395, -0.8129783], [-0.42233336, -2.2794487], [0.7388424, -0.72260535]] + }, + "outputs": { + "carry": { + "rens": [[[-0.3660397, 0.15880063, 0.00044736266], [0.036673993, 0.61590123, 0.17654964], [-0.51233095, 0.27093, 0.03210792], [1.7893178, 0.7034454, 0.014463529]], [[0.44876438, 0.50365317, -0.12795065], [0.9454677, 1.5925939, 0.29631212], [-1.0013816, 1.6160669, 0.057855874], [-0.44210353, 0.9778185, -0.018672213]]], + "dyn_in": null, + "dyn_out": null + }, + "y": [[-1.2216825, -1.7355092], [-3.9162192, -3.0779958], [-3.9796667, 2.7313135], [-3.787119, -1.7968867]] + } + }, + { + "name": "single_layer_tanh", + "config": { + "input_size": 2, + "state_size": 3, + "features": 5, + "num_layers": 1, + "mu": 0.5, + "nu": 5.0, + "init_method": "long_memory", + "dyn_state_multiplier": 2, + "activation": "tanh" + }, + "params": { + "input_orth": { + "W": [[0.6080354, 0.21324682], [0.47848976, 0.5815927]], + "a": [1.2606953], + "b": [0.527532, -0.28574768] + }, + "orths_0": { + "W": [[-0.29047346, 0.74285054], [-0.11651683, 0.5489019]], + "a": [1.1531724], + "b": [0.06703662, -0.018006446] + }, + "output_orth": { + "W": [[0.9959489, -0.12705973], [0.71283805, 1.7561897]], + "a": [1.6873204], + "b": [-0.081635706, 0.08692985] + }, + "rens_0": { + "B2": [[-0.17238617, -0.9431958], [-0.69857436, 0.57450026], [-0.7798148, -1.3744943]], + "C2": [[-0.4474784, 0.30388245, 0.7744658], [-0.080657884, -0.34619182, -0.16719487]], + "D12": [[-0.0740574, 0.8742485], [0.25093067, -0.4040264], [0.27773798, -0.38112858], [-0.74707854, 0.14851071], [0.07109031, -0.4128703]], + "D21": [[-0.8704101, -1.0819156, -0.3138271, 0.054530285, 0.2160817], [0.95126766, -0.049327336, 0.0019182861, -0.753124, 0.52487314]], + "D22": [[0.3495763, -0.17183444], [-0.24241568, -0.07214588]], + "X": [[1.5640178, 0.09506073, 0.43434542, -0.26123792, -0.045662638, 0.21881598, 0.08676931, -0.14666514, 0.33412474, 0.16952632, 0.23468678], [0.15189712, 0.50354683, -0.41593635, -0.16862541, -0.2999938, 0.31767017, -0.3223294, -0.09502228, -0.32752717, 0.46487242, 0.2992609], [0.049600568, -0.20849791, 1.0077027, -0.6487578, 0.47039148, -0.30816165, -0.56968325, -0.31588635, 0.066702425, 0.0024830645, 1.1907564], [0.20722955, 0.013317392, 0.30646893, 1.0308412, 0.6174767, -0.38635486, 0.2463236, 0.67569387, -0.1594967, -0.18098652, -0.5078777], [-0.15171437, 0.3190161, 0.4304491, -0.14504822, 0.76066375, -1.060412, 0.4569256, 0.49908444, 0.13876498, 0.42358306, -0.6065286], [-0.32168752, -0.14554523, -0.021926455, 0.11569685, 0.52634925, 0.41780534, 0.52219534, -0.084451355, 0.017328406, 0.09038574, 0.2558873], [0.037687786, -0.32765597, 0.14193663, 0.14622086, -0.46188208, 0.13138874, 0.95663244, -0.42512396, 0.026955018, 0.5184132, -0.49430355], [0.055393826, -0.5575651, 0.005151312, 0.27405807, -0.5400353, 0.34080064, 0.14304842, -0.014007358, -0.05069506, -0.0007541528, 0.023206793], [-0.00574179, 0.3205648, 0.24713404, -0.25696853, 0.33401164, 0.5602396, 0.49871323, 0.105458885, -0.29373312, -0.13397552, -0.013921797], [-0.0024815488, -0.41562173, 0.20566075, 0.2947915, -0.37485746, -0.19584098, -0.2707461, 0.043824356, 0.37656295, -0.39682192, 0.5068397], [0.33075726, -0.35076365, 0.52650917, 0.39365608, 0.20248036, 0.14027743, -0.22714022, 0.18955052, -0.20410827, 0.061864052, 0.48356888]], + "X3": [[0.93193305, -0.14760293], [0.19968203, 1.3289068]], + "Y1": [[1.3385136, -0.022865731, 0.458543], [0.5357285, 0.9432449, 0.22163855], [-0.49209517, -0.0054663233, 0.726336]], + "Y3": [[0.07906228, -0.14727548], [-0.6638723, 0.22170116]], + "Z3": {"shape": [0, 2]}, + "bv": [-0.26719907, 0.39023358, 0.4706348, 0.21138634, 0.011520501], + "bx": [0.10689885, -0.39189172, 0.06121092], + "by": [0.3247453, 0.24593802], + "p": [3.0453913] + } + }, + "inputs": { + "carry": { + "rens": [[[-0.39977813, -0.46264333, -0.8151618], [2.0070026, -0.72403944, 0.34630892], [1.2521546, -0.033779386, -0.32801417], [-0.07659477, 0.07505486, -0.129829]]], + "dyn_in": null, + "dyn_out": null + }, + "u": [[-0.36331755, -0.30605167], [0.18775529, 1.8282351], [-0.33249187, 0.8723645], [-0.10737055, -0.8790488]] + }, + "outputs": { + "carry": { + "rens": [[[0.10901423, -0.11972022, -0.3320217], [-0.31514162, -1.0879521, -0.4019963], [0.04807638, -0.53012204, -0.28673643], [-0.034107633, -0.0038553476, 0.18286784]]], + "dyn_in": null, + "dyn_out": null + }, + "y": [[0.16147694, -0.8149369], [1.515608, 4.3857913], [0.18141076, 1.8846686], [1.123688, -1.9721439]] + } + }, + { + "name": "dyn_in", + "config": { + "input_size": 2, + "state_size": 3, + "features": 5, + "num_layers": 2, + "mu": 0.5, + "nu": 5.0, + "init_method": "long_memory", + "dyn_state_multiplier": 2, + "dyn_orth_at_input": true + }, + "params": { + "dyn_in": { + "X": [[-0.5674996, 0.22052681, -0.59637296, -0.8918626, -0.2784689, 0.15098727, -0.2399278, -0.7332794], [-0.10820764, 0.018892705, 0.31319106, -0.17450652, -0.7023668, -0.19594274, -0.4431529, -0.95805275], [-0.0065456033, -0.06402299, 0.07703793, 0.3684168, -0.22760382, -0.21623333, 0.17309767, -0.1387924], [-0.5355333, 0.94057167, 0.20291762, 0.33655855, 0.20446143, -0.42505476, 0.41637653, -0.2678801], [0.70385, 0.22447823, -0.5909842, 0.028665517, -0.4250727, -0.29008237, -0.62283003, 0.8185555], [0.296013, -0.37991512, 0.013176814, 0.43509305, -0.45707625, -0.50160706, -0.95881355, -0.17780848], [-0.06659308, 0.33972263, -0.15315408, -0.21556486, 0.24446288, 0.75725126, -0.5596497, -0.13226694], [-0.37187228, -0.80518883, -0.20353475, 0.16659133, 0.122904174, 0.027961923, -0.44986787, -0.38028342]], + "bx": [0.06220214, 0.12857263, -0.13847394, 0.4907466, -0.14238545, -0.010351334], + "by": [0.5649558, -0.09389245] + }, + "orths_0": { + "W": [[-0.6850306, -0.13938281], [-0.39429426, 0.54693943]], + "a": [1.1139845], + "b": [-0.32599184, 0.33858997] + }, + "orths_1": { + "W": [[0.40929502, -0.4288528], [-0.6514607, -1.3449794]], + "a": [1.2944803], + "b": [-0.017530868, -0.1525936] + }, + "output_orth": { + "W": [[1.166057, -0.056015886], [0.6464591, 1.5306649]], + "a": [1.8015125], + "b": [0.11474373, 0.22581771] + }, + "rens_0": { + "B2": [[-0.022101883, -0.41350004], [-0.8154453, 0.6593988], [-0.31762236, -0.61679566]], + "C2": [[-0.84732646, 0.032432705, 0.13402349], [0.1789064, -0.6550042, -0.37448117]], + "D12": [[-0.4649351, 1.0744014], [0.55598354, -0.5583328], [-0.26495692, 0.46536744], [-0.5754589, -0.14860763], [-0.068960965, -0.23906875]], + "D21": [[-0.31330335, -1.0695972, -0.54364556, -0.2859279, 0.4442208], [0.80482274, 0.025006687, -0.5915194, 0.042138815, 0.25524735]], + "D22": [[0.11762129, -0.03040021], [0.28245953, -0.30275148]], + "X": [[1.2796719, -0.26436654, 0.051674332, -0.08025943, 0.6747231, -0.40478805, -0.22084494, -0.5643343, 0.6484857, -0.09256766, -0.14954816], [-0.28686363, 0.75737774, -0.108886495, -0.012050898, 0.43948552, 0.07058898, -0.057364013, -0.15083228, 0.15215777, 1.3226279, -0.123431094], [0.23641877, 0.107638456, 0.33640265, 0.5237594, 0.23208518, -0.44719476, -0.61041063, 0.019491192, -0.18511796, 0.113185726, 0.8410182], [0.32910737, -0.1993889, 0.09404443, 1.7432616, 0.36708805, -0.28766137, 0.15494682, 0.6995019, -0.084627345, -0.72263205, -0.3209995], [0.52061534, -0.13118683, -0.25385594, -0.07696839, 0.8417995, -0.31013697, 0.15805206, 0.27479154, 0.17120722, -0.5407791, -0.49265823], [-0.39088127, -0.22324684, 0.28488284, 0.37573844, 0.13441029, 1.2646853, -0.23707038, 0.24031349, -0.21756813, 0.13021825, -0.09413169], [-0.43498823, 0.32933217, 0.41435018, 0.06712881, -0.25162026, 0.5923953, 0.2781574, -0.32102683, -0.26834008, -0.21507701, 0.28290796], [-0.13900614, 0.2910327, 0.070333526, 0.40935072, 0.28002197, 0.25016847, 0.0074657258, 0.00213732, -0.28825822, 0.12968914, 0.21222386], [-0.20478626, 0.11184465, -0.2307844, -0.01286451, -0.32225657, 0.00025073116, 0.31399447, 0.08002603, -0.35893416, 0.19195351, -0.10182731], [0.37406906, 0.20341367, 0.19868635, -0.092811346, 0.040634003, -0.2266006, 0.021886388, 0.22622101, -0.44362924, 0.088368334, 0.21333194], [-0.30709904, -0.049029782, 0.13943727, 0.09251835, 0.1745535, 0.22315569, 0.35154316, -0.09333421, 0.14240779, 0.117415294, -0.2208446]], + "X3": [[0.8776454, 0.21221633], [0.1023274, 1.2021722]], + "Y1": [[0.99415094, -0.0118653625, 0.27771753], [-0.57428354, 0.50792813, 0.010176763], [0.07481049, 0.19226705, 1.1940999]], + "Y3": [[0.05368992, -0.34421322], [0.14919706, 0.05715011]], + "Z3": {"shape": [0, 2]}, + "bv": [-0.2960913, -0.49903408, -0.13651364, 0.19209653, -0.00059173617], + "bx": [0.33136204, -0.115240864, -0.21574818], + "by": [-0.18159983, 0.006342528], + "p": [3.8470378] + }, + "rens_1": { + "B2": [[0.17321585, 0.45677823], [-1.2720497, -0.6487513], [-0.17745934, 1.1486102]], + "C2": [[0.4180043, -0.44034073, 0.35891283], [-0.48066932, 0.35923815, -1.2620618]], + "D12": [[0.040674597, 0.63059163], [0.49079087, -0.66843766], [0.43944117, 0.25850275], [0.91100085, -0.15821294], [0.6332799, -0.6626997]], + "D21": [[0.29025495, -0.19784117, 0.2209379, 0.28236115, -0.97322774], [1.250076, -0.8107552, -0.077922896, -1.7930951, 0.55834895]], + "D22": [[0.49159086, -0.29410914], [0.27726933, -0.24592099]], + "X": [[1.268576, 0.34156793, -0.33061397, 0.49545074, -0.01632788, 0.028623814, 0.24192582, 0.2914417, 1.0161631, 0.20777841, 0.21196583], [-0.29890215, 1.1037375, 0.08561203, 0.030990334, 0.22586532, -0.1269642, -0.4916682, 0.20670134, 0.1355388, 1.0036789, -0.120847344], [0.6660259, 0.05129826, 0.7198185, 0.11175146, 0.17157984, -0.38302597, 0.20987274, -0.59064627, -0.10096772, 0.27114752, 1.322229], [0.12973735, 0.2498565, -0.22164306, 1.130237, 1.482722, -0.7603673, -0.80035764, -0.292243, 0.34343252, 0.24809569, 0.31652743], [0.26830664, -0.24116677, -0.24664739, -0.15191512, 1.1002499, -0.25480482, 0.9680488, 0.23591371, 0.083919674, 0.32908583, 0.035932343], [0.15887651, 0.3501285, -0.28497234, -0.15534401, 0.13980137, 2.3551438, 0.1325835, -0.72703993, 0.43873706, -0.12056681, -0.5991538], [-0.3164459, -0.28030777, 0.526928, -0.06318727, -0.06415963, -0.06266703, 0.75845164, -0.99868083, 0.012545253, 0.33418587, 0.045817405], [-0.4333303, 0.22222291, 0.13119997, -0.12184975, 0.051419105, -0.19887961, 0.64351696, 0.3479557, 0.07266938, -0.25340402, 0.1770312], [0.17047705, 0.008875574, -0.088338494, 0.57538736, -0.11283113, 0.26419646, 0.35408926, -0.31813735, 0.24388689, 0.2916007, 0.01724417], [-0.4257629, 0.18122074, 0.5928315, 0.14320156, 0.09907476, -0.21441236, -0.04770086, 0.13947938, -0.37848946, -0.07654172, 0.021239], [-0.0015302274, 0.028761582, -0.014395042, 0.40710655, -0.030797932, -0.01262736, -0.23285992, -0.2396692, -0.030339675, 0.7367709, 0.52827215]], + "X3": [[0.62829435, 0.46231872], [0.007761196, 0.6380253]], + "Y1": [[1.0862198, -0.2745791, -0.18745518], [-0.2766907, 0.95872325, -0.5785106], [0.28273866, 0.48502293, 0.9566299]], + "Y3": [[0.021587329, 0.1299331], [-0.057039753, -0.067234755]], + "Z3": {"shape": [0, 2]}, + "bv": [0.20520727, 0.30455214, -0.10592476, 0.20624241, 0.10000197], + "bx": [-0.16646117, -0.010869117, -0.53879803], + "by": [-0.31972876, 0.088177264], + "p": [4.6463323] + } + }, + "inputs": { + "carry": { + "rens": [[[1.6289308, 0.6234109, -0.83519083], [0.56229013, -0.82993746, -0.37776607], [0.14877732, 1.9583529, 2.5929193], [-1.4061909, 0.032713454, -0.06492732]], [[0.7435053, 0.44836333, -0.52381724], [0.394098, -0.57497746, -0.74969], [0.62576133, 0.24604008, -0.53462], [0.70938665, -0.33913225, -0.7180063]]], + "dyn_in": [[-0.0679673, 0.66473395, -0.56095886, 1.4887801, -0.1871171, 0.34055045], [0.8316214, 0.047234662, -0.5871153, -0.15932207, 0.9937553, 1.2939177], [0.67534935, -0.40718165, -0.911703, -0.51674277, -0.4108272, 0.2784929], [0.030033853, -0.7178539, 0.74603283, -0.6651096, 0.13022888, -0.687465]], + "dyn_out": null + }, + "u": [[1.0916207, 0.4599091], [0.25614944, 1.1476514], [1.0012176, -1.2107041], [0.26084974, -0.96309185]] + }, + "outputs": { + "carry": { + "rens": [[[0.20990247, -1.053519, -0.9611001], [0.49473315, -0.7169544, -0.4452259], [-0.65370584, -0.87942463, -0.48598188], [-0.07090771, 0.52886766, -0.92999613]], [[0.63233566, 0.6852012, -0.64769715], [0.17665043, 0.6422953, -0.57942325], [0.7156319, 0.7309953, -0.48981172], [0.62072176, 0.06168787, -0.2838608]]], + "dyn_in": [[-0.6485274, 1.5965654, 0.62199986, 0.35115469, -0.68767834, 0.5873409], [0.053753868, 0.28764838, -0.59787536, 1.2982904, -1.7784702, -0.47764218], [-0.5089708, -0.27287018, -1.1050677, 0.42508867, 0.6728139, 0.80602646], [1.0123504, -0.40324265, -0.7554263, 0.01476562, 0.8128035, 0.45424095]], + "dyn_out": null + }, + "y": [[-2.0739083, 0.361184], [-2.6419199, 2.2351105], [-1.9535317, -4.1864247], [1.347527, 0.7107525]] + } + }, + { + "name": "dyn_in_out", + "config": { + "input_size": 2, + "state_size": 3, + "features": 5, + "num_layers": 2, + "mu": 0.5, + "nu": 5.0, + "init_method": "long_memory", + "dyn_state_multiplier": 2, + "dyn_orth_at_input": true, + "dyn_orth_at_output": true + }, + "params": { + "dyn_in": { + "X": [[-0.113830656, 0.008300662, -0.4956172, -0.7078068, -0.6252345, 0.0869465, -0.16975674, -0.8091755], [-0.52942896, -0.44363898, -0.09743115, -0.114129335, -0.6234137, -0.41495872, -0.06407042, -0.35579306], [0.6540758, -0.20457266, 0.15541019, 0.5325308, 0.15087175, -0.033654258, 0.26224813, 0.08408859], [-0.025500055, 0.6109068, -0.45742288, 0.7483996, -0.7031579, -0.08722776, -0.10980196, -0.45978808], [-0.038004518, -0.05643873, -0.7931187, 0.2525794, 0.5284625, -0.022324651, -0.97043365, 0.44106445], [0.18618204, -0.3720205, 0.51813805, -0.102600634, -0.1486159, -0.9321691, -0.914526, -0.16585548], [0.7998458, 0.06658572, 0.6114011, 0.038436133, -0.28517437, 0.34646252, -0.5536788, 0.30968037], [-0.45155054, -0.32099953, 0.29999888, 0.12839326, 0.2730354, 0.15169305, -1.0164819, 0.09360172]], + "bx": [-0.12980887, 0.46448964, -0.12667605, 0.41332301, 0.1690268, -0.42666805], + "by": [0.3519741, 0.22017452] + }, + "dyn_out": { + "X": [[0.69582427, 0.2071739, 0.9330268, -0.41512722, 0.27793166, 0.2691482, 0.13136652, -0.66703683], [0.6367131, 0.47842997, -0.8577225, -0.09842178, -0.10323398, 0.072117954, -1.0735923, -0.5307909], [0.16089027, -0.15719312, -0.8309213, -0.37335122, 0.26451692, 0.30280438, 0.6801961, 0.2485137], [-0.31417814, 0.22469257, -0.124978796, -0.0567919, 0.089687556, 0.05856532, -0.15348724, 0.17756154], [0.10830906, 0.18148026, 0.14737937, 0.00027766638, -0.17314768, 0.0430907, -1.4876238, -0.1770656], [0.37870237, 0.27790383, -0.31337956, -0.035225764, -0.22174433, -0.06989761, 0.49794698, 0.27467102], [0.15031424, -0.16048606, 0.97099614, -0.24238926, 0.6595189, -0.2506101, 0.52528644, -0.46210778], [0.06246999, 0.34272838, 0.33843964, 0.79093575, 0.11403534, 0.26421648, -0.5221273, -0.13926545]], + "bx": [0.19503903, 0.31442058, 0.86458886, 0.71300995, -0.30127195, 0.24801417], + "by": [0.43978116, 0.21505757] + }, + "orths_0": { + "W": [[-0.65241253, 0.1189578], [-0.1877806, 0.90174055]], + "a": [1.1397276], + "b": [-0.25692514, -0.56220096] + }, + "orths_1": { + "W": [[0.77703595, -0.9605551], [-0.577621, -1.2199012]], + "a": [1.4795125], + "b": [-0.13310489, 0.02854462] + }, + "output_orth": { + "W": [[1.4619374, 0.18902908], [0.7254678, 0.87899846]], + "a": [2.0650609], + "b": [0.30251327, 0.3562203] + }, + "rens_0": { + "B2": [[0.054174405, -0.65318644], [-0.7295761, 0.9067675], [-0.18251127, -0.6430857]], + "C2": [[0.090764105, 0.82305145, 0.64511746], [-0.14380023, -1.3277757, -0.50916183]], + "D12": [[-0.06305957, -0.065710306], [0.44254723, -0.45935842], [-0.1826645, 0.3255638], [-0.7301792, 0.620517], [-0.5636616, -1.0050523]], + "D21": [[-0.3676541, -0.76723003, 0.0806441, 0.19305658, 0.2884171], [1.3248492, -0.050624758, -0.7445502, -0.007107407, 1.0809685]], + "D22": [[-0.07546679, 0.3044264], [0.55183494, -0.1999157]], + "X": [[0.07829046, 0.1680845, -0.07869963, 0.28038922, 0.061471794, -0.13024892, -0.31141368, -0.35884342, 0.8129771, -0.2897365, 0.23558958], [0.20773782, 0.8064886, 0.36139253, -0.11295193, -0.15453024, 0.2616988, 0.08064329, -0.17840712, 0.250711, 1.4850813, -0.39790723], [-0.08311272, -0.63822854, 0.9651666, 0.7439834, 0.072119676, -0.047244277, 0.04964077, -0.30104604, 0.05027415, -0.11165304, 0.7541736], [0.26302305, -0.070003286, 0.07441694, 1.3749049, 0.5238697, -0.35300547, 0.12935701, 1.1962031, -0.01164717, 0.117512465, -0.031283755], [0.2962633, -0.17692047, -0.30090195, 0.015833981, 0.821418, -1.0977789, 0.9798564, 0.41782194, 0.08118374, -0.3748903, -0.25737816], [-0.07491755, 0.45559567, -0.12154767, 0.07921872, -0.5946652, 0.5093992, 0.25917104, 0.1987545, 0.156701, -0.48817855, 0.30018148], [0.20381844, 0.24151404, -0.3461597, 0.5136331, -0.56243736, -0.022785114, 0.97660774, -0.21384834, 0.055567063, -0.17794941, 0.4862937], [-0.5894742, -0.2191909, -0.13104063, 0.3092327, 0.09892658, -0.5196171, 0.6584891, -0.012409499, 0.3709154, 0.6199518, -0.11981768], [0.10373243, -0.45649862, -0.21186484, 0.47770432, -0.07607683, 0.29802123, -0.00024257784, 0.22024608, 0.3640244, 0.63825864, 0.18213588], [0.05911485, 0.18500337, -0.17565441, -0.3134597, -0.7604626, 0.21307386, 0.08361583, -0.10039263, 0.28413117, 0.31327924, -0.08666432], [0.22755979, 0.18630914, 0.24153326, 0.36000714, 0.12654096, 0.22803487, -0.014937872, -0.28393513, -0.04767771, 0.019899257, -0.16082808]], + "X3": [[0.9894759, 0.09767487], [-0.26967448, 0.9720078]], + "Y1": [[1.0697607, -0.009380096, 0.3834455], [-0.3149284, 1.6593766, -0.13502507], [0.5295381, 0.04295586, 1.5899093]], + "Y3": [[-0.008594824, -0.0752022], [-0.3620891, -0.16613829]], + "Z3": {"shape": [0, 2]}, + "bv": [0.22414492, 0.050843213, 0.06828154, -0.004492135, 0.0010797475], + "bx": [0.21102266, 0.04665523, -0.23062386], + "by": [0.34871894, -0.0059112273], + "p": [3.5941415] + }, + "rens_1": { + "B2": [[-0.020115227, 0.3498252], [-1.0949469, -0.7205818], [0.055401593, 0.7540467]], + "C2": [[0.4410625, -0.76059973, 0.066307485], [-0.015550017, 0.5402415, -1.1027021]], + "D12": [[-0.41925618, 0.61214733], [0.7142761, -0.015666507], [0.64156264, 0.601846], [0.25083056, -0.6655468], [0.6489047, -0.036532223]], + "D21": [[-0.09266995, -0.1484161, -0.03043878, -0.5064137, -0.23801458], [1.0705209, -1.0629501, -0.64314085, -0.8640806, 0.8287241]], + "D22": [[-0.09941446, 0.5332548], [-0.3302614, -0.81943655]], + "X": [[0.92416763, 0.38042766, 0.04537584, -0.1408846, 0.23476325, -0.09225927, 0.09073088, -0.3487509, 1.2232606, -0.26419687, -0.16606043], [0.5549411, 1.0238411, 0.5222904, -0.42489657, -0.019948293, 0.11922358, 0.2947457, 0.41664782, 0.2032436, 1.3203535, 0.06129067], [0.33474198, 0.31138128, 1.1577471, 0.35752743, 0.04744574, -0.025750998, -0.23817109, -0.12078688, 0.1556771, -0.539336, 1.130973], [-0.12327442, 0.2811446, -0.2687748, 1.1159705, 0.9190607, -0.4433362, -0.7959468, 0.04981774, 0.17360133, 0.0596889, -0.36897105], [0.16526166, -0.26416036, 0.3038195, 0.22698335, 1.7446296, -1.2574042, 0.07751024, -0.40628138, -0.08432042, -0.086083844, -0.06578472], [0.15517052, 0.27388346, -0.13057972, -0.3410407, -0.017990973, 1.5319892, -0.04245657, -1.3381066, 0.18871856, 0.114836246, 0.12746188], [-0.32776, 0.26880148, 0.016941698, 0.12699717, 0.11432173, -0.31492412, 0.77753174, -0.736037, 0.038478717, 0.10917195, -0.31488612], [-0.44607547, -0.056819096, -0.7862344, 0.2705184, -0.26780117, 0.5041274, -0.25957954, -0.049248576, -0.4191514, -0.46857283, 0.03829936], [0.13584435, -0.46519127, 0.12677202, -0.21992682, -0.3124956, 0.0063360683, 0.2958328, -0.13071643, -0.03713368, 0.020180555, -0.25089872], [0.14204146, -0.22327422, -0.08960156, -0.63651025, -0.22817361, -0.5499812, 0.40153912, -0.27559987, -0.023005223, 0.18556383, -0.14457735], [-0.172755, 0.1922454, 0.31891713, 0.21642233, 0.046588473, -0.19790064, -0.019111304, 0.11729901, 0.4990204, 0.07963071, -0.04202564]], + "X3": [[0.74604774, -0.087629005], [-0.07039618, 1.435596]], + "Y1": [[1.0952003, 0.20699485, -0.1940164], [-0.41893932, 1.7624843, 0.6612988], [0.20788583, 0.280829, 0.8141998]], + "Y3": [[0.4383585, 0.9470881], [-0.014017383, 0.13182984]], + "Z3": {"shape": [0, 2]}, + "bv": [0.2711335, -0.11566371, -0.04459234, 0.07047892, -0.24216372], + "bx": [0.17545527, -0.35486922, -0.004232366], + "by": [-0.032851133, 0.10717332], + "p": [3.695745] + } + }, + "inputs": { + "carry": { + "rens": [[[1.0333982, 1.4243401, -1.0447086], [0.3021436, -1.5511218, -0.06074756], [-0.5504491, 0.4220689, -0.11526459], [0.042535007, -0.5438192, -0.012388292]], [[1.2584535, 0.75412124, 0.033225477], [-1.5662029, -1.6860622, -0.045609806], [0.3395537, 0.55708987, 0.23113416], [-0.726338, -1.6257595, 0.7857318]]], + "dyn_in": [[0.5438006, 1.1454773, -1.5704006, 0.6059825, 0.66254705, -1.0263681], [-0.940008, -1.0009769, -0.7807017, -0.110210985, -0.38089356, 1.2668165], [0.3789504, -0.5834237, 0.20578973, -0.69323546, -2.1293254, 0.2951065], [-0.5471568, 1.1826532, -0.09893206, 1.3153431, 0.7906134, -1.4501395]], + "dyn_out": [[1.6005592, -0.39086962, -0.04664974, -1.0772833, -0.5385725, 2.5245478], [0.15823023, -3.4155445, 1.6961215, 0.2052473, -0.60067254, 0.6370239], [0.36031926, -0.6385361, 1.4929024, 0.545123, 1.0904428, 0.21687771], [-0.029915571, 0.17139404, -0.94692034, -0.83010423, 0.8584114, -0.08894739]] + }, + "u": [[0.55959296, -0.014358907], [-0.4480066, 1.0207142], [-0.563474, -0.91396356], [-0.026693279, 1.0024941]] + }, + "outputs": { + "carry": { + "rens": [[[0.41381925, 0.19900045, -0.6955285], [-0.3086776, -0.27806953, 0.18259774], [0.52192336, -0.10766011, -0.31512788], [-0.06538774, -0.15267503, -0.22940992]], [[1.4805727, -0.0036133826, 0.16158642], [-1.4084945, -0.60549104, 0.04157417], [0.5850094, -0.086868554, 0.07260629], [-0.64394146, -0.4388168, 0.5061908]]], + "dyn_in": [[-1.3740478, 1.2116964, 0.112618536, -0.44796273, -0.65906096, 0.40673083], [-0.4510114, -0.8683465, 0.4749479, 1.7138023, -0.30133468, -0.09935039], [0.76472366, -1.1751246, 0.58735394, -0.32882002, 1.2446623, -0.80793405], [-0.5502387, 1.9188077, -0.08080202, 0.8641455, -1.5761971, -0.79175794]], + "dyn_out": [[3.5051844, 2.0083528, 1.2446519, -0.6867968, 2.5044897, 1.339299], [0.6912447, 2.8976521, 2.3130302, 2.182117, -0.086460024, 2.9965384], [-1.4124367, 0.44910923, 0.41551095, 0.239328, -0.19562218, 1.759753], [2.1841087, 0.5657331, 1.0429177, 1.0068431, 0.08896804, 0.07119392]] + }, + "y": [[-0.7208648, -0.5568476], [1.9714216, 1.4834958], [-0.11525372, 0.010774031], [-1.4742571, 0.37679225]] + } + } + ] +} diff --git a/tests/golden/unitary.json b/tests/golden/unitary.json new file mode 100644 index 0000000..bfa210d --- /dev/null +++ b/tests/golden/unitary.json @@ -0,0 +1,60 @@ +{ + "network": "unitary", + "description": "Cayley-parameterised orthogonal layer.", + "cases": [ + { + "name": "size3_bias", + "config": { + "input_size": 3, + "use_bias": true + }, + "params": { + "W": [[-0.1732375, -1.0011562, -1.1303244], [0.62248665, 0.7141395, 0.4424495], [-0.1291475, 0.45711297, 0.53315127]], + "a": [1.7525063], + "b": [0.38821915, 0.30201733, -0.8133488] + }, + "inputs": { + "x": [[0.34558418, 0.82161814, 0.33043706], [-1.3031572, 0.9053559, 0.44637457], [-0.5369532, 0.5811181, 0.3645724], [0.2941325, 0.028422242, 0.546713]] + }, + "outputs": { + "y": [[0.9852127, -0.23452526, -1.3226565], [1.8950858, 0.9233431, -0.5674919], [1.2479638, 0.424455, -0.7442236], [0.5245208, -0.27937406, -0.641236]] + } + }, + { + "name": "size1_bias", + "config": { + "input_size": 1, + "use_bias": true + }, + "params": { + "W": [[0.14781319]], + "a": [0.1261887], + "b": [-0.33360624] + }, + "inputs": { + "x": [[-1.8890133], [-0.1747721], [-0.4221904], [0.213643]] + }, + "outputs": { + "y": [[-2.2226195], [-0.5083783], [-0.7557967], [-0.11996324]] + } + }, + { + "name": "size5_bias", + "config": { + "input_size": 5, + "use_bias": true + }, + "params": { + "W": [[0.25644296, -0.88806736, -0.49976292, 0.02148056, 0.8182796], [0.40466166, -0.2929628, 0.31967443, 0.4273158, -0.03000369], [0.25417, -0.022405654, -0.3582716, 0.10055107, 1.0354909], [-0.6361142, -0.30021387, 0.36802354, 1.1174881, -0.8902899], [-0.0026487857, -0.11845435, 0.057094105, -0.16856101, 0.48294613]], + "a": [1.8833774], + "b": [-0.03300323, -0.13374844, 0.23259716, 0.058089856, -0.4892548] + }, + "inputs": { + "x": [[-0.37760502, 2.0427716, 0.646703, 0.66306335, -0.5140064], [-1.6480752, 0.16746475, 0.10901409, -1.227352, -0.68322664], [-0.07204368, -0.9447516, -0.09826997, 0.09548303, 0.035586238], [-0.5062917, 0.5937481, 0.8911669, 0.32084832, -0.8182302]] + }, + "outputs": { + "y": [[1.2404821, -0.54031503, 1.385833, 1.5548338, -0.20583978], [1.186876, 1.345527, 0.677386, -0.65079457, 0.10450968], [-0.6207451, -0.07311009, 0.23308565, -0.68108934, -0.63852817], [0.74568915, -0.48159507, 1.2934502, 0.12307829, 0.0751023]] + } + } + ] +} diff --git a/tests/jax_tests/test_bilipnet.py b/tests/jax_tests/test_bilipnet.py new file mode 100644 index 0000000..e2b61ea --- /dev/null +++ b/tests/jax_tests/test_bilipnet.py @@ -0,0 +1,76 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `BiLipNet`.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +import golden +from jax_utils import check_golden, empirical_gains +from robustnn.bilipnet_jax import BiLipNet + + +# ----------------------------- BiLipNet ---------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("depth", [1, 2, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_bilipnet_shape_explicit_and_bounds(key, batches, depth, use_bias): + size, mu, nu = 3, 0.5, 4.0 + net = BiLipNet(input_size=size, units=[4, 4], mu=mu, nu=nu, tau=nu / mu, + is_mu_fixed=True, is_nu_fixed=True, depth=depth, + use_bias=use_bias) + x = jax.random.normal(key, (batches, size)) + params = net.init(key, x) + + y = net.apply(params, x) + assert y.shape == (batches, size) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + lo, hi, _ = net.get_bounds(params) + np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-5) + gains = empirical_gains(lambda u: net.apply(params, u), key, 200, size) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_bilipnet_inverse_round_trip(key, depth): + net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True, depth=depth, act_fn=nn.relu) + x = jax.random.normal(key, (6, 3)) + params = net.init(key, x) + y = net.apply(params, x) + inv_act = nn.relu + e_inv = net.direct_to_explicit_inverse( + params, [0.1] * depth, [inv_act] * depth, [500] * depth, [1.0] * depth) + x_rec = net.inverse_call(params, y, e_inv) + np.testing.assert_allclose(x_rec, x, atol=1e-3) + + +def test_bilipnet_jit_and_grad(key): + net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True) + x = jax.random.normal(key, (5, 3)) + params = net.init(key, x) + np.testing.assert_allclose(jax.jit(net.apply)(params, x), + net.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +def test_bilipnet_fixing_all_raises(key): + net = BiLipNet(input_size=2, units=[2], is_mu_fixed=True, + is_nu_fixed=True, is_tau_fixed=True) + with pytest.raises(ValueError): + net.init(key, jnp.ones((1, 2))) + + +@pytest.mark.parametrize("case", golden.cases("bilipnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipnet_matches_golden(case, explicit): + check_golden("bilipnet", case, explicit) diff --git a/tests/jax_tests/test_bilipschitz_ren.py b/tests/jax_tests/test_bilipschitz_ren.py new file mode 100644 index 0000000..622df4e --- /dev/null +++ b/tests/jax_tests/test_bilipschitz_ren.py @@ -0,0 +1,55 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `BiLipschitzREN`.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +from robustnn import ren_jax as ren +import golden +from jax_utils import check_golden, check_ren_forward, setup_ren + + +# ----------------------------- BiLipschitz REN ---------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("nio", [1, 3]) +@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) +def test_bilipschitz_ren(key, batches, nio, mu_nu): + mu, nu = mu_nu + nx, nv = 4, 8 + model = ren.BiLipschitzREN(nio, nx, nv, nio, mu=mu, nu=nu, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = setup_ren(model, key, batches, nio) + check_ren_forward(model, params, states, inputs, nx, nio) + + e = model.direct_to_explicit(params) + _, y = model.explicit_call(params, states, inputs, e) + e_inv = model.direct_to_explicit_inverse(params) + _, u_rec = model.inverse_call(params, states, y, e_inv) + np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) + + n = 300 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nio)) + ub = jax.random.normal(kb, (n, nio)) + z = jnp.zeros((n, nx)) + _, ya = model.explicit_call(params, z, ua, e) + _, yb = model.explicit_call(params, z, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.min() >= mu - 1e-3 + assert gain.max() <= nu + 1e-3 + + +def test_bilipschitz_ren_requires_square(key): + model = ren.BiLipschitzREN(3, 3, 4, 2) + with pytest.raises(ValueError): + setup_ren(model, key, 2, 3) + + +@pytest.mark.parametrize("case", golden.cases("bilipschitz_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipschitz_ren_matches_golden(case, explicit): + check_golden("bilipschitz_ren", case, explicit) diff --git a/tests/jax_tests/test_contracting_ren.py b/tests/jax_tests/test_contracting_ren.py new file mode 100644 index 0000000..b0216b2 --- /dev/null +++ b/tests/jax_tests/test_contracting_ren.py @@ -0,0 +1,91 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `ContractingREN`.""" + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +from helpers import compute_p_contractingren +from robustnn import ren_jax as ren +import golden +from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, setup_ren + + +# --------------------------- Contracting REN ----------------------------- +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("activation", ACTIVATIONS) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +@pytest.mark.parametrize("do_polar_param", [True, False]) +@pytest.mark.parametrize("sizes", [(1, 1, 1, 1), (5, 3, 4, 2)]) +def test_contracting_ren(key, batches, activation, init_method, do_polar_param, sizes): + nu, nx, nv, ny = sizes + model = ren.ContractingREN(nu, nx, nv, ny, activation=activation, + init_method=init_method, do_polar_param=do_polar_param) + params, states, inputs = setup_ren(model, key, batches, nu) + check_ren_forward(model, params, states, inputs, nx, ny) + + +def test_contracting_ren_contraction(key): + model = ren.ContractingREN(5, 3, 4, 2, activation=nn.tanh) + k1, k2 = jax.random.split(key) + params, x1, u = setup_ren(model, key, 8, 5) + x0 = 10 * jax.random.normal(k1, x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingren(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) + + +def test_ren_identity_output_and_zero_output(key): + model = ren.ContractingREN(3, 3, 4, 3, identity_output=True) + params, states, inputs = setup_ren(model, key, 4, 3) + _, y = model.apply(params, states, inputs) + # identity output: y_t = x_t, i.e. the state before the update + np.testing.assert_allclose(y, states, atol=1e-5) + + zero = ren.ContractingREN(3, 3, 4, 2, init_output_zero=True) + params, states, inputs = setup_ren(zero, key, 4, 3) + _, y = zero.apply(params, states, inputs) + np.testing.assert_allclose(y, 0.0, atol=1e-6) + + +def test_ren_identity_output_size_mismatch(key): + model = ren.ContractingREN(3, 3, 4, 2, identity_output=True) + with pytest.raises(ValueError): + setup_ren(model, key, 2, 3) + + +def test_ren_bad_init_method(key): + model = ren.ContractingREN(3, 3, 4, 2, init_method="nope") + with pytest.raises(ValueError): + setup_ren(model, key, 2, 3) + +# ------------------------------ simulate_sequence ------------------------- +@pytest.mark.parametrize("horizon", [1, 5]) +def test_simulate_sequence_matches_loop(key, horizon): + nu, nx, nv, ny = 3, 4, 5, 2 + model = ren.ContractingREN(nu, nx, nv, ny, activation=nn.tanh) + params, x0, u0 = setup_ren(model, key, 4, nu) + seq = jax.random.normal(key, (horizon, 4, nu)) + + xT, ys = model.simulate_sequence(params, x0, seq) + assert ys.shape == (horizon, 4, ny) + + x, outs = x0, [] + for t in range(horizon): + x, y = model.apply(params, x, seq[t]) + outs.append(y) + np.testing.assert_allclose(xT, x, atol=1e-4) + np.testing.assert_allclose(ys, jnp.stack(outs), atol=1e-4) + + +@pytest.mark.parametrize("case", golden.cases("contracting_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_contracting_ren_matches_golden(case, explicit): + check_golden("contracting_ren", case, explicit) diff --git a/tests/jax_tests/test_dyn_orthogonal.py b/tests/jax_tests/test_dyn_orthogonal.py new file mode 100644 index 0000000..112c562 --- /dev/null +++ b/tests/jax_tests/test_dyn_orthogonal.py @@ -0,0 +1,50 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `DynUnitary`.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +import golden +from jax_utils import check_golden +from robustnn.dyn_orthogonal_jax import DynUnitary + + +# ------------------------------ DynUnitary -------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_dyn_unitary(key, batches, io, nx, use_bias): + layer = DynUnitary(io, nx, use_bias=use_bias) + x = jax.random.normal(key, (batches, nx)) + u = jax.random.normal(jax.random.key(1), (batches, io)) + params = layer.init(key, x, u) + + assert layer.initialize_carry(key, (batches, io)).shape == (batches, nx) + e = layer.direct_to_explicit(params) + x1, y = layer.explicit_call(params, x, u, e) + assert x1.shape == (batches, nx) and y.shape == (batches, io) + x1b, yb = layer.apply(params, x, u) + np.testing.assert_allclose(x1, x1b, atol=1e-5) + np.testing.assert_allclose(y, yb, atol=1e-5) + + # Stacked state-space matrix is orthogonal. + G = jnp.block([[e.A, e.B], [e.C, e.D]]) + np.testing.assert_allclose(G @ G.T, jnp.eye(nx + io), atol=1e-5) + + # Non-causal inverse recovers the previous state and the input. + x_rec, u_rec = layer.inverse_call(params, x1, y, e) + np.testing.assert_allclose(x_rec, x, atol=1e-4) + np.testing.assert_allclose(u_rec, u, atol=1e-4) + + g = jax.grad(lambda p: sum(jnp.sum(a ** 2) for a in layer.apply(p, x, u)))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +@pytest.mark.parametrize("case", golden.cases("dyn_unitary"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_dyn_unitary_matches_golden(case, explicit): + check_golden("dyn_unitary", case, explicit) diff --git a/tests/jax_tests/test_general_ren.py b/tests/jax_tests/test_general_ren.py new file mode 100644 index 0000000..3d3cc05 --- /dev/null +++ b/tests/jax_tests/test_general_ren.py @@ -0,0 +1,45 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX QSR-constrained `GeneralREN`.""" + +import jax.numpy as jnp +import pytest + +from robustnn import ren_jax as ren +import golden +from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, make_qsr, setup_ren + + +# ------------------------------ General REN ------------------------------ +@pytest.mark.parametrize("activation", ACTIVATIONS) +@pytest.mark.parametrize("sizes", [(5, 3, 4, 2), (2, 2, 3, 2)]) +def test_general_ren(key, activation, sizes): + nu, nx, nv, ny = sizes + Q, S, R = make_qsr(key, nu, ny) + model = ren.GeneralREN(nu, nx, nv, ny, Q=Q, S=S, R=R, activation=activation, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = setup_ren(model, key, 4, nu) + check_ren_forward(model, params, states, inputs, nx, ny) + + +def test_general_ren_invalid_qsr(key): + nu, ny = 3, 2 + Q, S, R = make_qsr(key, nu, ny) + bad = [ + dict(Q=jnp.eye(ny + 1), S=S, R=R), # wrong Q size + dict(Q=Q, S=jnp.ones((ny, nu)), R=R), # wrong S size + dict(Q=Q, S=S, R=jnp.eye(nu + 1)), # wrong R size + dict(Q=jnp.eye(ny), S=S, R=R), # Q not neg. definite + dict(Q=Q, S=S, R=-jnp.eye(nu)), # Schur complement not PD + ] + for kw in bad: + model = ren.GeneralREN(nu, 3, 4, ny, **kw) + with pytest.raises(ValueError): + model.check_valid_qsr() + + +@pytest.mark.parametrize("case", golden.cases("general_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_general_ren_matches_golden(case, explicit): + check_golden("general_ren", case, explicit) diff --git a/tests/jax_tests/test_plnet_lbdn.py b/tests/jax_tests/test_lbdn.py similarity index 55% rename from tests/jax_tests/test_plnet_lbdn.py rename to tests/jax_tests/test_lbdn.py index 5fcc872..81eda4c 100644 --- a/tests/jax_tests/test_plnet_lbdn.py +++ b/tests/jax_tests/test_lbdn.py @@ -1,6 +1,6 @@ # This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE -"""Tests for the JAX `PLNet` and `LBDN`.""" +"""Tests for the JAX `LBDN`.""" import flax.linen as nn import jax @@ -9,56 +9,9 @@ import pytest from jax.flatten_util import ravel_pytree -from robustnn.bilipnet_jax import BiLipNet +import golden +from jax_utils import check_golden from robustnn.lbdn_jax import LBDN -from robustnn.plnet_jax import PLNet - - -def _plnet(size=3, **kw): - bilip = BiLipNet(input_size=size, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True, depth=2) - return PLNet(BiLipBlock=bilip, **kw) - - -# ------------------------------- PLNet ----------------------------------- -@pytest.mark.parametrize("batches", [1, 6]) -def test_plnet_shape_and_explicit(key, batches): - net = _plnet() - x = jax.random.normal(key, (batches, 3)) - params = net.init(key, x) - y = net.apply(params, x) - assert y.shape == (batches,) - assert bool(jnp.all(y >= 0)) - - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - -def test_plnet_zero_at_optimal_point(key): - net = _plnet() - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - explicit = net.direct_to_explicit(params, x_optimal=x[:1]) - y_opt = net.explicit_call(params, x[:1], explicit) - np.testing.assert_allclose(y_opt, 0.0, atol=1e-6) - assert bool(jnp.all(net.explicit_call(params, x, explicit) >= 0)) - - -def test_plnet_add_constant(key): - net = _plnet(add_constant=True, c=1.5) - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), - net.apply(params, x), atol=1e-5) - - -def test_plnet_grad(key): - net = _plnet() - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x)))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) # -------------------------------- LBDN ----------------------------------- @@ -102,3 +55,9 @@ def test_lbdn_init_output_zero(key): x = jax.random.normal(key, (4, 3)) params = net.init(key, x) np.testing.assert_allclose(net.apply(params, x), 0.0, atol=1e-6) + + +@pytest.mark.parametrize("case", golden.cases("lbdn"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_lbdn_matches_golden(case, explicit): + check_golden("lbdn", case, explicit) diff --git a/tests/jax_tests/test_linear_ren.py b/tests/jax_tests/test_linear_ren.py new file mode 100644 index 0000000..c7f5a2b --- /dev/null +++ b/tests/jax_tests/test_linear_ren.py @@ -0,0 +1,69 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX linear RENs (`ContractingLinREN`, `LipschitzLinREN`, `GeneralLinREN`).""" + +import jax +import jax.numpy as jnp +import pytest + +from robustnn import linear_ren +import golden +from jax_utils import check_golden, check_ren_forward, make_qsr, setup_ren + + +# --------------------------------- Linear RENs ---------------------------- +@pytest.mark.parametrize("cls", ["contracting", "lipschitz", "general"]) +@pytest.mark.parametrize("batches", [1, 4]) +def test_linear_ren(key, cls, batches): + nu, nx, ny = 5, 3, 2 + if cls == "contracting": + model = linear_ren.ContractingLinREN(nu, nx, 0, ny) + elif cls == "lipschitz": + model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=2.0) + else: + Q, S, R = make_qsr(key, nu, ny) + model = linear_ren.GeneralLinREN(nu, nx, 0, ny, Q=Q, S=S, R=R, + init_method="long_memory") + model.check_valid_qsr() + params, states, inputs = setup_ren(model, key, batches, nu) + check_ren_forward(model, params, states, inputs, nx, ny) + + +def test_linear_ren_lipschitz_bound(key): + nu, nx, ny, gamma = 4, 3, 2, 2.0 + model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=gamma) + params, _, _ = setup_ren(model, key, 2, nu) + n = 200 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nu)) + ub = jax.random.normal(kb, (n, nu)) + z = jnp.zeros((n, nx)) + e = model.direct_to_explicit(params) + _, ya = model.explicit_call(params, z, ua, e) + _, yb = model.explicit_call(params, z, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.max() <= gamma + 1e-3 + + +def test_linear_ren_requires_zero_features(key): + with pytest.raises(ValueError): + model = linear_ren.ContractingLinREN(3, 3, 4, 2) + setup_ren(model, key, 2, 3) + + +@pytest.mark.parametrize("case", golden.cases("contracting_linear_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_contracting_linear_ren_matches_golden(case, explicit): + check_golden("contracting_linear_ren", case, explicit) + + +@pytest.mark.parametrize("case", golden.cases("lipschitz_linear_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_lipschitz_linear_ren_matches_golden(case, explicit): + check_golden("lipschitz_linear_ren", case, explicit) + + +@pytest.mark.parametrize("case", golden.cases("general_linear_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_general_linear_ren_matches_golden(case, explicit): + check_golden("general_linear_ren", case, explicit) diff --git a/tests/jax_tests/test_lipnets.py b/tests/jax_tests/test_lipnets.py deleted file mode 100644 index 3ab84e9..0000000 --- a/tests/jax_tests/test_lipnets.py +++ /dev/null @@ -1,130 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `MonLipNet` and `BiLipNet`.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -from robustnn.bilipnet_jax import BiLipNet -from robustnn.monlipnet_jax import MonLipNet - - -def _gains(f, key, n, size): - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, size)) - ub = jax.random.normal(kb, (n, size)) - return jnp.linalg.norm(f(ua) - f(ub), axis=1) / jnp.linalg.norm(ua - ub, axis=1) - - -# ----------------------------- MonLipNet --------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) -@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) -def test_monlipnet_shape_explicit_and_bounds(key, batches, units, fixed): - size, mu, nu, tau = 3, 0.5, 4.0, 4.0 - flags = dict(is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, - is_tau_fixed="tau" in fixed) - net = MonLipNet(input_size=size, units=units, mu=mu, nu=nu, tau=tau, **flags) - x = jax.random.normal(key, (batches, size)) - params = net.init(key, x) - - y = net.apply(params, x) - assert y.shape == (batches, size) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - lo, hi, _ = net.get_bounds(params) - gains = _gains(lambda u: net.apply(params, u), key, 200, size) - assert gains.min() >= float(lo) - 1e-3 - assert gains.max() <= float(hi) + 1e-3 - - -def test_monlipnet_fixing_all_raises(key): - net = MonLipNet(input_size=2, units=[2], is_mu_fixed=True, - is_nu_fixed=True, is_tau_fixed=True) - with pytest.raises(ValueError): - net.init(key, jnp.ones((1, 2))) - - -@pytest.mark.parametrize("units", [[3], [4, 4]]) -def test_monlipnet_inverse_round_trip(key, units): - net = MonLipNet(input_size=3, units=units, mu=0.5, nu=4.0, tau=4.0, - is_tau_fixed=True) - x = jax.random.normal(key, (6, 3)) - params = net.init(key, x) - y = net.apply(params, x) - e_inv = net.direct_to_explicit_inverse(params, alpha=0.1, iterations=500) - x_rec = net.inverse_call(params, y, e_inv) - np.testing.assert_allclose(x_rec, x, atol=1e-3) - - -def test_monlipnet_jit_and_grad(key): - net = MonLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=4.0, - is_tau_fixed=True) - x = jax.random.normal(key, (5, 3)) - params = net.init(key, x) - np.testing.assert_allclose(jax.jit(net.apply)(params, x), - net.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) - flat = ravel_pytree(g)[0] - assert bool(jnp.isfinite(flat).all()) and bool(jnp.any(flat != 0)) - - -# ----------------------------- BiLipNet ---------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("depth", [1, 2, 3]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_bilipnet_shape_explicit_and_bounds(key, batches, depth, use_bias): - size, mu, nu = 3, 0.5, 4.0 - net = BiLipNet(input_size=size, units=[4, 4], mu=mu, nu=nu, tau=nu / mu, - is_mu_fixed=True, is_nu_fixed=True, depth=depth, - use_bias=use_bias) - x = jax.random.normal(key, (batches, size)) - params = net.init(key, x) - - y = net.apply(params, x) - assert y.shape == (batches, size) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - lo, hi, _ = net.get_bounds(params) - np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-5) - gains = _gains(lambda u: net.apply(params, u), key, 200, size) - assert gains.min() >= mu - 1e-3 - assert gains.max() <= nu + 1e-3 - - -@pytest.mark.parametrize("depth", [1, 2]) -def test_bilipnet_inverse_round_trip(key, depth): - net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True, depth=depth, act_fn=nn.relu) - x = jax.random.normal(key, (6, 3)) - params = net.init(key, x) - y = net.apply(params, x) - inv_act = nn.relu - e_inv = net.direct_to_explicit_inverse( - params, [0.1] * depth, [inv_act] * depth, [500] * depth, [1.0] * depth) - x_rec = net.inverse_call(params, y, e_inv) - np.testing.assert_allclose(x_rec, x, atol=1e-3) - - -def test_bilipnet_jit_and_grad(key): - net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True) - x = jax.random.normal(key, (5, 3)) - params = net.init(key, x) - np.testing.assert_allclose(jax.jit(net.apply)(params, x), - net.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def test_bilipnet_fixing_all_raises(key): - net = BiLipNet(input_size=2, units=[2], is_mu_fixed=True, - is_nu_fixed=True, is_tau_fixed=True) - with pytest.raises(ValueError): - net.init(key, jnp.ones((1, 2))) diff --git a/tests/jax_tests/test_lipschitz_ren.py b/tests/jax_tests/test_lipschitz_ren.py new file mode 100644 index 0000000..e51a31a --- /dev/null +++ b/tests/jax_tests/test_lipschitz_ren.py @@ -0,0 +1,45 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `LipschitzREN`.""" + +import jax +import jax.numpy as jnp +import pytest + +from robustnn import ren_jax as ren +import golden +from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, setup_ren + + +# ----------------------------- Lipschitz REN ----------------------------- +@pytest.mark.parametrize("gamma", [0.5, 1.0, 5.0]) +@pytest.mark.parametrize("activation", ACTIVATIONS) +def test_lipschitz_ren(key, gamma, activation): + nu, nx, nv, ny = 4, 3, 5, 2 + model = ren.LipschitzREN(nu, nx, nv, ny, gamma=gamma, activation=activation) + params, states, inputs = setup_ren(model, key, 4, nu) + check_ren_forward(model, params, states, inputs, nx, ny) + + # Same initial state => one-step output gain is bounded by gamma. + n = 200 + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, nu)) + ub = jax.random.normal(kb, (n, nu)) + x = jnp.zeros((n, nx)) + e = model.direct_to_explicit(params) + _, ya = model.explicit_call(params, x, ua, e) + _, yb = model.explicit_call(params, x, ub, e) + gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) + assert gain.max() <= gamma + 1e-3 + + +def test_lipschitz_ren_identity_output_unsupported(key): + with pytest.raises(NotImplementedError): + model = ren.LipschitzREN(3, 3, 4, 3, identity_output=True) + setup_ren(model, key, 2, 3) + + +@pytest.mark.parametrize("case", golden.cases("lipschitz_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_lipschitz_ren_matches_golden(case, explicit): + check_golden("lipschitz_ren", case, explicit) diff --git a/tests/jax_tests/test_monlipnet.py b/tests/jax_tests/test_monlipnet.py new file mode 100644 index 0000000..6dfd905 --- /dev/null +++ b/tests/jax_tests/test_monlipnet.py @@ -0,0 +1,73 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `MonLipNet`.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +import golden +from jax_utils import check_golden, empirical_gains +from robustnn.monlipnet_jax import MonLipNet + + +# ----------------------------- MonLipNet --------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) +@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) +def test_monlipnet_shape_explicit_and_bounds(key, batches, units, fixed): + size, mu, nu, tau = 3, 0.5, 4.0, 4.0 + flags = dict(is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, + is_tau_fixed="tau" in fixed) + net = MonLipNet(input_size=size, units=units, mu=mu, nu=nu, tau=tau, **flags) + x = jax.random.normal(key, (batches, size)) + params = net.init(key, x) + + y = net.apply(params, x) + assert y.shape == (batches, size) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + lo, hi, _ = net.get_bounds(params) + gains = empirical_gains(lambda u: net.apply(params, u), key, 200, size) + assert gains.min() >= float(lo) - 1e-3 + assert gains.max() <= float(hi) + 1e-3 + + +def test_monlipnet_fixing_all_raises(key): + net = MonLipNet(input_size=2, units=[2], is_mu_fixed=True, + is_nu_fixed=True, is_tau_fixed=True) + with pytest.raises(ValueError): + net.init(key, jnp.ones((1, 2))) + + +@pytest.mark.parametrize("units", [[3], [4, 4]]) +def test_monlipnet_inverse_round_trip(key, units): + net = MonLipNet(input_size=3, units=units, mu=0.5, nu=4.0, tau=4.0, + is_tau_fixed=True) + x = jax.random.normal(key, (6, 3)) + params = net.init(key, x) + y = net.apply(params, x) + e_inv = net.direct_to_explicit_inverse(params, alpha=0.1, iterations=500) + x_rec = net.inverse_call(params, y, e_inv) + np.testing.assert_allclose(x_rec, x, atol=1e-3) + + +def test_monlipnet_jit_and_grad(key): + net = MonLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=4.0, + is_tau_fixed=True) + x = jax.random.normal(key, (5, 3)) + params = net.init(key, x) + np.testing.assert_allclose(jax.jit(net.apply)(params, x), + net.apply(params, x), atol=1e-5) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) + flat = ravel_pytree(g)[0] + assert bool(jnp.isfinite(flat).all()) and bool(jnp.any(flat != 0)) + + +@pytest.mark.parametrize("case", golden.cases("monlipnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_monlipnet_matches_golden(case, explicit): + check_golden("monlipnet", case, explicit) diff --git a/tests/jax_tests/test_orthogonal.py b/tests/jax_tests/test_orthogonal.py index 85e657f..72adf99 100644 --- a/tests/jax_tests/test_orthogonal.py +++ b/tests/jax_tests/test_orthogonal.py @@ -8,6 +8,8 @@ import pytest from jax.flatten_util import ravel_pytree +import golden +from jax_utils import check_golden from robustnn.orthogonal_jax import Unitary @@ -65,3 +67,9 @@ def test_jit_and_grad(key): g = jax.grad(lambda p: jnp.sum(layer.apply(p, x) ** 2))(params) flat = ravel_pytree(g)[0] assert bool(jnp.isfinite(flat).all()) + + +@pytest.mark.parametrize("case", golden.cases("unitary"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_unitary_matches_golden(case, explicit): + check_golden("unitary", case, explicit) diff --git a/tests/jax_tests/test_plnet.py b/tests/jax_tests/test_plnet.py new file mode 100644 index 0000000..a9153d4 --- /dev/null +++ b/tests/jax_tests/test_plnet.py @@ -0,0 +1,67 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `PLNet`.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.flatten_util import ravel_pytree + +import golden +from jax_utils import check_golden +from robustnn.bilipnet_jax import BiLipNet +from robustnn.plnet_jax import PLNet + + +def _plnet(size=3, **kw): + bilip = BiLipNet(input_size=size, units=[4], mu=0.5, nu=4.0, tau=8.0, + is_mu_fixed=True, is_nu_fixed=True, depth=2) + return PLNet(BiLipBlock=bilip, **kw) + + +# ------------------------------- PLNet ----------------------------------- +@pytest.mark.parametrize("batches", [1, 6]) +def test_plnet_shape_and_explicit(key, batches): + net = _plnet() + x = jax.random.normal(key, (batches, 3)) + params = net.init(key, x) + y = net.apply(params, x) + assert y.shape == (batches,) + assert bool(jnp.all(y >= 0)) + + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) + + +def test_plnet_zero_at_optimal_point(key): + net = _plnet() + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + explicit = net.direct_to_explicit(params, x_optimal=x[:1]) + y_opt = net.explicit_call(params, x[:1], explicit) + np.testing.assert_allclose(y_opt, 0.0, atol=1e-6) + assert bool(jnp.all(net.explicit_call(params, x, explicit) >= 0)) + + +def test_plnet_add_constant(key): + net = _plnet(add_constant=True, c=1.5) + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + explicit = net.direct_to_explicit(params) + np.testing.assert_allclose(net.explicit_call(params, x, explicit), + net.apply(params, x), atol=1e-5) + + +def test_plnet_grad(key): + net = _plnet() + x = jax.random.normal(key, (4, 3)) + params = net.init(key, x) + g = jax.grad(lambda p: jnp.sum(net.apply(p, x)))(params) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +@pytest.mark.parametrize("case", golden.cases("plnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_plnet_matches_golden(case, explicit): + check_golden("plnet", case, explicit) diff --git a/tests/jax_tests/test_r2dn.py b/tests/jax_tests/test_r2dn.py new file mode 100644 index 0000000..ef7c47b --- /dev/null +++ b/tests/jax_tests/test_r2dn.py @@ -0,0 +1,48 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the JAX `ContractingR2DN`.""" + +import jax +import jax.numpy as jnp +import pytest + +from helpers import compute_p_contractingr2dn +from robustnn import r2dn_jax as r2dn +import golden +from jax_utils import check_golden, check_ren_forward, setup_ren + + +# ----------------------------------- R2DN --------------------------------- +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("hidden", [(2,), (2, 2), (4, 3, 2)]) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +def test_contracting_r2dn(key, batches, hidden, init_method): + nu, nx, nv, ny = 5, 3, 4, 2 + model = r2dn.ContractingR2DN(nu, nx, nv, ny, hidden, init_method=init_method) + params, states, inputs = setup_ren(model, key, batches, nu) + check_ren_forward(model, params, states, inputs, nx, ny) + + +def test_r2dn_simulate_and_contraction(key): + nu, nx, nv, ny, horizon = 5, 3, 4, 2, 3 + model = r2dn.ContractingR2DN(nu, nx, nv, ny, (2, 2), init_method="long_memory") + params, x1, u = setup_ren(model, key, 4, nu) + + seq = jax.random.normal(key, (horizon, 4, nu)) + xT, ys = jax.jit(model.simulate_sequence)(params, x1, seq) + assert ys.shape == (horizon, 4, ny) and xT.shape == x1.shape + + x0 = 10 * jax.random.normal(key, x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingr2dn(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) + + +@pytest.mark.parametrize("case", golden.cases("r2dn"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_r2dn_matches_golden(case, explicit): + check_golden("r2dn", case, explicit) diff --git a/tests/jax_tests/test_ren.py b/tests/jax_tests/test_ren.py deleted file mode 100644 index 733d97d..0000000 --- a/tests/jax_tests/test_ren.py +++ /dev/null @@ -1,294 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX RENs: Contracting, Lipschitz, General (QSR) and -BiLipschitz, plus the linear RENs and the contracting R2DN.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -from helpers import compute_p_contractingr2dn, compute_p_contractingren -from robustnn import linear_ren -from robustnn import r2dn_jax as r2dn -from robustnn import ren_jax as ren - -ACTIVATIONS = [nn.relu, nn.tanh, lambda x: x] - - -def _setup(model, key, batches, nin, x_scale=1.0): - k1, k2, k3 = jax.random.split(key, 3) - states = model.initialize_carry(k1, (batches, nin)) - states = x_scale * jax.random.normal(k2, states.shape) - inputs = jax.random.normal(k3, (batches, nin)) - params = model.init(key, states, inputs) - return params, states, inputs - - -def _qsr(key, nu, ny): - kx, ky, ks = jax.random.split(key, 3) - X = jax.random.normal(kx, (ny, ny)) - Y = jax.random.normal(ky, (nu, nu)) - S = jax.random.normal(ks, (nu, ny)) - Q = -X.T @ X - R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y - return Q, S, R - - -def _check_forward(model, params, states, inputs, nx, ny): - batches = inputs.shape[0] - explicit = model.direct_to_explicit(params) - xn, y = model.explicit_call(params, states, inputs, explicit) - assert xn.shape == (batches, nx) - assert y.shape == (batches, ny) - xn2, y2 = model.apply(params, states, inputs) - np.testing.assert_allclose(xn, xn2, atol=1e-5) - np.testing.assert_allclose(y, y2, atol=1e-5) - - # jit gives the same answer, and gradients w.r.t. everything are finite - jit_xn, jit_y = jax.jit(model.apply)(params, states, inputs) - np.testing.assert_allclose(y, jit_y, atol=1e-4) - - def loss(p, x, u): - a, b = model.apply(p, x, u) - return jnp.sum(a ** 2) + jnp.sum(b ** 2) - g = jax.grad(loss, argnums=(0, 1, 2))(params, states, inputs) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -# --------------------------- Contracting REN ----------------------------- -@pytest.mark.parametrize("batches", [1, 4]) -@pytest.mark.parametrize("activation", ACTIVATIONS) -@pytest.mark.parametrize("init_method", ["random", "long_memory"]) -@pytest.mark.parametrize("do_polar_param", [True, False]) -@pytest.mark.parametrize("sizes", [(1, 1, 1, 1), (5, 3, 4, 2)]) -def test_contracting_ren(key, batches, activation, init_method, do_polar_param, sizes): - nu, nx, nv, ny = sizes - model = ren.ContractingREN(nu, nx, nv, ny, activation=activation, - init_method=init_method, do_polar_param=do_polar_param) - params, states, inputs = _setup(model, key, batches, nu) - _check_forward(model, params, states, inputs, nx, ny) - - -def test_contracting_ren_contraction(key): - model = ren.ContractingREN(5, 3, 4, 2, activation=nn.tanh) - k1, k2 = jax.random.split(key) - params, x1, u = _setup(model, key, 8, 5) - x0 = 10 * jax.random.normal(k1, x1.shape) - xn0, _ = model.apply(params, x0, u) - xn1, _ = model.apply(params, x1, u) - P = compute_p_contractingren(model, params) - - def norm2(x): - return jnp.sum((x @ P.T) * x, axis=-1) - assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) - - -def test_ren_identity_output_and_zero_output(key): - model = ren.ContractingREN(3, 3, 4, 3, identity_output=True) - params, states, inputs = _setup(model, key, 4, 3) - _, y = model.apply(params, states, inputs) - # identity output: y_t = x_t, i.e. the state before the update - np.testing.assert_allclose(y, states, atol=1e-5) - - zero = ren.ContractingREN(3, 3, 4, 2, init_output_zero=True) - params, states, inputs = _setup(zero, key, 4, 3) - _, y = zero.apply(params, states, inputs) - np.testing.assert_allclose(y, 0.0, atol=1e-6) - - -def test_ren_identity_output_size_mismatch(key): - model = ren.ContractingREN(3, 3, 4, 2, identity_output=True) - with pytest.raises(ValueError): - _setup(model, key, 2, 3) - - -def test_ren_bad_init_method(key): - model = ren.ContractingREN(3, 3, 4, 2, init_method="nope") - with pytest.raises(ValueError): - _setup(model, key, 2, 3) - - -# ----------------------------- Lipschitz REN ----------------------------- -@pytest.mark.parametrize("gamma", [0.5, 1.0, 5.0]) -@pytest.mark.parametrize("activation", ACTIVATIONS) -def test_lipschitz_ren(key, gamma, activation): - nu, nx, nv, ny = 4, 3, 5, 2 - model = ren.LipschitzREN(nu, nx, nv, ny, gamma=gamma, activation=activation) - params, states, inputs = _setup(model, key, 4, nu) - _check_forward(model, params, states, inputs, nx, ny) - - # Same initial state => one-step output gain is bounded by gamma. - n = 200 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nu)) - ub = jax.random.normal(kb, (n, nu)) - x = jnp.zeros((n, nx)) - e = model.direct_to_explicit(params) - _, ya = model.explicit_call(params, x, ua, e) - _, yb = model.explicit_call(params, x, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.max() <= gamma + 1e-3 - - -def test_lipschitz_ren_identity_output_unsupported(key): - with pytest.raises(NotImplementedError): - model = ren.LipschitzREN(3, 3, 4, 3, identity_output=True) - _setup(model, key, 2, 3) - - -# ------------------------------ General REN ------------------------------ -@pytest.mark.parametrize("activation", ACTIVATIONS) -@pytest.mark.parametrize("sizes", [(5, 3, 4, 2), (2, 2, 3, 2)]) -def test_general_ren(key, activation, sizes): - nu, nx, nv, ny = sizes - Q, S, R = _qsr(key, nu, ny) - model = ren.GeneralREN(nu, nx, nv, ny, Q=Q, S=S, R=R, activation=activation, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = _setup(model, key, 4, nu) - _check_forward(model, params, states, inputs, nx, ny) - - -def test_general_ren_invalid_qsr(key): - nu, ny = 3, 2 - Q, S, R = _qsr(key, nu, ny) - bad = [ - dict(Q=jnp.eye(ny + 1), S=S, R=R), # wrong Q size - dict(Q=Q, S=jnp.ones((ny, nu)), R=R), # wrong S size - dict(Q=Q, S=S, R=jnp.eye(nu + 1)), # wrong R size - dict(Q=jnp.eye(ny), S=S, R=R), # Q not neg. definite - dict(Q=Q, S=S, R=-jnp.eye(nu)), # Schur complement not PD - ] - for kw in bad: - model = ren.GeneralREN(nu, 3, 4, ny, **kw) - with pytest.raises(ValueError): - model.check_valid_qsr() - - -# ----------------------------- BiLipschitz REN ---------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("nio", [1, 3]) -@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) -def test_bilipschitz_ren(key, batches, nio, mu_nu): - mu, nu = mu_nu - nx, nv = 4, 8 - model = ren.BiLipschitzREN(nio, nx, nv, nio, mu=mu, nu=nu, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = _setup(model, key, batches, nio) - _check_forward(model, params, states, inputs, nx, nio) - - e = model.direct_to_explicit(params) - _, y = model.explicit_call(params, states, inputs, e) - e_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call(params, states, y, e_inv) - np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) - - n = 300 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nio)) - ub = jax.random.normal(kb, (n, nio)) - z = jnp.zeros((n, nx)) - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.min() >= mu - 1e-3 - assert gain.max() <= nu + 1e-3 - - -def test_bilipschitz_ren_requires_square(key): - model = ren.BiLipschitzREN(3, 3, 4, 2) - with pytest.raises(ValueError): - _setup(model, key, 2, 3) - - -# ------------------------------ simulate_sequence ------------------------- -@pytest.mark.parametrize("horizon", [1, 5]) -def test_simulate_sequence_matches_loop(key, horizon): - nu, nx, nv, ny = 3, 4, 5, 2 - model = ren.ContractingREN(nu, nx, nv, ny, activation=nn.tanh) - params, x0, u0 = _setup(model, key, 4, nu) - seq = jax.random.normal(key, (horizon, 4, nu)) - - xT, ys = model.simulate_sequence(params, x0, seq) - assert ys.shape == (horizon, 4, ny) - - x, outs = x0, [] - for t in range(horizon): - x, y = model.apply(params, x, seq[t]) - outs.append(y) - np.testing.assert_allclose(xT, x, atol=1e-4) - np.testing.assert_allclose(ys, jnp.stack(outs), atol=1e-4) - - -# --------------------------------- Linear RENs ---------------------------- -@pytest.mark.parametrize("cls", ["contracting", "lipschitz", "general"]) -@pytest.mark.parametrize("batches", [1, 4]) -def test_linear_ren(key, cls, batches): - nu, nx, ny = 5, 3, 2 - if cls == "contracting": - model = linear_ren.ContractingLinREN(nu, nx, 0, ny) - elif cls == "lipschitz": - model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=2.0) - else: - Q, S, R = _qsr(key, nu, ny) - model = linear_ren.GeneralLinREN(nu, nx, 0, ny, Q=Q, S=S, R=R, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = _setup(model, key, batches, nu) - _check_forward(model, params, states, inputs, nx, ny) - - -def test_linear_ren_lipschitz_bound(key): - nu, nx, ny, gamma = 4, 3, 2, 2.0 - model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=gamma) - params, _, _ = _setup(model, key, 2, nu) - n = 200 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nu)) - ub = jax.random.normal(kb, (n, nu)) - z = jnp.zeros((n, nx)) - e = model.direct_to_explicit(params) - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.max() <= gamma + 1e-3 - - -def test_linear_ren_requires_zero_features(key): - with pytest.raises(ValueError): - model = linear_ren.ContractingLinREN(3, 3, 4, 2) - _setup(model, key, 2, 3) - - -# ----------------------------------- R2DN --------------------------------- -@pytest.mark.parametrize("batches", [1, 4]) -@pytest.mark.parametrize("hidden", [(2,), (2, 2), (4, 3, 2)]) -@pytest.mark.parametrize("init_method", ["random", "long_memory"]) -def test_contracting_r2dn(key, batches, hidden, init_method): - nu, nx, nv, ny = 5, 3, 4, 2 - model = r2dn.ContractingR2DN(nu, nx, nv, ny, hidden, init_method=init_method) - params, states, inputs = _setup(model, key, batches, nu) - _check_forward(model, params, states, inputs, nx, ny) - - -def test_r2dn_simulate_and_contraction(key): - nu, nx, nv, ny, horizon = 5, 3, 4, 2, 3 - model = r2dn.ContractingR2DN(nu, nx, nv, ny, (2, 2), init_method="long_memory") - params, x1, u = _setup(model, key, 4, nu) - - seq = jax.random.normal(key, (horizon, 4, nu)) - xT, ys = jax.jit(model.simulate_sequence)(params, x1, seq) - assert ys.shape == (horizon, 4, ny) and xT.shape == x1.shape - - x0 = 10 * jax.random.normal(key, x1.shape) - xn0, _ = model.apply(params, x0, u) - xn1, _ = model.apply(params, x1, u) - P = compute_p_contractingr2dn(model, params) - - def norm2(x): - return jnp.sum((x @ P.T) * x, axis=-1) - assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) diff --git a/tests/jax_tests/test_composition.py b/tests/jax_tests/test_ren_composition.py similarity index 76% rename from tests/jax_tests/test_composition.py rename to tests/jax_tests/test_ren_composition.py index c4d8242..0e1a864 100644 --- a/tests/jax_tests/test_composition.py +++ b/tests/jax_tests/test_ren_composition.py @@ -1,6 +1,6 @@ # This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE -"""Tests for the JAX `DynUnitary` and `CompositionREN`.""" +"""Tests for the JAX `CompositionREN`.""" import flax.linen as nn import jax @@ -9,41 +9,11 @@ import pytest from jax.flatten_util import ravel_pytree -from robustnn.dyn_orthogonal_jax import DynUnitary +import golden +from jax_utils import check_golden from robustnn.ren_composition_jax import CompositionREN -# ------------------------------ DynUnitary -------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_dyn_unitary(key, batches, io, nx, use_bias): - layer = DynUnitary(io, nx, use_bias=use_bias) - x = jax.random.normal(key, (batches, nx)) - u = jax.random.normal(jax.random.key(1), (batches, io)) - params = layer.init(key, x, u) - - assert layer.initialize_carry(key, (batches, io)).shape == (batches, nx) - e = layer.direct_to_explicit(params) - x1, y = layer.explicit_call(params, x, u, e) - assert x1.shape == (batches, nx) and y.shape == (batches, io) - x1b, yb = layer.apply(params, x, u) - np.testing.assert_allclose(x1, x1b, atol=1e-5) - np.testing.assert_allclose(y, yb, atol=1e-5) - - # Stacked state-space matrix is orthogonal. - G = jnp.block([[e.A, e.B], [e.C, e.D]]) - np.testing.assert_allclose(G @ G.T, jnp.eye(nx + io), atol=1e-5) - - # Non-causal inverse recovers the previous state and the input. - x_rec, u_rec = layer.inverse_call(params, x1, y, e) - np.testing.assert_allclose(x_rec, x, atol=1e-4) - np.testing.assert_allclose(u_rec, u, atol=1e-4) - - g = jax.grad(lambda p: sum(jnp.sum(a ** 2) for a in layer.apply(p, x, u)))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - # ----------------------------- CompositionREN ----------------------------- def _model(io=2, nx=3, nv=6, L=2, mu=0.5, nu=5.0, **kw): return CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, @@ -153,3 +123,9 @@ def test_composition_ren_invalid_args(key, kw): with pytest.raises(ValueError): model = _model(**kw) _setup(model, key, 2, 2) + + +@pytest.mark.parametrize("case", golden.cases("ren_composition"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_ren_composition_matches_golden(case, explicit): + check_golden("ren_composition", case, explicit) diff --git a/tests/jax_utils.py b/tests/jax_utils.py new file mode 100644 index 0000000..a2575f2 --- /dev/null +++ b/tests/jax_utils.py @@ -0,0 +1,177 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Shared helpers for the JAX tests.""" + +import jax +import jax.numpy as jnp +import numpy as np +import flax.linen as nn +from jax.flatten_util import ravel_pytree + +import golden + + +ACTIVATIONS = [nn.relu, nn.tanh, lambda x: x] + + +def setup_ren(model, key, batches, nin, x_scale=1.0): + k1, k2, k3 = jax.random.split(key, 3) + states = model.initialize_carry(k1, (batches, nin)) + states = x_scale * jax.random.normal(k2, states.shape) + inputs = jax.random.normal(k3, (batches, nin)) + params = model.init(key, states, inputs) + return params, states, inputs + + +def make_qsr(key, nu, ny): + kx, ky, ks = jax.random.split(key, 3) + X = jax.random.normal(kx, (ny, ny)) + Y = jax.random.normal(ky, (nu, nu)) + S = jax.random.normal(ks, (nu, ny)) + Q = -X.T @ X + R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y + return Q, S, R + + +def check_ren_forward(model, params, states, inputs, nx, ny): + batches = inputs.shape[0] + explicit = model.direct_to_explicit(params) + xn, y = model.explicit_call(params, states, inputs, explicit) + assert xn.shape == (batches, nx) + assert y.shape == (batches, ny) + xn2, y2 = model.apply(params, states, inputs) + np.testing.assert_allclose(xn, xn2, atol=1e-5) + np.testing.assert_allclose(y, y2, atol=1e-5) + + # jit gives the same answer, and gradients w.r.t. everything are finite + jit_xn, jit_y = jax.jit(model.apply)(params, states, inputs) + np.testing.assert_allclose(y, jit_y, atol=1e-4) + + def loss(p, x, u): + a, b = model.apply(p, x, u) + return jnp.sum(a ** 2) + jnp.sum(b ** 2) + g = jax.grad(loss, argnums=(0, 1, 2))(params, states, inputs) + assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) + + +def empirical_gains(f, key, n, size): + """Empirical gain |f(a) - f(b)| / |a - b| over `n` random input pairs.""" + ka, kb = jax.random.split(key) + ua = jax.random.normal(ka, (n, size)) + ub = jax.random.normal(kb, (n, size)) + return jnp.linalg.norm(f(ua) - f(ub), axis=1) / jnp.linalg.norm(ua - ub, axis=1) + + +# --------------------------------------------------------------------------- +# Golden-file support (see golden.py) +# --------------------------------------------------------------------------- + +_ACT = {"relu": nn.relu, "tanh": nn.tanh, "sigmoid": nn.sigmoid, "identity": lambda x: x} + + +def _tup(v): + return tuple(v) if isinstance(v, (list, tuple)) else v + + +def build_model(network, config): + """Construct the JAX module for a golden-file `config` (JAX argument names).""" + from robustnn import linear_ren, r2dn_jax + from robustnn import ren_jax + from robustnn.bilipnet_jax import BiLipNet + from robustnn.dyn_orthogonal_jax import DynUnitary + from robustnn.lbdn_jax import LBDN + from robustnn.monlipnet_jax import MonLipNet + from robustnn.orthogonal_jax import Unitary + from robustnn.plnet_jax import PLNet + from robustnn.ren_composition_jax import CompositionREN + + cfg = dict(config) + if "activation" in cfg: + cfg["activation"] = _ACT[cfg["activation"]] + if "act_fn" in cfg: + cfg["act_fn"] = _ACT[cfg["act_fn"]] + for k in ("units", "hidden", "hidden_sizes"): + if k in cfg: + cfg[k] = _tup(cfg[k]) + for k in ("Q", "S", "R", "optimal_point"): + if cfg.get(k) is not None: + cfg[k] = jnp.asarray(cfg[k], dtype=jnp.float32) + + if network == "plnet": + bilip_cfg = cfg.pop("BiLipBlock") + return PLNet(BiLipBlock=build_model("bilipnet", bilip_cfg), **cfg) + + cls = { + "unitary": Unitary, + "monlipnet": MonLipNet, + "bilipnet": BiLipNet, + "lbdn": LBDN, + "dyn_unitary": DynUnitary, + "contracting_ren": ren_jax.ContractingREN, + "lipschitz_ren": ren_jax.LipschitzREN, + "general_ren": ren_jax.GeneralREN, + "bilipschitz_ren": ren_jax.BiLipschitzREN, + "contracting_linear_ren": linear_ren.ContractingLinREN, + "lipschitz_linear_ren": linear_ren.LipschitzLinREN, + "general_linear_ren": linear_ren.GeneralLinREN, + "r2dn": r2dn_jax.ContractingR2DN, + "ren_composition": CompositionREN, + }[network] + return cls(**cfg) + + +def _carry_in(c): + return { + "rens": [jnp.asarray(r) for r in c["rens"]], + "dyn_in": None if c["dyn_in"] is None else jnp.asarray(c["dyn_in"]), + "dyn_out": None if c["dyn_out"] is None else jnp.asarray(c["dyn_out"]), + } + + +def _carry_out(c): + return { + "rens": np.stack([np.asarray(r) for r in c["rens"]]), + "dyn_in": None if c["dyn_in"] is None else np.asarray(c["dyn_in"]), + "dyn_out": None if c["dyn_out"] is None else np.asarray(c["dyn_out"]), + } + + +_STATIC_X = ("unitary", "monlipnet", "bilipnet", "plnet", "lbdn") + + +def run_model(network, model, params, inputs, explicit=False): + """Evaluate `model` on golden `inputs`, via `apply` or the explicit path.""" + if network in _STATIC_X: + x = jnp.asarray(inputs["x"]) + if explicit: + e = model.direct_to_explicit(params) + y = model.explicit_call(params, x, e) + else: + y = model.apply(params, x) + return {"y": np.asarray(y)} + + if network == "ren_composition": + carry, u = _carry_in(inputs["carry"]), jnp.asarray(inputs["u"]) + if explicit: + e = model.direct_to_explicit(params) + new_carry, y = model.explicit_call(params, carry, u, e) + else: + new_carry, y = model.apply(params, carry, u) + return {"carry": _carry_out(new_carry), "y": np.asarray(y)} + + # dyn_unitary and all the REN flavours: (state, u) -> (next_state, y) + x, u = jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"]) + if explicit: + e = model.direct_to_explicit(params) + xn, y = model.explicit_call(params, x, u, e) + else: + xn, y = model.apply(params, x, u) + return {"next_state": np.asarray(xn), "y": np.asarray(y)} + + +def check_golden(network, case, explicit=False): + """Check a golden case, via `apply` or `direct_to_explicit` + `explicit_call`.""" + model = build_model(network, case["config"]) + params = {"params": jax.tree.map(jnp.asarray, case["params"])} + out = run_model(network, model, params, case["inputs"], explicit=explicit) + golden.assert_close(out, case["outputs"]) diff --git a/tests/test_parity.py b/tests/test_parity.py deleted file mode 100644 index 4e97c22..0000000 --- a/tests/test_parity.py +++ /dev/null @@ -1,61 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""JAX vs PyTorch parity: build the torch layer, copy its *explicit* -parameters into the JAX layer, and check both give the same outputs. - -Initialisations differ between the two backends, so parameters have to be -copied across explicitly. Only the layers with a simple explicit -parameterisation are covered; MonLipNet / BiLipNet / REN parity is a TODO -(their explicit parameter layouts differ between backends). -""" - -import numpy as np -import pytest - -jax = pytest.importorskip("jax") -jnp = pytest.importorskip("jax.numpy") -torch = pytest.importorskip("torch") - -from robustnn.dyn_orthogonal_jax import DynUnitary as JaxDynUnitary -from robustnn.dyn_orthogonal_jax import ExplicitDynOrthogonalParams -from robustnn.dyn_orthogonal_torch import DynUnitary as TorchDynUnitary -from robustnn.orthogonal_jax import ExplicitOrthogonalParams, Unitary as JaxUnitary -from robustnn.orthogonal_torch import Unitary as TorchUnitary - - -@pytest.mark.parametrize("size", [2, 5]) -def test_unitary_parity(size): - torch.manual_seed(0) - t_layer = TorchUnitary(size, size, bias=True) - with torch.no_grad(): - t_layer.bias.copy_(torch.randn(size)) - x = torch.randn(6, size) - y_torch = t_layer(x).detach().numpy() - - e = t_layer.direct_to_explicit() - j_layer = JaxUnitary(input_size=size, use_bias=True) - params = j_layer.init(jax.random.key(0), jnp.asarray(x.numpy())) - j_explicit = ExplicitOrthogonalParams(R=jnp.asarray(e.Q), b=jnp.asarray(e.b)) - y_jax = j_layer.explicit_call(params, jnp.asarray(x.numpy()), j_explicit) - np.testing.assert_allclose(np.asarray(y_jax), y_torch, atol=1e-5) - - -@pytest.mark.parametrize("io,nx", [(2, 3), (3, 6)]) -def test_dyn_unitary_parity(io, nx): - torch.manual_seed(0) - t_layer = TorchDynUnitary(io, nx) - x, u = torch.randn(5, nx), torch.randn(5, io) - with torch.no_grad(): - x1_t, y_t = t_layer(x, u) - A, B, C, D = [m.numpy() for m in t_layer._blocks()] - - j_layer = JaxDynUnitary(io, nx) - jx, ju = jnp.asarray(x.numpy()), jnp.asarray(u.numpy()) - params = j_layer.init(jax.random.key(0), jx, ju) - e = ExplicitDynOrthogonalParams( - jnp.asarray(A), jnp.asarray(B), jnp.asarray(C), jnp.asarray(D), - jnp.asarray(t_layer.bx.detach().numpy()), - jnp.asarray(t_layer.by.detach().numpy())) - x1_j, y_j = j_layer.explicit_call(params, jx, ju, e) - np.testing.assert_allclose(np.asarray(x1_j), x1_t.numpy(), atol=1e-5) - np.testing.assert_allclose(np.asarray(y_j), y_t.numpy(), atol=1e-5) diff --git a/tests/torch_tests/test_bilipnet.py b/tests/torch_tests/test_bilipnet.py new file mode 100644 index 0000000..a4fc46c --- /dev/null +++ b/tests/torch_tests/test_bilipnet.py @@ -0,0 +1,57 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `BiLipNet`.""" + +import numpy as np +import pytest +import torch + +import golden +from robustnn.bilipnet_torch import BiLipNet +from torch_utils import check_golden, empirical_gains, grads_finite + + +relu_np = lambda x: np.maximum(0, x) + + +# ------------------------------- BiLipNet --------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("depth", [1, 2, 3]) +def test_bilipnet(batches, depth): + size, mu, nu = 3, 0.5, 4.0 + net = BiLipNet(size, [4, 4], mu=mu, nu=nu, tau=nu / mu, + is_mu_fixed=True, is_nu_fixed=True, depth=depth) + x = torch.randn(batches, size) + y = net(x) + assert tuple(y.shape) == (batches, size) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + + lo, hi, _ = net.get_bounds() + np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-4) + gains = empirical_gains(net, 200, size) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + (y ** 2).sum().backward() + grads_finite(net) + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_bilipnet_inverse_round_trip(depth): + net = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, + is_nu_fixed=True, depth=depth) + x = torch.randn(6, 3) + with torch.no_grad(): + y = net(x) + x_rec = net.inverse(y.numpy(), alphas=[0.1] * depth, + inverse_activation_fns=[relu_np] * depth, + iterations=[500] * depth, Lambdas=[1.0] * depth) + np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) + + +@pytest.mark.parametrize("case", golden.cases("bilipnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipnet_matches_golden(case, explicit): + check_golden("bilipnet", case, explicit) diff --git a/tests/torch_tests/test_bilipschitz_ren.py b/tests/torch_tests/test_bilipschitz_ren.py new file mode 100644 index 0000000..b9d6402 --- /dev/null +++ b/tests/torch_tests/test_bilipschitz_ren.py @@ -0,0 +1,70 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `BiLipschitzREN`.""" + +import pytest +import torch +import torch.nn as nn + +import golden +from robustnn.ren_torch import BiLipschitzREN, RENBase +from torch_utils import check_golden, grads_finite, rel_err + + +# ------------------------------ BiLipschitzREN ---------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("nio", [1, 3]) +@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) +@pytest.mark.parametrize("activation", [nn.ReLU(), nn.Tanh()]) +def test_bilipschitz_ren(batches, nio, mu_nu, activation): + mu, nu = mu_nu + nx, nv = 4, 8 + model = BiLipschitzREN(nio, nx, nv, mu=mu, nu=nu, activation=activation) + + state = model.initialize_carry(batches) + assert tuple(state.shape) == (batches, nx) + u = torch.randn(batches, nio) + new_state, y = model(state, u) + assert tuple(new_state.shape) == (batches, nx) + assert tuple(y.shape) == (batches, nio) + + with torch.no_grad(): + e = model.direct_to_explicit() + _, y_exp = model.explicit_call(state, u, e) + torch.testing.assert_close(y_exp, y, atol=1e-4, rtol=1e-4) + + if isinstance(activation, nn.ReLU): + with torch.no_grad(): + _, u_rec = model.inverse(state, y) + assert rel_err(u_rec, u) < 1e-3 + + n = 300 + z = torch.zeros(n, nx) + ua, ub = torch.randn(n, nio), torch.randn(n, nio) + with torch.no_grad(): + _, ya = model.explicit_call(z, ua, e) + _, yb = model.explicit_call(z, ub, e) + gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) + assert gain.min().item() >= mu - 1e-3 + assert gain.max().item() <= nu + 1e-3 + + (y ** 2).sum().backward() + grads_finite(model) + + +@pytest.mark.parametrize("mu,nu", [(2.0, 1.0), (1.0, 1.0)]) +def test_bilipschitz_ren_invalid_bounds(mu, nu): + with pytest.raises(ValueError): + BiLipschitzREN(2, 3, 4, mu=mu, nu=nu) + + +def test_ren_base_direct_to_explicit_not_implemented(): + base = RENBase(2, 3, 4, 2) + with pytest.raises(NotImplementedError): + base.direct_to_explicit() + + +@pytest.mark.parametrize("case", golden.cases("bilipschitz_ren"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipschitz_ren_matches_golden(case, explicit): + check_golden("bilipschitz_ren", case, explicit) diff --git a/tests/torch_tests/test_dyn_orthogonal.py b/tests/torch_tests/test_dyn_orthogonal.py new file mode 100644 index 0000000..4f17d91 --- /dev/null +++ b/tests/torch_tests/test_dyn_orthogonal.py @@ -0,0 +1,40 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `DynUnitary`.""" + +import pytest +import torch + +import golden +from robustnn.dyn_orthogonal_torch import DynUnitary +from torch_utils import check_golden, grads_finite + + +# ------------------------------- DynUnitary ------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) +@pytest.mark.parametrize("bias", [True, False]) +def test_dyn_unitary(batches, io, nx, bias): + layer = DynUnitary(io, nx, bias=bias) + x, u = torch.randn(batches, nx), torch.randn(batches, io) + assert tuple(layer.initialize_carry(batches).shape) == (batches, nx) + + x1, y = layer(x, u) + assert tuple(x1.shape) == (batches, nx) and tuple(y.shape) == (batches, io) + + A, B, C, D = layer._blocks() + G = torch.cat([torch.cat([A, B], 1), torch.cat([C, D], 1)], 0) + torch.testing.assert_close(G @ G.T, torch.eye(nx + io), atol=1e-5, rtol=0) + + x_rec, u_rec = layer.inverse(x1, y) + torch.testing.assert_close(x_rec, x, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(u_rec, u, atol=1e-4, rtol=1e-4) + + (x1 ** 2).sum().add((y ** 2).sum()).backward() + grads_finite(layer) + + +@pytest.mark.parametrize("case", golden.cases("dyn_unitary"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_dyn_unitary_matches_golden(case, explicit): + check_golden("dyn_unitary", case, explicit) diff --git a/tests/torch_tests/test_lipnets.py b/tests/torch_tests/test_lipnets.py deleted file mode 100644 index b215e61..0000000 --- a/tests/torch_tests/test_lipnets.py +++ /dev/null @@ -1,169 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `Unitary`, `MonLipNet`, `BiLipNet` and `PLNet`.""" - -import numpy as np -import pytest -import torch -import torch.nn as nn - -from robustnn.bilipnet_torch import BiLipNet -from robustnn.monlipnet_torch import MonLipNet -from robustnn.orthogonal_torch import Unitary -from robustnn.plnet_torch import PLNet - - -def _gains(f, n, size): - ua, ub = torch.randn(n, size), torch.randn(n, size) - with torch.no_grad(): - return (f(ua) - f(ub)).norm(dim=1) / (ua - ub).norm(dim=1) - - -def _grads_finite(model): - grads = [p.grad for p in model.parameters() if p.grad is not None] - assert grads - assert all(bool(torch.isfinite(g).all()) for g in grads) - - -relu_np = lambda x: np.maximum(0, x) - - -# -------------------------------- Unitary --------------------------------- -@pytest.mark.parametrize("batches", [1, 7]) -@pytest.mark.parametrize("size", [1, 2, 5]) -def test_unitary(batches, size): - layer = Unitary(size, size, bias=True) - x = torch.randn(batches, size) - y = layer(x) - assert tuple(y.shape) == (batches, size) - - e = layer.direct_to_explicit() - np.testing.assert_allclose(e.Q @ e.Q.T, np.eye(size), atol=1e-5) - np.testing.assert_allclose(layer.explicit_call(x.numpy(), e), - y.detach().numpy(), atol=1e-5) - - with torch.no_grad(): - x_rec = layer.inverse(y) - np.testing.assert_allclose(np.asarray(x_rec), x.numpy(), atol=1e-4) - - (y ** 2).sum().backward() - _grads_finite(layer) - - -@pytest.mark.parametrize("size", [2, 5]) -def test_unitary_no_bias_forward_preserves_norm(size): - layer = Unitary(size, size, bias=False) - x = torch.randn(8, size) - np.testing.assert_allclose(layer(x).norm(dim=1).detach(), x.norm(dim=1), rtol=1e-4) - - -def test_unitary_no_bias_explicit(): - Unitary(3, 3, bias=False).direct_to_explicit() - - -# ------------------------------- MonLipNet -------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) -@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) -def test_monlipnet(batches, units, fixed): - size, mu, nu, tau = 3, 0.5, 4.0, 4.0 - net = MonLipNet(size, units, mu=mu, nu=nu, tau=tau, - is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, - is_tau_fixed="tau" in fixed) - x = torch.randn(batches, size) - y = net(x) - assert tuple(y.shape) == (batches, size) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - - lo, hi, _ = net.get_bounds() - gains = _gains(net, 200, size) - assert gains.min() >= float(lo) - 1e-3 - assert gains.max() <= float(hi) + 1e-3 - - (y ** 2).sum().backward() - _grads_finite(net) - - -@pytest.mark.parametrize("units", [[3], [4, 4]]) -def test_monlipnet_inverse_round_trip(units): - net = MonLipNet(3, units, mu=0.5, nu=4.0, tau=4.0, is_tau_fixed=True) - x = torch.randn(6, 3) - with torch.no_grad(): - y = net(x) - x_rec = net.inverse(y.numpy(), alpha=0.1, iterations=500) - np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) - - -@pytest.mark.parametrize("kw", [dict(mu=1.0), dict(), dict(nu=2.0)]) -def test_monlipnet_needs_two_bounds(kw): - with pytest.raises(ValueError): - MonLipNet(2, [2], **kw) - - -# ------------------------------- BiLipNet --------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("depth", [1, 2, 3]) -def test_bilipnet(batches, depth): - size, mu, nu = 3, 0.5, 4.0 - net = BiLipNet(size, [4, 4], mu=mu, nu=nu, tau=nu / mu, - is_mu_fixed=True, is_nu_fixed=True, depth=depth) - x = torch.randn(batches, size) - y = net(x) - assert tuple(y.shape) == (batches, size) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - - lo, hi, _ = net.get_bounds() - np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-4) - gains = _gains(net, 200, size) - assert gains.min() >= mu - 1e-3 - assert gains.max() <= nu + 1e-3 - - (y ** 2).sum().backward() - _grads_finite(net) - - -@pytest.mark.parametrize("depth", [1, 2]) -def test_bilipnet_inverse_round_trip(depth): - net = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, - is_nu_fixed=True, depth=depth) - x = torch.randn(6, 3) - with torch.no_grad(): - y = net(x) - x_rec = net.inverse(y.numpy(), alphas=[0.1] * depth, - inverse_activation_fns=[relu_np] * depth, - iterations=[500] * depth, Lambdas=[1.0] * depth) - np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) - - -# --------------------------------- PLNet ---------------------------------- -def _plnet(**kw): - bilip = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, - is_nu_fixed=True, depth=2) - return PLNet(BiLipBlock=bilip, **kw) - - -@pytest.mark.parametrize("batches", [1, 6]) -def test_plnet(batches): - net = _plnet() - x = torch.randn(batches, 3) - y = net(x) - assert tuple(y.shape) == (batches,) - assert bool((y >= 0).all()) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - y.sum().backward() - _grads_finite(net) - - -def test_plnet_zero_at_optimal_point(): - net = _plnet() - x = torch.randn(4, 3) - e = net.direct_to_explicit(x_optimal=x[:1].numpy()) - y = net.explicit_call(x[:1].numpy(), e) - np.testing.assert_allclose(y, 0.0, atol=1e-6) - assert bool((net.explicit_call(x.numpy(), e) >= 0).all()) diff --git a/tests/torch_tests/test_monlipnet.py b/tests/torch_tests/test_monlipnet.py new file mode 100644 index 0000000..5311162 --- /dev/null +++ b/tests/torch_tests/test_monlipnet.py @@ -0,0 +1,58 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `MonLipNet`.""" + +import numpy as np +import pytest +import torch + +import golden +from robustnn.monlipnet_torch import MonLipNet +from torch_utils import check_golden, empirical_gains, grads_finite + + +# ------------------------------- MonLipNet -------------------------------- +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) +@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) +def test_monlipnet(batches, units, fixed): + size, mu, nu, tau = 3, 0.5, 4.0, 4.0 + net = MonLipNet(size, units, mu=mu, nu=nu, tau=tau, + is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, + is_tau_fixed="tau" in fixed) + x = torch.randn(batches, size) + y = net(x) + assert tuple(y.shape) == (batches, size) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + + lo, hi, _ = net.get_bounds() + gains = empirical_gains(net, 200, size) + assert gains.min() >= float(lo) - 1e-3 + assert gains.max() <= float(hi) + 1e-3 + + (y ** 2).sum().backward() + grads_finite(net) + + +@pytest.mark.parametrize("units", [[3], [4, 4]]) +def test_monlipnet_inverse_round_trip(units): + net = MonLipNet(3, units, mu=0.5, nu=4.0, tau=4.0, is_tau_fixed=True) + x = torch.randn(6, 3) + with torch.no_grad(): + y = net(x) + x_rec = net.inverse(y.numpy(), alpha=0.1, iterations=500) + np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) + + +@pytest.mark.parametrize("kw", [dict(mu=1.0), dict(), dict(nu=2.0)]) +def test_monlipnet_needs_two_bounds(kw): + with pytest.raises(ValueError): + MonLipNet(2, [2], **kw) + + +@pytest.mark.parametrize("case", golden.cases("monlipnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_monlipnet_matches_golden(case, explicit): + check_golden("monlipnet", case, explicit) diff --git a/tests/torch_tests/test_orthogonal.py b/tests/torch_tests/test_orthogonal.py new file mode 100644 index 0000000..f091f78 --- /dev/null +++ b/tests/torch_tests/test_orthogonal.py @@ -0,0 +1,50 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `Unitary` layer.""" + +import numpy as np +import pytest +import torch + +import golden +from robustnn.orthogonal_torch import Unitary +from torch_utils import check_golden, grads_finite + + +# -------------------------------- Unitary --------------------------------- +@pytest.mark.parametrize("batches", [1, 7]) +@pytest.mark.parametrize("size", [1, 2, 5]) +def test_unitary(batches, size): + layer = Unitary(size, size, bias=True) + x = torch.randn(batches, size) + y = layer(x) + assert tuple(y.shape) == (batches, size) + + e = layer.direct_to_explicit() + np.testing.assert_allclose(e.Q @ e.Q.T, np.eye(size), atol=1e-5) + np.testing.assert_allclose(layer.explicit_call(x.numpy(), e), + y.detach().numpy(), atol=1e-5) + + with torch.no_grad(): + x_rec = layer.inverse(y) + np.testing.assert_allclose(np.asarray(x_rec), x.numpy(), atol=1e-4) + + (y ** 2).sum().backward() + grads_finite(layer) + + +@pytest.mark.parametrize("size", [2, 5]) +def test_unitary_no_bias_forward_preserves_norm(size): + layer = Unitary(size, size, bias=False) + x = torch.randn(8, size) + np.testing.assert_allclose(layer(x).norm(dim=1).detach(), x.norm(dim=1), rtol=1e-4) + + +def test_unitary_no_bias_explicit(): + Unitary(3, 3, bias=False).direct_to_explicit() + + +@pytest.mark.parametrize("case", golden.cases("unitary"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_unitary_matches_golden(case, explicit): + check_golden("unitary", case, explicit) diff --git a/tests/torch_tests/test_plnet.py b/tests/torch_tests/test_plnet.py new file mode 100644 index 0000000..4cba659 --- /dev/null +++ b/tests/torch_tests/test_plnet.py @@ -0,0 +1,48 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `PLNet`.""" + +import numpy as np +import pytest +import torch + +import golden +from robustnn.bilipnet_torch import BiLipNet +from robustnn.plnet_torch import PLNet +from torch_utils import check_golden, grads_finite + + +# --------------------------------- PLNet ---------------------------------- +def _plnet(**kw): + bilip = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, + is_nu_fixed=True, depth=2) + return PLNet(BiLipBlock=bilip, **kw) + + +@pytest.mark.parametrize("batches", [1, 6]) +def test_plnet(batches): + net = _plnet() + x = torch.randn(batches, 3) + y = net(x) + assert tuple(y.shape) == (batches,) + assert bool((y >= 0).all()) + np.testing.assert_allclose( + net.explicit_call(x.numpy(), net.direct_to_explicit()), + y.detach().numpy(), atol=1e-4) + y.sum().backward() + grads_finite(net) + + +def test_plnet_zero_at_optimal_point(): + net = _plnet() + x = torch.randn(4, 3) + e = net.direct_to_explicit(x_optimal=x[:1].numpy()) + y = net.explicit_call(x[:1].numpy(), e) + np.testing.assert_allclose(y, 0.0, atol=1e-6) + assert bool((net.explicit_call(x.numpy(), e) >= 0).all()) + + +@pytest.mark.parametrize("case", golden.cases("plnet"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_plnet_matches_golden(case, explicit): + check_golden("plnet", case, explicit) diff --git a/tests/torch_tests/test_ren.py b/tests/torch_tests/test_ren.py deleted file mode 100644 index 10e8f4d..0000000 --- a/tests/torch_tests/test_ren.py +++ /dev/null @@ -1,176 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `BiLipschitzREN`, `DynUnitary` and `CompositionREN`.""" - -import pytest -import torch -import torch.nn as nn - -from robustnn.dyn_orthogonal_torch import DynUnitary -from robustnn.ren_composition_torch import CompositionREN -from robustnn.ren_torch import BiLipschitzREN, RENBase - - -def _grads_finite(model): - grads = [p.grad for p in model.parameters() if p.grad is not None] - assert grads - assert all(bool(torch.isfinite(g).all()) for g in grads) - - -def _rel_err(a, b): - return ((a - b).norm() / b.norm()).item() - - -# ------------------------------ BiLipschitzREN ---------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("nio", [1, 3]) -@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) -@pytest.mark.parametrize("activation", [nn.ReLU(), nn.Tanh()]) -def test_bilipschitz_ren(batches, nio, mu_nu, activation): - mu, nu = mu_nu - nx, nv = 4, 8 - model = BiLipschitzREN(nio, nx, nv, mu=mu, nu=nu, activation=activation) - - state = model.initialize_carry(batches) - assert tuple(state.shape) == (batches, nx) - u = torch.randn(batches, nio) - new_state, y = model(state, u) - assert tuple(new_state.shape) == (batches, nx) - assert tuple(y.shape) == (batches, nio) - - with torch.no_grad(): - e = model.direct_to_explicit() - _, y_exp = model.explicit_call(state, u, e) - torch.testing.assert_close(y_exp, y, atol=1e-4, rtol=1e-4) - - if isinstance(activation, nn.ReLU): - with torch.no_grad(): - _, u_rec = model.inverse(state, y) - assert _rel_err(u_rec, u) < 1e-3 - - n = 300 - z = torch.zeros(n, nx) - ua, ub = torch.randn(n, nio), torch.randn(n, nio) - with torch.no_grad(): - _, ya = model.explicit_call(z, ua, e) - _, yb = model.explicit_call(z, ub, e) - gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - - (y ** 2).sum().backward() - _grads_finite(model) - - -@pytest.mark.parametrize("mu,nu", [(2.0, 1.0), (1.0, 1.0)]) -def test_bilipschitz_ren_invalid_bounds(mu, nu): - with pytest.raises(ValueError): - BiLipschitzREN(2, 3, 4, mu=mu, nu=nu) - - -def test_ren_base_direct_to_explicit_not_implemented(): - base = RENBase(2, 3, 4, 2) - with pytest.raises(NotImplementedError): - base.direct_to_explicit() - - -# ------------------------------- DynUnitary ------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) -@pytest.mark.parametrize("bias", [True, False]) -def test_dyn_unitary(batches, io, nx, bias): - layer = DynUnitary(io, nx, bias=bias) - x, u = torch.randn(batches, nx), torch.randn(batches, io) - assert tuple(layer.initialize_carry(batches).shape) == (batches, nx) - - x1, y = layer(x, u) - assert tuple(x1.shape) == (batches, nx) and tuple(y.shape) == (batches, io) - - A, B, C, D = layer._blocks() - G = torch.cat([torch.cat([A, B], 1), torch.cat([C, D], 1)], 0) - torch.testing.assert_close(G @ G.T, torch.eye(nx + io), atol=1e-5, rtol=0) - - x_rec, u_rec = layer.inverse(x1, y) - torch.testing.assert_close(x_rec, x, atol=1e-4, rtol=1e-4) - torch.testing.assert_close(u_rec, u, atol=1e-4, rtol=1e-4) - - (x1 ** 2).sum().add((y ** 2).sum()).backward() - _grads_finite(layer) - - -# ------------------------------ CompositionREN ---------------------------- -@pytest.mark.parametrize("batches", [1, 6]) -@pytest.mark.parametrize("L", [1, 3]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_composition_ren(batches, L, use_bias): - io, nx, nv, mu, nu = 2, 3, 6, 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, use_bias=use_bias) - - carry = model.initialize_carry(batches) - u = torch.randn(batches, io) - new_carry, y = model(carry, u) - assert len(new_carry["rens"]) == L - assert tuple(y.shape) == (batches, io) - assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None - - with torch.no_grad(): - _, u_rec = model.inverse(carry, y) - assert _rel_err(u_rec, u) < 1e-3 - - n = 300 - zc = {"rens": [torch.zeros(n, nx) for _ in range(L)], - "dyn_in": None, "dyn_out": None} - ua, ub = torch.randn(n, io), torch.randn(n, io) - with torch.no_grad(): - _, ya = model(zc, ua) - _, yb = model(zc, ub) - gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - assert model.get_bounds() == (mu, nu) - - (y ** 2).sum().backward() - _grads_finite(model) - - -@pytest.mark.parametrize("at_input,at_output", - [(True, False), (False, True), (True, True)]) -def test_composition_ren_dyn_orth_noncausal_inverse(at_input, at_output): - io, nx, L = 2, 3, 2 - model = CompositionREN(io, nx, 6, num_layers=L, mu=0.5, nu=5.0, - dyn_orth_at_input=at_input, dyn_orth_at_output=at_output, - dyn_state_multiplier=4) - carry = model.initialize_carry(5) - assert (carry["dyn_in"] is not None) == at_input - assert (carry["dyn_out"] is not None) == at_output - - u = torch.randn(5, io) - new_carry, y = model(carry, u) - assert tuple(y.shape) == (5, io) - if at_input: - assert tuple(new_carry["dyn_in"].shape) == (5, 4 * nx) - - with torch.no_grad(): - _, u_rec = model.inverse_noncausal(carry, new_carry, y) - assert _rel_err(u_rec, u) < 1e-3 - - -def test_composition_ren_ignore_inverse_with_dyn_in(): - model = CompositionREN(2, 3, 6, num_layers=2, mu=0.5, nu=5.0, - dyn_orth_at_input=True, dyn_state_multiplier=4) - carry = model.initialize_carry(5) - u = torch.randn(5, 2) - _, y = model(carry, u) - with torch.no_grad(): - _, sig_rec = model.inverse(carry, y) - _, dyn_out = model.dyn_in(carry["dyn_in"], u) - assert _rel_err(sig_rec, dyn_out) < 1e-3 - - -@pytest.mark.parametrize("kw", [dict(num_layers=0), dict(mu=2.0, nu=1.0), - dict(mu=1.0, nu=1.0)]) -def test_composition_ren_invalid_args(kw): - args = dict(num_layers=2, mu=0.5, nu=5.0) - args.update(kw) - with pytest.raises(ValueError): - CompositionREN(2, 3, 6, **args) diff --git a/tests/torch_tests/test_ren_composition.py b/tests/torch_tests/test_ren_composition.py new file mode 100644 index 0000000..e625c53 --- /dev/null +++ b/tests/torch_tests/test_ren_composition.py @@ -0,0 +1,94 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the PyTorch `CompositionREN`.""" + +import pytest +import torch + +import golden +from robustnn.ren_composition_torch import CompositionREN +from torch_utils import check_golden, grads_finite, rel_err + + +# ------------------------------ CompositionREN ---------------------------- +@pytest.mark.parametrize("batches", [1, 6]) +@pytest.mark.parametrize("L", [1, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_composition_ren(batches, L, use_bias): + io, nx, nv, mu, nu = 2, 3, 6, 0.5, 5.0 + model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, use_bias=use_bias) + + carry = model.initialize_carry(batches) + u = torch.randn(batches, io) + new_carry, y = model(carry, u) + assert len(new_carry["rens"]) == L + assert tuple(y.shape) == (batches, io) + assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None + + with torch.no_grad(): + _, u_rec = model.inverse(carry, y) + assert rel_err(u_rec, u) < 1e-3 + + n = 300 + zc = {"rens": [torch.zeros(n, nx) for _ in range(L)], + "dyn_in": None, "dyn_out": None} + ua, ub = torch.randn(n, io), torch.randn(n, io) + with torch.no_grad(): + _, ya = model(zc, ua) + _, yb = model(zc, ub) + gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) + assert gain.min().item() >= mu - 1e-3 + assert gain.max().item() <= nu + 1e-3 + assert model.get_bounds() == (mu, nu) + + (y ** 2).sum().backward() + grads_finite(model) + + +@pytest.mark.parametrize("at_input,at_output", + [(True, False), (False, True), (True, True)]) +def test_composition_ren_dyn_orth_noncausal_inverse(at_input, at_output): + io, nx, L = 2, 3, 2 + model = CompositionREN(io, nx, 6, num_layers=L, mu=0.5, nu=5.0, + dyn_orth_at_input=at_input, dyn_orth_at_output=at_output, + dyn_state_multiplier=4) + carry = model.initialize_carry(5) + assert (carry["dyn_in"] is not None) == at_input + assert (carry["dyn_out"] is not None) == at_output + + u = torch.randn(5, io) + new_carry, y = model(carry, u) + assert tuple(y.shape) == (5, io) + if at_input: + assert tuple(new_carry["dyn_in"].shape) == (5, 4 * nx) + + with torch.no_grad(): + _, u_rec = model.inverse_noncausal(carry, new_carry, y) + assert rel_err(u_rec, u) < 1e-3 + + +def test_composition_ren_ignore_inverse_with_dyn_in(): + model = CompositionREN(2, 3, 6, num_layers=2, mu=0.5, nu=5.0, + dyn_orth_at_input=True, dyn_state_multiplier=4) + carry = model.initialize_carry(5) + u = torch.randn(5, 2) + _, y = model(carry, u) + with torch.no_grad(): + _, sig_rec = model.inverse(carry, y) + _, dyn_out = model.dyn_in(carry["dyn_in"], u) + assert rel_err(sig_rec, dyn_out) < 1e-3 + + +@pytest.mark.parametrize("kw", [dict(num_layers=0), dict(mu=2.0, nu=1.0), + dict(mu=1.0, nu=1.0)]) +def test_composition_ren_invalid_args(kw): + args = dict(num_layers=2, mu=0.5, nu=5.0) + args.update(kw) + with pytest.raises(ValueError): + CompositionREN(2, 3, 6, **args) + + +@pytest.mark.parametrize("case", golden.cases("ren_composition"), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_ren_composition_matches_golden(case, explicit): + check_golden("ren_composition", case, explicit) diff --git a/tests/torch_utils.py b/tests/torch_utils.py new file mode 100644 index 0000000..583516f --- /dev/null +++ b/tests/torch_utils.py @@ -0,0 +1,219 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Shared helpers for the PyTorch tests.""" + +import numpy as np +import pytest +import torch +import torch.nn as nn + +import golden + + +def empirical_gains(f, n, size): + ua, ub = torch.randn(n, size), torch.randn(n, size) + with torch.no_grad(): + return (f(ua) - f(ub)).norm(dim=1) / (ua - ub).norm(dim=1) + + +def grads_finite(model): + grads = [p.grad for p in model.parameters() if p.grad is not None] + assert grads + assert all(bool(torch.isfinite(g).all()) for g in grads) + + + +def rel_err(a, b): + return ((a - b).norm() / b.norm()).item() + + +# --------------------------------------------------------------------------- +# Golden-file support (see golden.py) +# --------------------------------------------------------------------------- +_MODULE_ACT = {"relu": nn.ReLU, "tanh": nn.Tanh, "sigmoid": nn.Sigmoid, "identity": nn.Identity} +_FN_ACT = {"relu": torch.relu, "tanh": torch.tanh, "sigmoid": torch.sigmoid, + "identity": lambda x: x} + + +def build_model(network, config): + """Construct the torch module for a golden-file `config` (JAX argument names).""" + from robustnn.bilipnet_torch import BiLipNet + from robustnn.dyn_orthogonal_torch import DynUnitary + from robustnn.monlipnet_torch import MonLipNet + from robustnn.orthogonal_torch import Unitary + from robustnn.plnet_torch import PLNet + from robustnn.ren_composition_torch import CompositionREN + from robustnn.ren_torch import BiLipschitzREN + + c = config + if network == "unitary": + return Unitary(c["input_size"], c["input_size"], bias=c.get("use_bias", True)) + if network in ("monlipnet", "bilipnet"): + kw = dict(features=c["input_size"], unit_features=list(c["units"]), + mu=c.get("mu"), nu=c.get("nu"), tau=c.get("tau"), + is_mu_fixed=c.get("is_mu_fixed", False), + is_nu_fixed=c.get("is_nu_fixed", False), + is_tau_fixed=c.get("is_tau_fixed", False)) + if "act_fn" in c: + kw["act"] = _MODULE_ACT[c["act_fn"]]() + if network == "bilipnet": + return BiLipNet(depth=c.get("depth", 2), **kw) + return MonLipNet(**kw) + if network == "plnet": + op = c.get("optimal_point") + return PLNet( + BiLipBlock=build_model("bilipnet", c["BiLipBlock"]), + add_constant=c.get("add_constant", False), + optimal_point=None if op is None else torch.tensor(op, dtype=torch.float32), + ) + if network == "dyn_unitary": + return DynUnitary(c["input_size"], c["state_size"], bias=c.get("use_bias", True)) + if network == "bilipschitz_ren": + act = _MODULE_ACT[c.get("activation", "relu")]() + return BiLipschitzREN(c["input_size"], c["state_size"], c["features"], + mu=c["mu"], nu=c["nu"], activation=act) + if network == "ren_composition": + return CompositionREN( + c["input_size"], c["state_size"], c["features"], c["num_layers"], + mu=c["mu"], nu=c["nu"], activation=_FN_ACT[c.get("activation", "relu")], + use_bias=c.get("use_bias", True), + dyn_orth_at_input=c.get("dyn_orth_at_input", False), + dyn_orth_at_output=c.get("dyn_orth_at_output", False), + dyn_state_multiplier=c.get("dyn_state_multiplier", 50), + ) + raise KeyError(f"no PyTorch implementation of '{network}'") + + +# --- JAX/Flax parameter layout -> torch state_dict --------------------------- +def _unitary_sd(p, prefix=""): + sd = {f"{prefix}weight": p["W"], f"{prefix}alpha": p["a"]} + if "b" in p: + sd[f"{prefix}bias"] = p["b"] + return sd + + +def _monlip_sd(p, prefix=""): + sd = {f"{prefix}Fq": p["Fq"].T, f"{prefix}fq": p["fq"], f"{prefix}by": p["by"]} + for k in range(sum(1 for name in p if name.startswith("Fab"))): + sd[f"{prefix}Fr.{k}"] = p[f"Fab{k}"].T + sd[f"{prefix}fr.{k}"] = p[f"fab{k}"] + sd[f"{prefix}bs.{k}"] = p[f"b{k}"] + return sd + + +def _bilip_sd(p, prefix=""): + sd = {} + for name, sub in p.items(): + kind, k = name.split("_") + if kind == "uni": + sd.update(_unitary_sd(sub, f"{prefix}orth_layers.{k}.")) + else: + sd.update(_monlip_sd(sub, f"{prefix}mon_layers.{k}.")) + return sd + + +def _flat_sd(p, prefix=""): + return {f"{prefix}{k}": v for k, v in p.items()} + + +def _ren_sd(p, prefix=""): + # The JAX BiLipschitzREN carries a `D22` direct parameter that its QSR + # parameterisation never reads (D22 is derived from Q, S, R); torch has none. + return _flat_sd({k: v for k, v in p.items() if k != "D22"}, prefix) + + +def to_state_dict(network, params): + """Translate golden (JAX-layout) `params` to a torch `state_dict`.""" + if network == "unitary": + return _unitary_sd(params) + if network == "monlipnet": + return _monlip_sd(params) + if network == "bilipnet": + return _bilip_sd(params) + if network == "plnet": + sd = _bilip_sd(params["BiLipBlock"], "bln.") + if "c" in params: + sd["bias"] = params["c"] + return sd + if network == "dyn_unitary": + return _flat_sd(params) + if network == "bilipschitz_ren": + return _ren_sd(params) + if network == "ren_composition": + sd = {} + for name, sub in params.items(): + if name in ("input_orth", "output_orth"): + sd.update(_unitary_sd(sub, f"{name}.")) + elif name.startswith("orths_"): + sd.update(_unitary_sd(sub, f"orths.{name.split('_')[1]}.")) + elif name.startswith("rens_"): + sd.update(_ren_sd(sub, f"rens.{name.split('_')[1]}.")) + else: # dyn_in / dyn_out + sd.update(_flat_sd(sub, f"{name}.")) + return sd + raise KeyError(network) + + +# Bounds are constructor arguments (buffers/parameters named mu, nu, tau): they +# come from the config, not from the golden parameters. +_CONFIG_KEYS = ("mu", "nu", "tau") + + +def load_params(network, model, params): + sd = {k: torch.tensor(np.asarray(v), dtype=torch.float32) + for k, v in to_state_dict(network, params).items()} + missing, unexpected = model.load_state_dict(sd, strict=False) + missing = [k for k in missing if k.split(".")[-1] not in _CONFIG_KEYS] + assert not unexpected, f"unexpected keys: {unexpected}" + assert not missing, f"keys not set from golden params: {missing}" + + +def _t(a): + return None if a is None else torch.tensor(np.asarray(a), dtype=torch.float32) + + +def _n(t): + return None if t is None else t.detach().numpy() + + +def run_model(network, model, inputs, explicit=False): + """Evaluate `model` on golden `inputs` (forward pass or explicit path). + + Returns None if the explicit path is not available for this network. + """ + with torch.no_grad(): + if network in ("unitary", "monlipnet", "bilipnet", "plnet"): + if explicit: + x = np.asarray(inputs["x"]) + return {"y": np.asarray(model.explicit_call(x, model.direct_to_explicit()))} + return {"y": _n(model(_t(inputs["x"])))} + + if network == "ren_composition": + if explicit: + return None + c = inputs["carry"] + carry = {"rens": [_t(r) for r in c["rens"]], + "dyn_in": _t(c["dyn_in"]), "dyn_out": _t(c["dyn_out"])} + new, y = model(carry, _t(inputs["u"])) + return {"carry": {"rens": np.stack([_n(r) for r in new["rens"]]), + "dyn_in": _n(new["dyn_in"]), "dyn_out": _n(new["dyn_out"])}, + "y": _n(y)} + + x, u = _t(inputs["state"]), _t(inputs["u"]) + if explicit: + if network != "bilipschitz_ren": + return None + xn, y = model.explicit_call(x, u, model.direct_to_explicit()) + else: + xn, y = model(x, u) + return {"next_state": _n(xn), "y": _n(y)} + + +def check_golden(network, case, explicit=False): + """Check a golden case against the torch model (forward or explicit path).""" + model = build_model(network, case["config"]) + load_params(network, model, case["params"]) + out = run_model(network, model, case["inputs"], explicit=explicit) + if out is None: + pytest.skip(f"no explicit path for torch '{network}'") + golden.assert_close(out, case["outputs"]) From b461235ed9010834cd757b88e1aac2bccdd7ef05 Mon Sep 17 00:00:00 2001 From: juraph Date: Wed, 14 Oct 2026 15:23:03 +1100 Subject: [PATCH 4/4] Unified testing files No longer have separate jax/torch testing files. I wasn't happy with the likelihood of someone writing a test for one of the models, and forgetting to implement it for the other. By having this unified schema, one writes the test once, and both versions of the network are compared against it. A few notes: 1. Not all networks have been implemented in torch. Their files state this explicitly, and they will need to be added to this testing once implemented. 2. There are often API differences between the two network backends that definitely do not need to exist. We will address this with the refactor. This is where the single backends.py is surprisingly useful. --- README.md | 2 +- tests/backends.py | 691 ++++++++++++++++++++++ tests/conftest.py | 44 +- tests/generate_golden.py | 6 +- tests/golden.py | 2 +- tests/jax_tests/conftest.py | 19 - tests/jax_tests/test_bilipnet.py | 76 --- tests/jax_tests/test_bilipschitz_ren.py | 55 -- tests/jax_tests/test_contracting_ren.py | 91 --- tests/jax_tests/test_dyn_orthogonal.py | 50 -- tests/jax_tests/test_general_ren.py | 45 -- tests/jax_tests/test_lbdn.py | 63 -- tests/jax_tests/test_linear_ren.py | 69 --- tests/jax_tests/test_lipschitz_ren.py | 45 -- tests/jax_tests/test_monlipnet.py | 73 --- tests/jax_tests/test_orthogonal.py | 75 --- tests/jax_tests/test_plnet.py | 67 --- tests/jax_tests/test_r2dn.py | 48 -- tests/jax_tests/test_ren_composition.py | 131 ---- tests/jax_utils.py | 177 ------ tests/test_bilipnet.py | 70 +++ tests/test_bilipschitz_ren.py | 71 +++ tests/test_contracting_ren.py | 113 ++++ tests/test_dyn_orthogonal.py | 36 ++ tests/test_general_ren.py | 60 ++ tests/test_lbdn.py | 53 ++ tests/test_linear_ren.py | 64 ++ tests/test_lipschitz_ren.py | 41 ++ tests/test_monlipnet.py | 61 ++ tests/test_orthogonal.py | 54 ++ tests/test_plnet.py | 48 ++ tests/test_r2dn.py | 61 ++ tests/test_ren_composition.py | 91 +++ tests/torch_tests/conftest.py | 18 - tests/torch_tests/test_bilipnet.py | 57 -- tests/torch_tests/test_bilipschitz_ren.py | 70 --- tests/torch_tests/test_dyn_orthogonal.py | 40 -- tests/torch_tests/test_monlipnet.py | 58 -- tests/torch_tests/test_orthogonal.py | 50 -- tests/torch_tests/test_plnet.py | 48 -- tests/torch_tests/test_ren_composition.py | 94 --- tests/torch_utils.py | 219 ------- 42 files changed, 1560 insertions(+), 1746 deletions(-) create mode 100644 tests/backends.py delete mode 100644 tests/jax_tests/conftest.py delete mode 100644 tests/jax_tests/test_bilipnet.py delete mode 100644 tests/jax_tests/test_bilipschitz_ren.py delete mode 100644 tests/jax_tests/test_contracting_ren.py delete mode 100644 tests/jax_tests/test_dyn_orthogonal.py delete mode 100644 tests/jax_tests/test_general_ren.py delete mode 100644 tests/jax_tests/test_lbdn.py delete mode 100644 tests/jax_tests/test_linear_ren.py delete mode 100644 tests/jax_tests/test_lipschitz_ren.py delete mode 100644 tests/jax_tests/test_monlipnet.py delete mode 100644 tests/jax_tests/test_orthogonal.py delete mode 100644 tests/jax_tests/test_plnet.py delete mode 100644 tests/jax_tests/test_r2dn.py delete mode 100644 tests/jax_tests/test_ren_composition.py delete mode 100644 tests/jax_utils.py create mode 100644 tests/test_bilipnet.py create mode 100644 tests/test_bilipschitz_ren.py create mode 100644 tests/test_contracting_ren.py create mode 100644 tests/test_dyn_orthogonal.py create mode 100644 tests/test_general_ren.py create mode 100644 tests/test_lbdn.py create mode 100644 tests/test_linear_ren.py create mode 100644 tests/test_lipschitz_ren.py create mode 100644 tests/test_monlipnet.py create mode 100644 tests/test_orthogonal.py create mode 100644 tests/test_plnet.py create mode 100644 tests/test_r2dn.py create mode 100644 tests/test_ren_composition.py delete mode 100644 tests/torch_tests/conftest.py delete mode 100644 tests/torch_tests/test_bilipnet.py delete mode 100644 tests/torch_tests/test_bilipschitz_ren.py delete mode 100644 tests/torch_tests/test_dyn_orthogonal.py delete mode 100644 tests/torch_tests/test_monlipnet.py delete mode 100644 tests/torch_tests/test_orthogonal.py delete mode 100644 tests/torch_tests/test_plnet.py delete mode 100644 tests/torch_tests/test_ren_composition.py delete mode 100644 tests/torch_utils.py diff --git a/README.md b/README.md index 287ba9e..00fe6a4 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This will create a Python virtual environment at `./.venv`, install the pinned P ### Testing -Tests are written with [pytest](https://pytest.org) and live in `tests/`. Each network has its own test file, in `tests/jax_tests/` for the JAX/Flax implementations and `tests/torch_tests/` for the PyTorch implementations (e.g. `test_plnet.py`, `test_bilipschitz_ren.py`). `./run_tests.sh` is a thin wrapper around `uv run pytest`, so any pytest arguments work: +Tests are written with [pytest](https://pytest.org) and live in `tests/`. Each network has a single test file (e.g. `test_plnet.py`, `test_bilipschitz_ren.py`) whose tests run against both the JAX/Flax and PyTorch implementations through the `backend` fixture (see `tests/backends.py`). Tests for networks or features that only exist in JAX use the `jax_backend` fixture, and the PyTorch side is skipped with a `TODO(torch)` note in the file. `./run_tests.sh` is a thin wrapper around `uv run pytest`, so any pytest arguments work: ```bash ./run_tests.sh # everything diff --git a/tests/backends.py b/tests/backends.py new file mode 100644 index 0000000..e0bb345 --- /dev/null +++ b/tests/backends.py @@ -0,0 +1,691 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Thin adapters that let one test exercise both the JAX and PyTorch models. + +A test builds a `Net` through a backend and then only talks to the `Net`: + + net = backend.build("bilipnet", config, inputs) # config uses the JAX argument names + out = net.forward(inputs) # {"y": ...} (numpy) + out = net.explicit(inputs) # same, via direct_to_explicit + net.invert(inputs, out) # round trip, e.g. {"x": ...} + net.gains(n) / net.bounds() / net.grads_finite(inputs) + +`config`, `inputs` and `outputs` use the same layout as the golden files (see +`golden.py`), so the same numbers can be fed to either backend. Anything that +differs between the backends (constructor names, call signatures, parameter +layouts) is hidden in here; keep it that way so the tests stay backend-agnostic. +""" + +import numpy as np +import pytest + +import golden + +# Tolerances shared by every comparison, for both backends. +ATOL = golden.ATOL +RTOL = golden.RTOL + +STATIC = ("unitary", "monlipnet", "bilipnet", "plnet", "lbdn") +TORCH_NETWORKS = ( + "unitary", "monlipnet", "bilipnet", "plnet", "dyn_unitary", + "bilipschitz_ren", "ren_composition", +) + + +# --------------------------------------------------------------------------- +# Backend-independent helpers +# --------------------------------------------------------------------------- +def input_size(network, config): + return config["BiLipBlock"]["input_size"] if network == "plnet" else config["input_size"] + + +def make_inputs(network, config, batch=4, seed=0, zero_state=False): + """Random numpy inputs in the golden layout (the same for both backends).""" + rng = np.random.default_rng(seed) + + def randn(*shape): + return rng.standard_normal(shape).astype(np.float32) + + def state(*shape): + return np.zeros(shape, np.float32) if zero_state else randn(*shape) + + nin = input_size(network, config) + if network in STATIC: + return {"x": randn(batch, nin)} + if network == "ren_composition": + nx, layers = config["state_size"], config["num_layers"] + dyn = config.get("dyn_state_multiplier", 50) * nx + carry = { + "rens": state(layers, batch, nx), + "dyn_in": state(batch, dyn) if config.get("dyn_orth_at_input") else None, + "dyn_out": state(batch, dyn) if config.get("dyn_orth_at_output") else None, + } + return {"carry": carry, "u": randn(batch, nin)} + return {"state": state(batch, config["state_size"]), "u": randn(batch, nin)} + + +def make_qsr(nu, ny, seed=0): + """A valid (Q, S, R) supply-rate triple as numpy arrays: Q < 0 and R - S Q^-1 S^T > 0.""" + rng = np.random.default_rng(seed) + X, Y, S = (rng.standard_normal(s).astype(np.float32) + for s in ((ny, ny), (nu, nu), (nu, ny))) + Q = -X.T @ X + R = S @ np.linalg.solve(Q, S.T) + Y.T @ Y + return Q.astype(np.float32), S, R.astype(np.float32) + + +def _primary(network, inputs): + return inputs["x"] if network in STATIC else inputs["u"] + + +class Net: + """A built model plus its parameters, with a backend-independent interface.""" + + def __init__(self, network, config, model, params=None): + self.network, self.config = network, config + self.model, self.params = model, params + + # -- implemented per backend ------------------------------------------ + def forward(self, inputs): + raise NotImplementedError + + def explicit(self, inputs): + """Outputs via `direct_to_explicit`, or None if there is no such path.""" + raise NotImplementedError + + def invert(self, inputs, outputs, noncausal=False): + """Recover the model inputs from `outputs`, in the layout of `inputs`. + + `noncausal` (ren_composition only) uses the saved dynamic-orthogonal + states in `outputs["carry"]` to recover the true input.""" + raise NotImplementedError + + def bounds(self): + raise NotImplementedError + + def grads_finite(self, inputs): + raise NotImplementedError + + def check_jit(self, inputs): + """Compare against a jit-compiled call (only meaningful for JAX).""" + + def orthogonal_matrix(self): + """The (square) orthogonal matrix of an orthogonal layer.""" + raise NotImplementedError + + def dyn_in_output(self, inputs): + """Output of the input dynamic-orthogonal layer of a ren_composition, + which is what the causal `invert` returns in its place.""" + raise NotImplementedError + + # -- shared --------------------------------------------------------- + def gains(self, n=200, seed=0): + """Empirical gain |f(a) - f(b)| / |a - b| over `n` random input pairs + (stateful models start from the zero state).""" + a = make_inputs(self.network, self.config, n, seed, zero_state=True) + b = make_inputs(self.network, self.config, n, seed + 1, zero_state=True) + ya, yb = self.forward(a)["y"], self.forward(b)["y"] + du = _primary(self.network, a) - _primary(self.network, b) + return np.linalg.norm(ya - yb, axis=1) / np.linalg.norm(du, axis=1) + + +class Backend: + name = None + + def supports(self, network): + raise NotImplementedError + + def build(self, network, config, inputs, params=None): + """Build a `Net`. `params` are golden (JAX-layout) parameters; if omitted + the model keeps its random initialisation.""" + if not self.supports(network): + pytest.skip(f"'{network}' is not implemented for {self.name}") + return self._build(network, config, inputs, params) + + +def check_contract(net, inputs, shapes): + """Checks every network should pass: output shapes, the explicit path + agreeing with the forward pass, a jit-compiled call agreeing (JAX only) + and finite gradients. `shapes` maps output name -> expected shape.""" + out = net.forward(inputs) + for k, shape in shapes.items(): + assert out[k].shape == shape, k + exp = net.explicit(inputs) + if exp is not None: + golden.assert_close(exp, out, atol=ATOL, rtol=RTOL) + net.check_jit(inputs) + assert net.grads_finite(inputs) + return out + + +def check_golden(backend, network, case, explicit=False): + """Check a golden case, via the forward pass or the explicit path.""" + net = backend.build(network, case["config"], case["inputs"], case["params"]) + out = net.explicit(case["inputs"]) if explicit else net.forward(case["inputs"]) + if out is None: + pytest.skip(f"no explicit path for {backend.name} '{network}'") + golden.assert_close(out, case["outputs"], atol=ATOL, rtol=RTOL) + + +def assert_close(actual, expected, **kw): + kw.setdefault("atol", ATOL) + kw.setdefault("rtol", RTOL) + np.testing.assert_allclose(actual, expected, **kw) + + +# --------------------------------------------------------------------------- +# JAX +# --------------------------------------------------------------------------- +def _jax_carry(c): + import jax.numpy as jnp + + return { + "rens": [jnp.asarray(r) for r in c["rens"]], + "dyn_in": None if c["dyn_in"] is None else jnp.asarray(c["dyn_in"]), + "dyn_out": None if c["dyn_out"] is None else jnp.asarray(c["dyn_out"]), + } + + +class JaxNet(Net): + def _call(self, params, inputs, explicit=False): + """Run the model; returns (nested) jax arrays.""" + import jax.numpy as jnp + + n, m = self.network, self.model + if n in STATIC: + x = jnp.asarray(inputs["x"]) + if explicit: + return {"y": m.explicit_call(params, x, m.direct_to_explicit(params))} + return {"y": m.apply(params, x)} + + if n == "ren_composition": + carry, u = _jax_carry(inputs["carry"]), jnp.asarray(inputs["u"]) + if explicit: + new, y = m.explicit_call(params, carry, u, m.direct_to_explicit(params)) + else: + new, y = m.apply(params, carry, u) + return {"carry": new, "y": y} + + x, u = jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"]) + if explicit: + xn, y = m.explicit_call(params, x, u, m.direct_to_explicit(params)) + else: + xn, y = m.apply(params, x, u) + return {"next_state": xn, "y": y} + + @staticmethod + def _numpy(out): + res = {} + for k, v in out.items(): + if k == "carry": + res[k] = { + "rens": np.stack([np.asarray(r) for r in v["rens"]]), + "dyn_in": None if v["dyn_in"] is None else np.asarray(v["dyn_in"]), + "dyn_out": None if v["dyn_out"] is None else np.asarray(v["dyn_out"]), + } + else: + res[k] = np.asarray(v) + return res + + def forward(self, inputs): + return self._numpy(self._call(self.params, inputs)) + + def explicit(self, inputs): + return self._numpy(self._call(self.params, inputs, explicit=True)) + + def check_jit(self, inputs): + import jax + + jitted = jax.jit(lambda p: self._call(p, inputs))(self.params) + golden.assert_close(self._numpy(jitted), self.forward(inputs), atol=ATOL, rtol=RTOL) + + def grads_finite(self, inputs): + import jax + import jax.numpy as jnp + from jax.flatten_util import ravel_pytree + + def loss(p): + out = self._call(p, inputs) + return sum(jnp.sum(out[k] ** 2) for k in ("y", "next_state") if k in out) + + flat = ravel_pytree(jax.grad(loss)(self.params))[0] + return bool(jnp.isfinite(flat).all()) + + def bounds(self): + return tuple(float(b) for b in self.model.get_bounds(self.params)[:2]) + + def dyn_in_output(self, inputs): + e = self.model.direct_to_explicit_inverse(self.params).dyn_in + carry, u = inputs["carry"]["dyn_in"], inputs["u"] + return np.asarray(carry @ np.asarray(e.C).T + u @ np.asarray(e.D).T + np.asarray(e.by)) + + def orthogonal_matrix(self): + import jax.numpy as jnp + + e = self.model.direct_to_explicit(self.params) + if self.network == "unitary": + return np.asarray(e.R) + return np.asarray(jnp.block([[e.A, e.B], [e.C, e.D]])) # dyn_unitary + + def invert(self, inputs, outputs, noncausal=False): + import jax.numpy as jnp + import flax.linen as nn + + n, m, p, c = self.network, self.model, self.params, self.config + y = jnp.asarray(outputs["y"]) + if n == "unitary": + e = m.direct_to_explicit(p) + return {"x": np.asarray(m.inverse_call(p, y, e))} + if n == "monlipnet": + e = m.direct_to_explicit_inverse(p, alpha=0.1, iterations=500) + return {"x": np.asarray(m.inverse_call(p, y, e))} + if n == "bilipnet": + d = c["depth"] + e = m.direct_to_explicit_inverse(p, [0.1] * d, [nn.relu] * d, [500] * d, [1.0] * d) + return {"x": np.asarray(m.inverse_call(p, y, e))} + if n == "dyn_unitary": + x_rec, u_rec = m.inverse_call( + p, jnp.asarray(outputs["next_state"]), y, m.direct_to_explicit(p)) + return {"state": np.asarray(x_rec), "u": np.asarray(u_rec)} + if n == "bilipschitz_ren": + e = m.direct_to_explicit_inverse(p) + _, u = m.inverse_call(p, jnp.asarray(inputs["state"]), y, e) + return {"u": np.asarray(u)} + if n == "ren_composition": + e = m.direct_to_explicit_inverse(p) + carry = _jax_carry(inputs["carry"]) + if noncausal: + new_carry = _jax_carry(outputs["carry"]) + _, u = m.inverse_call_noncausal(p, carry, new_carry, y, e) + else: + _, u = m.inverse_call(p, carry, y, e) + return {"u": np.asarray(u)} + raise NotImplementedError(n) + + +_ACT = None + + +def _jax_act(): + global _ACT + if _ACT is None: + import flax.linen as nn + + _ACT = {"relu": nn.relu, "tanh": nn.tanh, "sigmoid": nn.sigmoid, + "identity": lambda x: x} + return _ACT + + +def _tup(v): + return tuple(v) if isinstance(v, (list, tuple)) else v + + +def build_jax_model(network, config): + """Construct the JAX module for a golden-file `config` (JAX argument names).""" + import jax.numpy as jnp + from robustnn import linear_ren, r2dn_jax, ren_jax + from robustnn.bilipnet_jax import BiLipNet + from robustnn.dyn_orthogonal_jax import DynUnitary + from robustnn.lbdn_jax import LBDN + from robustnn.monlipnet_jax import MonLipNet + from robustnn.orthogonal_jax import Unitary + from robustnn.plnet_jax import PLNet + from robustnn.ren_composition_jax import CompositionREN + + cfg = dict(config) + for k in ("activation", "act_fn"): + if k in cfg: + cfg[k] = _jax_act()[cfg[k]] + for k in ("units", "hidden", "hidden_sizes"): + if k in cfg: + cfg[k] = _tup(cfg[k]) + for k in ("Q", "S", "R", "optimal_point"): + if cfg.get(k) is not None: + cfg[k] = jnp.asarray(cfg[k], dtype=jnp.float32) + + if network == "plnet": + bilip_cfg = cfg.pop("BiLipBlock") + return PLNet(BiLipBlock=build_jax_model("bilipnet", bilip_cfg), **cfg) + + cls = { + "unitary": Unitary, + "monlipnet": MonLipNet, + "bilipnet": BiLipNet, + "lbdn": LBDN, + "dyn_unitary": DynUnitary, + "contracting_ren": ren_jax.ContractingREN, + "lipschitz_ren": ren_jax.LipschitzREN, + "general_ren": ren_jax.GeneralREN, + "bilipschitz_ren": ren_jax.BiLipschitzREN, + "contracting_linear_ren": linear_ren.ContractingLinREN, + "lipschitz_linear_ren": linear_ren.LipschitzLinREN, + "general_linear_ren": linear_ren.GeneralLinREN, + "r2dn": r2dn_jax.ContractingR2DN, + "ren_composition": CompositionREN, + }[network] + return cls(**cfg) + + +class JaxBackend(Backend): + name = "jax" + + def supports(self, network): + return True + + def init_params(self, model, network, inputs, seed=0): + """Flax `init` with the right call signature for `network`.""" + import jax + import jax.numpy as jnp + + if network in STATIC: + args = (jnp.asarray(inputs["x"]),) + elif network == "ren_composition": + args = (_jax_carry(inputs["carry"]), jnp.asarray(inputs["u"])) + else: + args = (jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"])) + return model.init(jax.random.key(seed), *args) + + def _build(self, network, config, inputs, params): + import jax + import jax.numpy as jnp + + model = build_jax_model(network, config) + if params is None: + params = self.init_params(model, network, inputs) + else: + params = {"params": jax.tree.map(jnp.asarray, params)} + return JaxNet(network, config, model, params) + + +# --------------------------------------------------------------------------- +# PyTorch +# --------------------------------------------------------------------------- +def _t(a): + import torch + + return None if a is None else torch.tensor(np.asarray(a), dtype=torch.float32) + + +def _n(t): + return None if t is None else t.detach().numpy() + + +def _torch_carry(c): + return {"rens": [_t(r) for r in c["rens"]], "dyn_in": _t(c["dyn_in"]), + "dyn_out": _t(c["dyn_out"])} + + +class TorchNet(Net): + def _forward_raw(self, inputs): + """Forward pass returning torch tensors (keeps the autograd graph).""" + n, m = self.network, self.model + if n in STATIC: + return {"y": m(_t(inputs["x"]))} + if n == "ren_composition": + new, y = m(_torch_carry(inputs["carry"]), _t(inputs["u"])) + return {"carry": new, "y": y} + xn, y = m(_t(inputs["state"]), _t(inputs["u"])) + return {"next_state": xn, "y": y} + + def forward(self, inputs): + import torch + + with torch.no_grad(): + out = self._forward_raw(inputs) + res = {} + for k, v in out.items(): + if k == "carry": + res[k] = {"rens": np.stack([_n(r) for r in v["rens"]]), + "dyn_in": _n(v["dyn_in"]), "dyn_out": _n(v["dyn_out"])} + else: + res[k] = _n(v) + return res + + def explicit(self, inputs): + import torch + + n, m = self.network, self.model + with torch.no_grad(): + if n in ("unitary", "monlipnet", "bilipnet", "plnet"): + y = m.explicit_call(np.asarray(inputs["x"]), m.direct_to_explicit()) + return {"y": np.asarray(y)} + if n == "bilipschitz_ren": + xn, y = m.explicit_call( + _t(inputs["state"]), _t(inputs["u"]), m.direct_to_explicit()) + return {"next_state": _n(xn), "y": _n(y)} + return None # dyn_unitary, ren_composition: no explicit path in torch + + def grads_finite(self, inputs): + import torch + + self.model.zero_grad() + out = self._forward_raw(inputs) + sum(out[k].pow(2).sum() for k in ("y", "next_state") if k in out).backward() + grads = [p.grad for p in self.model.parameters() if p.grad is not None] + return bool(grads) and all(bool(torch.isfinite(g).all()) for g in grads) + + def bounds(self): + return tuple(float(b) for b in self.model.get_bounds()[:2]) + + def dyn_in_output(self, inputs): + import torch + + with torch.no_grad(): + _, out = self.model.dyn_in(_t(inputs["carry"]["dyn_in"]), _t(inputs["u"])) + return _n(out) + + def orthogonal_matrix(self): + import torch + + if self.network == "unitary": + return np.asarray(self.model.direct_to_explicit().Q) + A, B, C, D = self.model._blocks() # dyn_unitary + return torch.cat([torch.cat([A, B], 1), torch.cat([C, D], 1)], 0).detach().numpy() + + def invert(self, inputs, outputs, noncausal=False): + import torch + + n, m, c = self.network, self.model, self.config + with torch.no_grad(): + y = _t(outputs["y"]) + if n == "unitary": + return {"x": np.asarray(m.inverse(y))} + if n == "monlipnet": + return {"x": np.asarray(m.inverse(outputs["y"], alpha=0.1, iterations=500))} + if n == "bilipnet": + d = c["depth"] + x = m.inverse(outputs["y"], alphas=[0.1] * d, + inverse_activation_fns=[lambda v: np.maximum(0, v)] * d, + iterations=[500] * d, Lambdas=[1.0] * d) + return {"x": np.asarray(x)} + if n == "dyn_unitary": + x_rec, u_rec = m.inverse(_t(outputs["next_state"]), y) + return {"state": _n(x_rec), "u": _n(u_rec)} + if n == "bilipschitz_ren": + _, u = m.inverse(_t(inputs["state"]), y) + return {"u": _n(u)} + if n == "ren_composition": + carry = _torch_carry(inputs["carry"]) + if noncausal: + _, u = m.inverse_noncausal(carry, _torch_carry(outputs["carry"]), y) + else: + _, u = m.inverse(carry, y) + return {"u": _n(u)} + raise NotImplementedError(n) + + +_MODULE_ACT = None +_FN_ACT = None + + +def _torch_acts(): + global _MODULE_ACT, _FN_ACT + if _MODULE_ACT is None: + import torch + import torch.nn as nn + + _MODULE_ACT = {"relu": nn.ReLU, "tanh": nn.Tanh, "sigmoid": nn.Sigmoid, + "identity": nn.Identity} + _FN_ACT = {"relu": torch.relu, "tanh": torch.tanh, "sigmoid": torch.sigmoid, + "identity": lambda x: x} + return _MODULE_ACT, _FN_ACT + + +def build_torch_model(network, config): + """Construct the torch module for a golden-file `config` (JAX argument names).""" + import torch + from robustnn.bilipnet_torch import BiLipNet + from robustnn.dyn_orthogonal_torch import DynUnitary + from robustnn.monlipnet_torch import MonLipNet + from robustnn.orthogonal_torch import Unitary + from robustnn.plnet_torch import PLNet + from robustnn.ren_composition_torch import CompositionREN + from robustnn.ren_torch import BiLipschitzREN + + module_act, fn_act = _torch_acts() + c = config + if network == "unitary": + return Unitary(c["input_size"], c["input_size"], bias=c.get("use_bias", True)) + if network in ("monlipnet", "bilipnet"): + kw = dict(features=c["input_size"], unit_features=list(c["units"]), + mu=c.get("mu"), nu=c.get("nu"), tau=c.get("tau"), + is_mu_fixed=c.get("is_mu_fixed", False), + is_nu_fixed=c.get("is_nu_fixed", False), + is_tau_fixed=c.get("is_tau_fixed", False)) + if "act_fn" in c: + kw["act"] = module_act[c["act_fn"]]() + if network == "bilipnet": + return BiLipNet(depth=c.get("depth", 2), **kw) + return MonLipNet(**kw) + if network == "plnet": + op = c.get("optimal_point") + return PLNet( + BiLipBlock=build_torch_model("bilipnet", c["BiLipBlock"]), + add_constant=c.get("add_constant", False), + optimal_point=None if op is None else torch.tensor(op, dtype=torch.float32), + ) + if network == "dyn_unitary": + return DynUnitary(c["input_size"], c["state_size"], bias=c.get("use_bias", True)) + if network == "bilipschitz_ren": + act = module_act[c.get("activation", "relu")]() + return BiLipschitzREN(c["input_size"], c["state_size"], c["features"], + mu=c["mu"], nu=c["nu"], activation=act) + if network == "ren_composition": + return CompositionREN( + c["input_size"], c["state_size"], c["features"], c["num_layers"], + mu=c["mu"], nu=c["nu"], activation=fn_act[c.get("activation", "relu")], + use_bias=c.get("use_bias", True), + dyn_orth_at_input=c.get("dyn_orth_at_input", False), + dyn_orth_at_output=c.get("dyn_orth_at_output", False), + dyn_state_multiplier=c.get("dyn_state_multiplier", 50), + ) + raise KeyError(f"no PyTorch implementation of '{network}'") + + +# --- JAX/Flax parameter layout -> torch state_dict --------------------------- +def _unitary_sd(p, prefix=""): + sd = {f"{prefix}weight": p["W"], f"{prefix}alpha": p["a"]} + if "b" in p: + sd[f"{prefix}bias"] = p["b"] + return sd + + +def _monlip_sd(p, prefix=""): + sd = {f"{prefix}Fq": p["Fq"].T, f"{prefix}fq": p["fq"], f"{prefix}by": p["by"]} + for k in range(sum(1 for name in p if name.startswith("Fab"))): + sd[f"{prefix}Fr.{k}"] = p[f"Fab{k}"].T + sd[f"{prefix}fr.{k}"] = p[f"fab{k}"] + sd[f"{prefix}bs.{k}"] = p[f"b{k}"] + return sd + + +def _bilip_sd(p, prefix=""): + sd = {} + for name, sub in p.items(): + kind, k = name.split("_") + if kind == "uni": + sd.update(_unitary_sd(sub, f"{prefix}orth_layers.{k}.")) + else: + sd.update(_monlip_sd(sub, f"{prefix}mon_layers.{k}.")) + return sd + + +def _flat_sd(p, prefix=""): + return {f"{prefix}{k}": v for k, v in p.items()} + + +def _ren_sd(p, prefix=""): + # The JAX BiLipschitzREN carries a `D22` direct parameter that its QSR + # parameterisation never reads (D22 is derived from Q, S, R); torch has none. + return _flat_sd({k: v for k, v in p.items() if k != "D22"}, prefix) + + +def to_state_dict(network, params): + """Translate golden (JAX-layout) `params` to a torch `state_dict`.""" + if network == "unitary": + return _unitary_sd(params) + if network == "monlipnet": + return _monlip_sd(params) + if network == "bilipnet": + return _bilip_sd(params) + if network == "plnet": + sd = _bilip_sd(params["BiLipBlock"], "bln.") + if "c" in params: + sd["bias"] = params["c"] + return sd + if network == "dyn_unitary": + return _flat_sd(params) + if network == "bilipschitz_ren": + return _ren_sd(params) + if network == "ren_composition": + sd = {} + for name, sub in params.items(): + if name in ("input_orth", "output_orth"): + sd.update(_unitary_sd(sub, f"{name}.")) + elif name.startswith("orths_"): + sd.update(_unitary_sd(sub, f"orths.{name.split('_')[1]}.")) + elif name.startswith("rens_"): + sd.update(_ren_sd(sub, f"rens.{name.split('_')[1]}.")) + else: # dyn_in / dyn_out + sd.update(_flat_sd(sub, f"{name}.")) + return sd + raise KeyError(network) + + +# Bounds are constructor arguments (buffers/parameters named mu, nu, tau): they +# come from the config, not from the golden parameters. +_CONFIG_KEYS = ("mu", "nu", "tau") + + +def load_torch_params(network, model, params): + sd = {k: _t(v) for k, v in to_state_dict(network, params).items()} + missing, unexpected = model.load_state_dict(sd, strict=False) + missing = [k for k in missing if k.split(".")[-1] not in _CONFIG_KEYS] + assert not unexpected, f"unexpected keys: {unexpected}" + assert not missing, f"keys not set from golden params: {missing}" + + +class TorchBackend(Backend): + name = "torch" + + def supports(self, network): + return network in TORCH_NETWORKS + + def _build(self, network, config, inputs, params): + model = build_torch_model(network, config) + if params is not None: + load_torch_params(network, model, params) + return TorchNet(network, config, model) + + +_BACKENDS = {"jax": JaxBackend, "torch": TorchBackend} + + +def get_backend(name): + """Backend instance, skipping the test if its libraries are not installed.""" + if name == "jax": + pytest.importorskip("jax") + pytest.importorskip("flax") + else: + pytest.importorskip("torch") + return _BACKENDS[name]() diff --git a/tests/conftest.py b/tests/conftest.py index a6fec62..8fba7ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,50 @@ # This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE -"""Shared pytest fixtures. Backend-specific fixtures live in the -`jax_tests/` and `torch_tests/` conftest files.""" +"""Shared pytest fixtures. + +Most tests take the `backend` fixture and run once per backend (JAX and PyTorch). +Tests that only make sense for one backend take `jax_backend` / `torch_backend` +instead. Both kinds are marked `jax` / `torch` so `-m jax` and `-m torch` work. +""" + +import importlib.util import numpy as np import pytest +from backends import get_backend + @pytest.fixture(autouse=True) -def _seed_numpy(): +def _seed(): np.random.seed(0) + if importlib.util.find_spec("torch") is not None: + import torch + + torch.manual_seed(0) + + +@pytest.fixture( + params=[pytest.param("jax", marks=pytest.mark.jax), + pytest.param("torch", marks=pytest.mark.torch)] +) +def backend(request): + return get_backend(request.param) + + +@pytest.fixture +def jax_backend(): + return get_backend("jax") + + +@pytest.fixture +def torch_backend(): + return get_backend("torch") + + +def pytest_collection_modifyitems(items): + for item in items: + if "jax_backend" in item.fixturenames: + item.add_marker(pytest.mark.jax) + if "torch_backend" in item.fixturenames: + item.add_marker(pytest.mark.torch) diff --git a/tests/generate_golden.py b/tests/generate_golden.py index a37a276..9e2e9f9 100644 --- a/tests/generate_golden.py +++ b/tests/generate_golden.py @@ -24,7 +24,7 @@ import numpy as np import golden -from jax_utils import build_model, run_model +from backends import JaxNet, build_jax_model BATCH = 4 @@ -59,11 +59,11 @@ def _randn(rng, *shape): def _make_case(network, name, config, rng, inputs, init_args): - model = build_model(network, config) + model = build_jax_model(network, config) variables = model.init(jax.random.key(0), *init_args(inputs)) params = perturb(jax.tree.map(np.asarray, variables["params"]), rng) jparams = {"params": jax.tree.map(jnp.asarray, params)} - outputs = run_model(network, model, jparams, inputs) + outputs = JaxNet(network, config, model, jparams).forward(inputs) return { "name": name, "config": config, diff --git a/tests/golden.py b/tests/golden.py index 1140b6f..4d90834 100644 --- a/tests/golden.py +++ b/tests/golden.py @@ -13,7 +13,7 @@ } The JAX tests load `params` as-is; the PyTorch tests translate them to -a `state_dict` (see `torch_utils.py`). Because both backends are checked against +a `state_dict` (see `backends.py`). Because both backends are checked against the same file, passing both implies they agree with each other. Regenerate the files with `uv run python tests/generate_golden.py`. diff --git a/tests/jax_tests/conftest.py b/tests/jax_tests/conftest.py deleted file mode 100644 index daad96a..0000000 --- a/tests/jax_tests/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -import pytest - -pytest.importorskip("jax") -pytest.importorskip("flax") - -import jax - - -def pytest_collection_modifyitems(items): - for item in items: - if "jax_tests" in str(item.fspath): - item.add_marker(pytest.mark.jax) - - -@pytest.fixture -def key(): - return jax.random.key(0) diff --git a/tests/jax_tests/test_bilipnet.py b/tests/jax_tests/test_bilipnet.py deleted file mode 100644 index e2b61ea..0000000 --- a/tests/jax_tests/test_bilipnet.py +++ /dev/null @@ -1,76 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `BiLipNet`.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden, empirical_gains -from robustnn.bilipnet_jax import BiLipNet - - -# ----------------------------- BiLipNet ---------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("depth", [1, 2, 3]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_bilipnet_shape_explicit_and_bounds(key, batches, depth, use_bias): - size, mu, nu = 3, 0.5, 4.0 - net = BiLipNet(input_size=size, units=[4, 4], mu=mu, nu=nu, tau=nu / mu, - is_mu_fixed=True, is_nu_fixed=True, depth=depth, - use_bias=use_bias) - x = jax.random.normal(key, (batches, size)) - params = net.init(key, x) - - y = net.apply(params, x) - assert y.shape == (batches, size) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - lo, hi, _ = net.get_bounds(params) - np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-5) - gains = empirical_gains(lambda u: net.apply(params, u), key, 200, size) - assert gains.min() >= mu - 1e-3 - assert gains.max() <= nu + 1e-3 - - -@pytest.mark.parametrize("depth", [1, 2]) -def test_bilipnet_inverse_round_trip(key, depth): - net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True, depth=depth, act_fn=nn.relu) - x = jax.random.normal(key, (6, 3)) - params = net.init(key, x) - y = net.apply(params, x) - inv_act = nn.relu - e_inv = net.direct_to_explicit_inverse( - params, [0.1] * depth, [inv_act] * depth, [500] * depth, [1.0] * depth) - x_rec = net.inverse_call(params, y, e_inv) - np.testing.assert_allclose(x_rec, x, atol=1e-3) - - -def test_bilipnet_jit_and_grad(key): - net = BiLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True) - x = jax.random.normal(key, (5, 3)) - params = net.init(key, x) - np.testing.assert_allclose(jax.jit(net.apply)(params, x), - net.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def test_bilipnet_fixing_all_raises(key): - net = BiLipNet(input_size=2, units=[2], is_mu_fixed=True, - is_nu_fixed=True, is_tau_fixed=True) - with pytest.raises(ValueError): - net.init(key, jnp.ones((1, 2))) - - -@pytest.mark.parametrize("case", golden.cases("bilipnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_bilipnet_matches_golden(case, explicit): - check_golden("bilipnet", case, explicit) diff --git a/tests/jax_tests/test_bilipschitz_ren.py b/tests/jax_tests/test_bilipschitz_ren.py deleted file mode 100644 index 622df4e..0000000 --- a/tests/jax_tests/test_bilipschitz_ren.py +++ /dev/null @@ -1,55 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `BiLipschitzREN`.""" - -import jax -import jax.numpy as jnp -import numpy as np -import pytest - -from robustnn import ren_jax as ren -import golden -from jax_utils import check_golden, check_ren_forward, setup_ren - - -# ----------------------------- BiLipschitz REN ---------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("nio", [1, 3]) -@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) -def test_bilipschitz_ren(key, batches, nio, mu_nu): - mu, nu = mu_nu - nx, nv = 4, 8 - model = ren.BiLipschitzREN(nio, nx, nv, nio, mu=mu, nu=nu, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = setup_ren(model, key, batches, nio) - check_ren_forward(model, params, states, inputs, nx, nio) - - e = model.direct_to_explicit(params) - _, y = model.explicit_call(params, states, inputs, e) - e_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call(params, states, y, e_inv) - np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) - - n = 300 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nio)) - ub = jax.random.normal(kb, (n, nio)) - z = jnp.zeros((n, nx)) - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.min() >= mu - 1e-3 - assert gain.max() <= nu + 1e-3 - - -def test_bilipschitz_ren_requires_square(key): - model = ren.BiLipschitzREN(3, 3, 4, 2) - with pytest.raises(ValueError): - setup_ren(model, key, 2, 3) - - -@pytest.mark.parametrize("case", golden.cases("bilipschitz_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_bilipschitz_ren_matches_golden(case, explicit): - check_golden("bilipschitz_ren", case, explicit) diff --git a/tests/jax_tests/test_contracting_ren.py b/tests/jax_tests/test_contracting_ren.py deleted file mode 100644 index b0216b2..0000000 --- a/tests/jax_tests/test_contracting_ren.py +++ /dev/null @@ -1,91 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `ContractingREN`.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest - -from helpers import compute_p_contractingren -from robustnn import ren_jax as ren -import golden -from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, setup_ren - - -# --------------------------- Contracting REN ----------------------------- -@pytest.mark.parametrize("batches", [1, 4]) -@pytest.mark.parametrize("activation", ACTIVATIONS) -@pytest.mark.parametrize("init_method", ["random", "long_memory"]) -@pytest.mark.parametrize("do_polar_param", [True, False]) -@pytest.mark.parametrize("sizes", [(1, 1, 1, 1), (5, 3, 4, 2)]) -def test_contracting_ren(key, batches, activation, init_method, do_polar_param, sizes): - nu, nx, nv, ny = sizes - model = ren.ContractingREN(nu, nx, nv, ny, activation=activation, - init_method=init_method, do_polar_param=do_polar_param) - params, states, inputs = setup_ren(model, key, batches, nu) - check_ren_forward(model, params, states, inputs, nx, ny) - - -def test_contracting_ren_contraction(key): - model = ren.ContractingREN(5, 3, 4, 2, activation=nn.tanh) - k1, k2 = jax.random.split(key) - params, x1, u = setup_ren(model, key, 8, 5) - x0 = 10 * jax.random.normal(k1, x1.shape) - xn0, _ = model.apply(params, x0, u) - xn1, _ = model.apply(params, x1, u) - P = compute_p_contractingren(model, params) - - def norm2(x): - return jnp.sum((x @ P.T) * x, axis=-1) - assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) - - -def test_ren_identity_output_and_zero_output(key): - model = ren.ContractingREN(3, 3, 4, 3, identity_output=True) - params, states, inputs = setup_ren(model, key, 4, 3) - _, y = model.apply(params, states, inputs) - # identity output: y_t = x_t, i.e. the state before the update - np.testing.assert_allclose(y, states, atol=1e-5) - - zero = ren.ContractingREN(3, 3, 4, 2, init_output_zero=True) - params, states, inputs = setup_ren(zero, key, 4, 3) - _, y = zero.apply(params, states, inputs) - np.testing.assert_allclose(y, 0.0, atol=1e-6) - - -def test_ren_identity_output_size_mismatch(key): - model = ren.ContractingREN(3, 3, 4, 2, identity_output=True) - with pytest.raises(ValueError): - setup_ren(model, key, 2, 3) - - -def test_ren_bad_init_method(key): - model = ren.ContractingREN(3, 3, 4, 2, init_method="nope") - with pytest.raises(ValueError): - setup_ren(model, key, 2, 3) - -# ------------------------------ simulate_sequence ------------------------- -@pytest.mark.parametrize("horizon", [1, 5]) -def test_simulate_sequence_matches_loop(key, horizon): - nu, nx, nv, ny = 3, 4, 5, 2 - model = ren.ContractingREN(nu, nx, nv, ny, activation=nn.tanh) - params, x0, u0 = setup_ren(model, key, 4, nu) - seq = jax.random.normal(key, (horizon, 4, nu)) - - xT, ys = model.simulate_sequence(params, x0, seq) - assert ys.shape == (horizon, 4, ny) - - x, outs = x0, [] - for t in range(horizon): - x, y = model.apply(params, x, seq[t]) - outs.append(y) - np.testing.assert_allclose(xT, x, atol=1e-4) - np.testing.assert_allclose(ys, jnp.stack(outs), atol=1e-4) - - -@pytest.mark.parametrize("case", golden.cases("contracting_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_contracting_ren_matches_golden(case, explicit): - check_golden("contracting_ren", case, explicit) diff --git a/tests/jax_tests/test_dyn_orthogonal.py b/tests/jax_tests/test_dyn_orthogonal.py deleted file mode 100644 index 112c562..0000000 --- a/tests/jax_tests/test_dyn_orthogonal.py +++ /dev/null @@ -1,50 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `DynUnitary`.""" - -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden -from robustnn.dyn_orthogonal_jax import DynUnitary - - -# ------------------------------ DynUnitary -------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_dyn_unitary(key, batches, io, nx, use_bias): - layer = DynUnitary(io, nx, use_bias=use_bias) - x = jax.random.normal(key, (batches, nx)) - u = jax.random.normal(jax.random.key(1), (batches, io)) - params = layer.init(key, x, u) - - assert layer.initialize_carry(key, (batches, io)).shape == (batches, nx) - e = layer.direct_to_explicit(params) - x1, y = layer.explicit_call(params, x, u, e) - assert x1.shape == (batches, nx) and y.shape == (batches, io) - x1b, yb = layer.apply(params, x, u) - np.testing.assert_allclose(x1, x1b, atol=1e-5) - np.testing.assert_allclose(y, yb, atol=1e-5) - - # Stacked state-space matrix is orthogonal. - G = jnp.block([[e.A, e.B], [e.C, e.D]]) - np.testing.assert_allclose(G @ G.T, jnp.eye(nx + io), atol=1e-5) - - # Non-causal inverse recovers the previous state and the input. - x_rec, u_rec = layer.inverse_call(params, x1, y, e) - np.testing.assert_allclose(x_rec, x, atol=1e-4) - np.testing.assert_allclose(u_rec, u, atol=1e-4) - - g = jax.grad(lambda p: sum(jnp.sum(a ** 2) for a in layer.apply(p, x, u)))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -@pytest.mark.parametrize("case", golden.cases("dyn_unitary"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_dyn_unitary_matches_golden(case, explicit): - check_golden("dyn_unitary", case, explicit) diff --git a/tests/jax_tests/test_general_ren.py b/tests/jax_tests/test_general_ren.py deleted file mode 100644 index 3d3cc05..0000000 --- a/tests/jax_tests/test_general_ren.py +++ /dev/null @@ -1,45 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX QSR-constrained `GeneralREN`.""" - -import jax.numpy as jnp -import pytest - -from robustnn import ren_jax as ren -import golden -from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, make_qsr, setup_ren - - -# ------------------------------ General REN ------------------------------ -@pytest.mark.parametrize("activation", ACTIVATIONS) -@pytest.mark.parametrize("sizes", [(5, 3, 4, 2), (2, 2, 3, 2)]) -def test_general_ren(key, activation, sizes): - nu, nx, nv, ny = sizes - Q, S, R = make_qsr(key, nu, ny) - model = ren.GeneralREN(nu, nx, nv, ny, Q=Q, S=S, R=R, activation=activation, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = setup_ren(model, key, 4, nu) - check_ren_forward(model, params, states, inputs, nx, ny) - - -def test_general_ren_invalid_qsr(key): - nu, ny = 3, 2 - Q, S, R = make_qsr(key, nu, ny) - bad = [ - dict(Q=jnp.eye(ny + 1), S=S, R=R), # wrong Q size - dict(Q=Q, S=jnp.ones((ny, nu)), R=R), # wrong S size - dict(Q=Q, S=S, R=jnp.eye(nu + 1)), # wrong R size - dict(Q=jnp.eye(ny), S=S, R=R), # Q not neg. definite - dict(Q=Q, S=S, R=-jnp.eye(nu)), # Schur complement not PD - ] - for kw in bad: - model = ren.GeneralREN(nu, 3, 4, ny, **kw) - with pytest.raises(ValueError): - model.check_valid_qsr() - - -@pytest.mark.parametrize("case", golden.cases("general_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_general_ren_matches_golden(case, explicit): - check_golden("general_ren", case, explicit) diff --git a/tests/jax_tests/test_lbdn.py b/tests/jax_tests/test_lbdn.py deleted file mode 100644 index 81eda4c..0000000 --- a/tests/jax_tests/test_lbdn.py +++ /dev/null @@ -1,63 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `LBDN`.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden -from robustnn.lbdn_jax import LBDN - - -# -------------------------------- LBDN ----------------------------------- -@pytest.mark.parametrize("activation", [nn.relu, nn.tanh, nn.sigmoid, lambda x: x]) -@pytest.mark.parametrize("hidden", [(4,), (8, 16)]) -@pytest.mark.parametrize("batches", [1, 5]) -def test_lbdn_shape_explicit_and_lipschitz(key, activation, hidden, batches): - nin, nout, gamma = 5, 2, 3.0 - net = LBDN(nin, hidden, nout, gamma=gamma, activation=activation) - x = jax.random.normal(key, (batches, nin)) - params = net.init(key, x) - - y = net.apply(params, x) - assert y.shape == (batches, nout) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (200, nin)) - ub = jax.random.normal(kb, (200, nin)) - gain = (jnp.linalg.norm(net.apply(params, ua) - net.apply(params, ub), axis=1) - / jnp.linalg.norm(ua - ub, axis=1)) - assert gain.max() <= gamma + 1e-3 - - -@pytest.mark.parametrize("use_bias", [True, False]) -@pytest.mark.parametrize("trainable_lipschitz", [True, False]) -def test_lbdn_options_and_grad(key, use_bias, trainable_lipschitz): - net = LBDN(3, (6,), 2, gamma=2.0, use_bias=use_bias, - trainable_lipschitz=trainable_lipschitz) - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - np.testing.assert_allclose(jax.jit(net.apply)(params, x), - net.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def test_lbdn_init_output_zero(key): - net = LBDN(3, (6,), 2, init_output_zero=True) - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - np.testing.assert_allclose(net.apply(params, x), 0.0, atol=1e-6) - - -@pytest.mark.parametrize("case", golden.cases("lbdn"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_lbdn_matches_golden(case, explicit): - check_golden("lbdn", case, explicit) diff --git a/tests/jax_tests/test_linear_ren.py b/tests/jax_tests/test_linear_ren.py deleted file mode 100644 index c7f5a2b..0000000 --- a/tests/jax_tests/test_linear_ren.py +++ /dev/null @@ -1,69 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX linear RENs (`ContractingLinREN`, `LipschitzLinREN`, `GeneralLinREN`).""" - -import jax -import jax.numpy as jnp -import pytest - -from robustnn import linear_ren -import golden -from jax_utils import check_golden, check_ren_forward, make_qsr, setup_ren - - -# --------------------------------- Linear RENs ---------------------------- -@pytest.mark.parametrize("cls", ["contracting", "lipschitz", "general"]) -@pytest.mark.parametrize("batches", [1, 4]) -def test_linear_ren(key, cls, batches): - nu, nx, ny = 5, 3, 2 - if cls == "contracting": - model = linear_ren.ContractingLinREN(nu, nx, 0, ny) - elif cls == "lipschitz": - model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=2.0) - else: - Q, S, R = make_qsr(key, nu, ny) - model = linear_ren.GeneralLinREN(nu, nx, 0, ny, Q=Q, S=S, R=R, - init_method="long_memory") - model.check_valid_qsr() - params, states, inputs = setup_ren(model, key, batches, nu) - check_ren_forward(model, params, states, inputs, nx, ny) - - -def test_linear_ren_lipschitz_bound(key): - nu, nx, ny, gamma = 4, 3, 2, 2.0 - model = linear_ren.LipschitzLinREN(nu, nx, 0, ny, gamma=gamma) - params, _, _ = setup_ren(model, key, 2, nu) - n = 200 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nu)) - ub = jax.random.normal(kb, (n, nu)) - z = jnp.zeros((n, nx)) - e = model.direct_to_explicit(params) - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.max() <= gamma + 1e-3 - - -def test_linear_ren_requires_zero_features(key): - with pytest.raises(ValueError): - model = linear_ren.ContractingLinREN(3, 3, 4, 2) - setup_ren(model, key, 2, 3) - - -@pytest.mark.parametrize("case", golden.cases("contracting_linear_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_contracting_linear_ren_matches_golden(case, explicit): - check_golden("contracting_linear_ren", case, explicit) - - -@pytest.mark.parametrize("case", golden.cases("lipschitz_linear_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_lipschitz_linear_ren_matches_golden(case, explicit): - check_golden("lipschitz_linear_ren", case, explicit) - - -@pytest.mark.parametrize("case", golden.cases("general_linear_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_general_linear_ren_matches_golden(case, explicit): - check_golden("general_linear_ren", case, explicit) diff --git a/tests/jax_tests/test_lipschitz_ren.py b/tests/jax_tests/test_lipschitz_ren.py deleted file mode 100644 index e51a31a..0000000 --- a/tests/jax_tests/test_lipschitz_ren.py +++ /dev/null @@ -1,45 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `LipschitzREN`.""" - -import jax -import jax.numpy as jnp -import pytest - -from robustnn import ren_jax as ren -import golden -from jax_utils import ACTIVATIONS, check_golden, check_ren_forward, setup_ren - - -# ----------------------------- Lipschitz REN ----------------------------- -@pytest.mark.parametrize("gamma", [0.5, 1.0, 5.0]) -@pytest.mark.parametrize("activation", ACTIVATIONS) -def test_lipschitz_ren(key, gamma, activation): - nu, nx, nv, ny = 4, 3, 5, 2 - model = ren.LipschitzREN(nu, nx, nv, ny, gamma=gamma, activation=activation) - params, states, inputs = setup_ren(model, key, 4, nu) - check_ren_forward(model, params, states, inputs, nx, ny) - - # Same initial state => one-step output gain is bounded by gamma. - n = 200 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, nu)) - ub = jax.random.normal(kb, (n, nu)) - x = jnp.zeros((n, nx)) - e = model.direct_to_explicit(params) - _, ya = model.explicit_call(params, x, ua, e) - _, yb = model.explicit_call(params, x, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.max() <= gamma + 1e-3 - - -def test_lipschitz_ren_identity_output_unsupported(key): - with pytest.raises(NotImplementedError): - model = ren.LipschitzREN(3, 3, 4, 3, identity_output=True) - setup_ren(model, key, 2, 3) - - -@pytest.mark.parametrize("case", golden.cases("lipschitz_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_lipschitz_ren_matches_golden(case, explicit): - check_golden("lipschitz_ren", case, explicit) diff --git a/tests/jax_tests/test_monlipnet.py b/tests/jax_tests/test_monlipnet.py deleted file mode 100644 index 6dfd905..0000000 --- a/tests/jax_tests/test_monlipnet.py +++ /dev/null @@ -1,73 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `MonLipNet`.""" - -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden, empirical_gains -from robustnn.monlipnet_jax import MonLipNet - - -# ----------------------------- MonLipNet --------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) -@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) -def test_monlipnet_shape_explicit_and_bounds(key, batches, units, fixed): - size, mu, nu, tau = 3, 0.5, 4.0, 4.0 - flags = dict(is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, - is_tau_fixed="tau" in fixed) - net = MonLipNet(input_size=size, units=units, mu=mu, nu=nu, tau=tau, **flags) - x = jax.random.normal(key, (batches, size)) - params = net.init(key, x) - - y = net.apply(params, x) - assert y.shape == (batches, size) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - lo, hi, _ = net.get_bounds(params) - gains = empirical_gains(lambda u: net.apply(params, u), key, 200, size) - assert gains.min() >= float(lo) - 1e-3 - assert gains.max() <= float(hi) + 1e-3 - - -def test_monlipnet_fixing_all_raises(key): - net = MonLipNet(input_size=2, units=[2], is_mu_fixed=True, - is_nu_fixed=True, is_tau_fixed=True) - with pytest.raises(ValueError): - net.init(key, jnp.ones((1, 2))) - - -@pytest.mark.parametrize("units", [[3], [4, 4]]) -def test_monlipnet_inverse_round_trip(key, units): - net = MonLipNet(input_size=3, units=units, mu=0.5, nu=4.0, tau=4.0, - is_tau_fixed=True) - x = jax.random.normal(key, (6, 3)) - params = net.init(key, x) - y = net.apply(params, x) - e_inv = net.direct_to_explicit_inverse(params, alpha=0.1, iterations=500) - x_rec = net.inverse_call(params, y, e_inv) - np.testing.assert_allclose(x_rec, x, atol=1e-3) - - -def test_monlipnet_jit_and_grad(key): - net = MonLipNet(input_size=3, units=[4], mu=0.5, nu=4.0, tau=4.0, - is_tau_fixed=True) - x = jax.random.normal(key, (5, 3)) - params = net.init(key, x) - np.testing.assert_allclose(jax.jit(net.apply)(params, x), - net.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x) ** 2))(params) - flat = ravel_pytree(g)[0] - assert bool(jnp.isfinite(flat).all()) and bool(jnp.any(flat != 0)) - - -@pytest.mark.parametrize("case", golden.cases("monlipnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_monlipnet_matches_golden(case, explicit): - check_golden("monlipnet", case, explicit) diff --git a/tests/jax_tests/test_orthogonal.py b/tests/jax_tests/test_orthogonal.py deleted file mode 100644 index 72adf99..0000000 --- a/tests/jax_tests/test_orthogonal.py +++ /dev/null @@ -1,75 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `Unitary` (Cayley) layer.""" - -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden -from robustnn.orthogonal_jax import Unitary - - -@pytest.mark.parametrize("batches", [1, 7]) -@pytest.mark.parametrize("size", [1, 2, 5]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_shape_and_explicit_matches_direct(key, batches, size, use_bias): - layer = Unitary(input_size=size, use_bias=use_bias) - x = jax.random.normal(key, (batches, size)) - params = layer.init(key, x) - - y = layer.apply(params, x) - assert y.shape == (batches, size) - - explicit = layer.direct_to_explicit(params) - y_exp = layer.explicit_call(params, x, explicit) - np.testing.assert_allclose(y, y_exp, atol=1e-5) - - -@pytest.mark.parametrize("size", [2, 5]) -def test_orthogonal_and_norm_preserving(key, size): - layer = Unitary(input_size=size, use_bias=False) - x = jax.random.normal(key, (16, size)) - params = layer.init(key, x) - R = layer.direct_to_explicit(params).R - np.testing.assert_allclose(R @ R.T, jnp.eye(size), atol=1e-5) - - y = layer.apply(params, x) - np.testing.assert_allclose(jnp.linalg.norm(y, axis=1), - jnp.linalg.norm(x, axis=1), rtol=1e-4) - - -@pytest.mark.parametrize("use_bias", [True, False]) -def test_inverse_round_trip(key, use_bias): - layer = Unitary(input_size=4, use_bias=use_bias) - x = jax.random.normal(key, (6, 4)) - params = layer.init(key, x) - # Non-zero bias so that the inverse has to undo it. - if use_bias: - params = jax.tree.map(lambda p: p, params) - params["params"]["b"] = jnp.arange(4, dtype=jnp.float32) - explicit = layer.direct_to_explicit(params) - y = layer.explicit_call(params, x, explicit) - x_rec = layer.inverse_call(params, y, explicit) - np.testing.assert_allclose(x_rec, x, atol=1e-4) - - -def test_jit_and_grad(key): - layer = Unitary(input_size=3) - x = jax.random.normal(key, (5, 3)) - params = layer.init(key, x) - - np.testing.assert_allclose(jax.jit(layer.apply)(params, x), - layer.apply(params, x), atol=1e-5) - g = jax.grad(lambda p: jnp.sum(layer.apply(p, x) ** 2))(params) - flat = ravel_pytree(g)[0] - assert bool(jnp.isfinite(flat).all()) - - -@pytest.mark.parametrize("case", golden.cases("unitary"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_unitary_matches_golden(case, explicit): - check_golden("unitary", case, explicit) diff --git a/tests/jax_tests/test_plnet.py b/tests/jax_tests/test_plnet.py deleted file mode 100644 index a9153d4..0000000 --- a/tests/jax_tests/test_plnet.py +++ /dev/null @@ -1,67 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `PLNet`.""" - -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden -from robustnn.bilipnet_jax import BiLipNet -from robustnn.plnet_jax import PLNet - - -def _plnet(size=3, **kw): - bilip = BiLipNet(input_size=size, units=[4], mu=0.5, nu=4.0, tau=8.0, - is_mu_fixed=True, is_nu_fixed=True, depth=2) - return PLNet(BiLipBlock=bilip, **kw) - - -# ------------------------------- PLNet ----------------------------------- -@pytest.mark.parametrize("batches", [1, 6]) -def test_plnet_shape_and_explicit(key, batches): - net = _plnet() - x = jax.random.normal(key, (batches, 3)) - params = net.init(key, x) - y = net.apply(params, x) - assert y.shape == (batches,) - assert bool(jnp.all(y >= 0)) - - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), y, atol=1e-5) - - -def test_plnet_zero_at_optimal_point(key): - net = _plnet() - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - explicit = net.direct_to_explicit(params, x_optimal=x[:1]) - y_opt = net.explicit_call(params, x[:1], explicit) - np.testing.assert_allclose(y_opt, 0.0, atol=1e-6) - assert bool(jnp.all(net.explicit_call(params, x, explicit) >= 0)) - - -def test_plnet_add_constant(key): - net = _plnet(add_constant=True, c=1.5) - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - explicit = net.direct_to_explicit(params) - np.testing.assert_allclose(net.explicit_call(params, x, explicit), - net.apply(params, x), atol=1e-5) - - -def test_plnet_grad(key): - net = _plnet() - x = jax.random.normal(key, (4, 3)) - params = net.init(key, x) - g = jax.grad(lambda p: jnp.sum(net.apply(p, x)))(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -@pytest.mark.parametrize("case", golden.cases("plnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_plnet_matches_golden(case, explicit): - check_golden("plnet", case, explicit) diff --git a/tests/jax_tests/test_r2dn.py b/tests/jax_tests/test_r2dn.py deleted file mode 100644 index ef7c47b..0000000 --- a/tests/jax_tests/test_r2dn.py +++ /dev/null @@ -1,48 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `ContractingR2DN`.""" - -import jax -import jax.numpy as jnp -import pytest - -from helpers import compute_p_contractingr2dn -from robustnn import r2dn_jax as r2dn -import golden -from jax_utils import check_golden, check_ren_forward, setup_ren - - -# ----------------------------------- R2DN --------------------------------- -@pytest.mark.parametrize("batches", [1, 4]) -@pytest.mark.parametrize("hidden", [(2,), (2, 2), (4, 3, 2)]) -@pytest.mark.parametrize("init_method", ["random", "long_memory"]) -def test_contracting_r2dn(key, batches, hidden, init_method): - nu, nx, nv, ny = 5, 3, 4, 2 - model = r2dn.ContractingR2DN(nu, nx, nv, ny, hidden, init_method=init_method) - params, states, inputs = setup_ren(model, key, batches, nu) - check_ren_forward(model, params, states, inputs, nx, ny) - - -def test_r2dn_simulate_and_contraction(key): - nu, nx, nv, ny, horizon = 5, 3, 4, 2, 3 - model = r2dn.ContractingR2DN(nu, nx, nv, ny, (2, 2), init_method="long_memory") - params, x1, u = setup_ren(model, key, 4, nu) - - seq = jax.random.normal(key, (horizon, 4, nu)) - xT, ys = jax.jit(model.simulate_sequence)(params, x1, seq) - assert ys.shape == (horizon, 4, ny) and xT.shape == x1.shape - - x0 = 10 * jax.random.normal(key, x1.shape) - xn0, _ = model.apply(params, x0, u) - xn1, _ = model.apply(params, x1, u) - P = compute_p_contractingr2dn(model, params) - - def norm2(x): - return jnp.sum((x @ P.T) * x, axis=-1) - assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) - - -@pytest.mark.parametrize("case", golden.cases("r2dn"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_r2dn_matches_golden(case, explicit): - check_golden("r2dn", case, explicit) diff --git a/tests/jax_tests/test_ren_composition.py b/tests/jax_tests/test_ren_composition.py deleted file mode 100644 index 0e1a864..0000000 --- a/tests/jax_tests/test_ren_composition.py +++ /dev/null @@ -1,131 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the JAX `CompositionREN`.""" - -import flax.linen as nn -import jax -import jax.numpy as jnp -import numpy as np -import pytest -from jax.flatten_util import ravel_pytree - -import golden -from jax_utils import check_golden -from robustnn.ren_composition_jax import CompositionREN - - -# ----------------------------- CompositionREN ----------------------------- -def _model(io=2, nx=3, nv=6, L=2, mu=0.5, nu=5.0, **kw): - return CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, - init_method="long_memory", **kw) - - -def _setup(model, key, batches, io): - k1, k2 = jax.random.split(key) - carry = model.initialize_carry(k1, (batches, io)) - inputs = jax.random.normal(k2, (batches, io)) - params = model.init(key, carry, inputs) - return params, carry, inputs - - -def _zero_carry(model, n, nx, L): - c = model.initialize_carry(jax.random.key(0), (n, model.input_size)) - return jax.tree.map(jnp.zeros_like, c) - - -@pytest.mark.parametrize("batches", [1, 6]) -@pytest.mark.parametrize("L", [1, 3]) -@pytest.mark.parametrize("use_bias", [True, False]) -@pytest.mark.parametrize("activation", [nn.relu, nn.tanh]) -def test_composition_ren(key, batches, L, use_bias, activation): - io, nx, mu, nu = 2, 3, 0.5, 5.0 - model = _model(io=io, nx=nx, L=L, mu=mu, nu=nu, use_bias=use_bias, - activation=activation) - params, carry, inputs = _setup(model, key, batches, io) - - new_carry, y = model.apply(params, carry, inputs) - assert len(new_carry["rens"]) == L - assert y.shape == (batches, io) - assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None - - e = model.direct_to_explicit(params) - _, y_exp = model.explicit_call(params, carry, inputs, e) - np.testing.assert_allclose(y, y_exp, atol=1e-5) - - if activation is nn.relu: - # The inverse solver is only accurate to ~1e-3 for smooth activations - # at the default iteration count, so only check it for relu. - e_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call(params, carry, y, e_inv) - np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) - - n = 300 - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, io)) - ub = jax.random.normal(kb, (n, io)) - z = {"rens": [jnp.zeros((n, nx)) for _ in range(L)], - "dyn_in": None, "dyn_out": None} - _, ya = model.explicit_call(params, z, ua, e) - _, yb = model.explicit_call(params, z, ub, e) - gain = jnp.linalg.norm(ya - yb, axis=1) / jnp.linalg.norm(ua - ub, axis=1) - assert gain.min() >= mu - 1e-3 - assert gain.max() <= nu + 1e-3 - assert model.get_bounds(params) == (mu, nu) - - def loss(p): - return jnp.sum(model.apply(p, carry, inputs)[1] ** 2) - g = jax.grad(loss)(params) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -@pytest.mark.parametrize("at_input,at_output", - [(True, False), (False, True), (True, True)]) -def test_composition_ren_dyn_orth_noncausal_inverse(key, at_input, at_output): - io, nx, L = 2, 3, 2 - model = _model(io=io, nx=nx, nv=6, L=L, dyn_orth_at_input=at_input, - dyn_orth_at_output=at_output, dyn_state_multiplier=4) - params, carry, inputs = _setup(model, key, 5, io) - assert (carry["dyn_in"] is not None) == at_input - assert (carry["dyn_out"] is not None) == at_output - - new_carry, y = model.apply(params, carry, inputs) - assert y.shape == (5, io) - if at_input: - assert new_carry["dyn_in"].shape == (5, 4 * nx) - - e_inv = model.direct_to_explicit_inverse(params) - _, u_rec = model.inverse_call_noncausal(params, carry, new_carry, y, e_inv) - np.testing.assert_allclose(u_rec, inputs, atol=1e-3, rtol=1e-3) - - -def test_composition_ren_ignore_inverse_with_dyn_in(key): - model = _model(dyn_orth_at_input=True, dyn_state_multiplier=4) - params, carry, inputs = _setup(model, key, 5, 2) - _, y = model.apply(params, carry, inputs) - e_inv = model.direct_to_explicit_inverse(params) - _, sig_rec = model.inverse_call(params, carry, y, e_inv) - ed = e_inv.dyn_in - expected = carry["dyn_in"] @ ed.C.T + inputs @ ed.D.T + ed.by - np.testing.assert_allclose(sig_rec, expected, atol=1e-3, rtol=1e-3) - - -def test_composition_ren_ignore_inverse_rejects_dyn_out(key): - model = _model(dyn_orth_at_output=True, dyn_state_multiplier=4) - params, carry, inputs = _setup(model, key, 3, 2) - _, y = model.apply(params, carry, inputs) - e_inv = model.direct_to_explicit_inverse(params) - with pytest.raises(ValueError): - model.inverse_call(params, carry, y, e_inv) - - -@pytest.mark.parametrize("kw", [dict(L=0), dict(mu=2.0, nu=1.0), dict(mu=1.0, nu=1.0)]) -def test_composition_ren_invalid_args(key, kw): - with pytest.raises(ValueError): - model = _model(**kw) - _setup(model, key, 2, 2) - - -@pytest.mark.parametrize("case", golden.cases("ren_composition"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_ren_composition_matches_golden(case, explicit): - check_golden("ren_composition", case, explicit) diff --git a/tests/jax_utils.py b/tests/jax_utils.py deleted file mode 100644 index a2575f2..0000000 --- a/tests/jax_utils.py +++ /dev/null @@ -1,177 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Shared helpers for the JAX tests.""" - -import jax -import jax.numpy as jnp -import numpy as np -import flax.linen as nn -from jax.flatten_util import ravel_pytree - -import golden - - -ACTIVATIONS = [nn.relu, nn.tanh, lambda x: x] - - -def setup_ren(model, key, batches, nin, x_scale=1.0): - k1, k2, k3 = jax.random.split(key, 3) - states = model.initialize_carry(k1, (batches, nin)) - states = x_scale * jax.random.normal(k2, states.shape) - inputs = jax.random.normal(k3, (batches, nin)) - params = model.init(key, states, inputs) - return params, states, inputs - - -def make_qsr(key, nu, ny): - kx, ky, ks = jax.random.split(key, 3) - X = jax.random.normal(kx, (ny, ny)) - Y = jax.random.normal(ky, (nu, nu)) - S = jax.random.normal(ks, (nu, ny)) - Q = -X.T @ X - R = S @ jnp.linalg.solve(Q, S.T) + Y.T @ Y - return Q, S, R - - -def check_ren_forward(model, params, states, inputs, nx, ny): - batches = inputs.shape[0] - explicit = model.direct_to_explicit(params) - xn, y = model.explicit_call(params, states, inputs, explicit) - assert xn.shape == (batches, nx) - assert y.shape == (batches, ny) - xn2, y2 = model.apply(params, states, inputs) - np.testing.assert_allclose(xn, xn2, atol=1e-5) - np.testing.assert_allclose(y, y2, atol=1e-5) - - # jit gives the same answer, and gradients w.r.t. everything are finite - jit_xn, jit_y = jax.jit(model.apply)(params, states, inputs) - np.testing.assert_allclose(y, jit_y, atol=1e-4) - - def loss(p, x, u): - a, b = model.apply(p, x, u) - return jnp.sum(a ** 2) + jnp.sum(b ** 2) - g = jax.grad(loss, argnums=(0, 1, 2))(params, states, inputs) - assert bool(jnp.isfinite(ravel_pytree(g)[0]).all()) - - -def empirical_gains(f, key, n, size): - """Empirical gain |f(a) - f(b)| / |a - b| over `n` random input pairs.""" - ka, kb = jax.random.split(key) - ua = jax.random.normal(ka, (n, size)) - ub = jax.random.normal(kb, (n, size)) - return jnp.linalg.norm(f(ua) - f(ub), axis=1) / jnp.linalg.norm(ua - ub, axis=1) - - -# --------------------------------------------------------------------------- -# Golden-file support (see golden.py) -# --------------------------------------------------------------------------- - -_ACT = {"relu": nn.relu, "tanh": nn.tanh, "sigmoid": nn.sigmoid, "identity": lambda x: x} - - -def _tup(v): - return tuple(v) if isinstance(v, (list, tuple)) else v - - -def build_model(network, config): - """Construct the JAX module for a golden-file `config` (JAX argument names).""" - from robustnn import linear_ren, r2dn_jax - from robustnn import ren_jax - from robustnn.bilipnet_jax import BiLipNet - from robustnn.dyn_orthogonal_jax import DynUnitary - from robustnn.lbdn_jax import LBDN - from robustnn.monlipnet_jax import MonLipNet - from robustnn.orthogonal_jax import Unitary - from robustnn.plnet_jax import PLNet - from robustnn.ren_composition_jax import CompositionREN - - cfg = dict(config) - if "activation" in cfg: - cfg["activation"] = _ACT[cfg["activation"]] - if "act_fn" in cfg: - cfg["act_fn"] = _ACT[cfg["act_fn"]] - for k in ("units", "hidden", "hidden_sizes"): - if k in cfg: - cfg[k] = _tup(cfg[k]) - for k in ("Q", "S", "R", "optimal_point"): - if cfg.get(k) is not None: - cfg[k] = jnp.asarray(cfg[k], dtype=jnp.float32) - - if network == "plnet": - bilip_cfg = cfg.pop("BiLipBlock") - return PLNet(BiLipBlock=build_model("bilipnet", bilip_cfg), **cfg) - - cls = { - "unitary": Unitary, - "monlipnet": MonLipNet, - "bilipnet": BiLipNet, - "lbdn": LBDN, - "dyn_unitary": DynUnitary, - "contracting_ren": ren_jax.ContractingREN, - "lipschitz_ren": ren_jax.LipschitzREN, - "general_ren": ren_jax.GeneralREN, - "bilipschitz_ren": ren_jax.BiLipschitzREN, - "contracting_linear_ren": linear_ren.ContractingLinREN, - "lipschitz_linear_ren": linear_ren.LipschitzLinREN, - "general_linear_ren": linear_ren.GeneralLinREN, - "r2dn": r2dn_jax.ContractingR2DN, - "ren_composition": CompositionREN, - }[network] - return cls(**cfg) - - -def _carry_in(c): - return { - "rens": [jnp.asarray(r) for r in c["rens"]], - "dyn_in": None if c["dyn_in"] is None else jnp.asarray(c["dyn_in"]), - "dyn_out": None if c["dyn_out"] is None else jnp.asarray(c["dyn_out"]), - } - - -def _carry_out(c): - return { - "rens": np.stack([np.asarray(r) for r in c["rens"]]), - "dyn_in": None if c["dyn_in"] is None else np.asarray(c["dyn_in"]), - "dyn_out": None if c["dyn_out"] is None else np.asarray(c["dyn_out"]), - } - - -_STATIC_X = ("unitary", "monlipnet", "bilipnet", "plnet", "lbdn") - - -def run_model(network, model, params, inputs, explicit=False): - """Evaluate `model` on golden `inputs`, via `apply` or the explicit path.""" - if network in _STATIC_X: - x = jnp.asarray(inputs["x"]) - if explicit: - e = model.direct_to_explicit(params) - y = model.explicit_call(params, x, e) - else: - y = model.apply(params, x) - return {"y": np.asarray(y)} - - if network == "ren_composition": - carry, u = _carry_in(inputs["carry"]), jnp.asarray(inputs["u"]) - if explicit: - e = model.direct_to_explicit(params) - new_carry, y = model.explicit_call(params, carry, u, e) - else: - new_carry, y = model.apply(params, carry, u) - return {"carry": _carry_out(new_carry), "y": np.asarray(y)} - - # dyn_unitary and all the REN flavours: (state, u) -> (next_state, y) - x, u = jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"]) - if explicit: - e = model.direct_to_explicit(params) - xn, y = model.explicit_call(params, x, u, e) - else: - xn, y = model.apply(params, x, u) - return {"next_state": np.asarray(xn), "y": np.asarray(y)} - - -def check_golden(network, case, explicit=False): - """Check a golden case, via `apply` or `direct_to_explicit` + `explicit_call`.""" - model = build_model(network, case["config"]) - params = {"params": jax.tree.map(jnp.asarray, case["params"])} - out = run_model(network, model, params, case["inputs"], explicit=explicit) - golden.assert_close(out, case["outputs"]) diff --git a/tests/test_bilipnet.py b/tests/test_bilipnet.py new file mode 100644 index 0000000..f3deeca --- /dev/null +++ b/tests/test_bilipnet.py @@ -0,0 +1,70 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `BiLipNet` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import ATOL, assert_close, check_contract, check_golden, make_inputs + +NETWORK = "bilipnet" + + +def _config(depth=2, units=(4, 4), size=3, mu=0.5, nu=4.0, **kw): + return {"input_size": size, "units": list(units), "mu": mu, "nu": nu, + "tau": nu / mu, "is_mu_fixed": True, "is_nu_fixed": True, + "depth": depth, **kw} + + +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("depth", [1, 2, 3]) +def test_bilipnet_contract_and_bounds(backend, batches, depth): + size, mu, nu = 3, 0.5, 4.0 + config = _config(depth=depth, size=size, mu=mu, nu=nu) + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (batches, size)}) + + assert_close(net.bounds(), [mu, nu]) + gains = net.gains(200) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_bilipnet_inverse_round_trip(backend, depth): + config = _config(depth=depth, units=(4,), nu=4.0, act_fn="relu") + inputs = make_inputs(NETWORK, config, 6) + net = backend.build(NETWORK, config, inputs) + rec = net.invert(inputs, net.forward(inputs)) + np.testing.assert_allclose(rec["x"], inputs["x"], atol=1e-3) + + +def test_bilipnet_no_bias(jax_backend): + # TODO(torch): BiLipNet has no `use_bias` option in PyTorch. + config = _config(use_bias=False) + inputs = make_inputs(NETWORK, config, 5) + net = jax_backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (5, 3)}) + + +def test_bilipnet_fixing_all_raises(jax_backend): + config = {"input_size": 2, "units": [2], "is_mu_fixed": True, + "is_nu_fixed": True, "is_tau_fixed": True} + with pytest.raises(ValueError): + jax_backend.build(NETWORK, config, make_inputs(NETWORK, config, 1)) + + +def test_bilipnet_needs_two_bounds(torch_backend): + # JAX raises when all of mu/nu/tau are fixed (above); PyTorch instead + # requires two of them to be specified at construction. + config = {"input_size": 2, "units": [2], "mu": 1.0} + with pytest.raises(ValueError): + torch_backend.build(NETWORK, config, make_inputs(NETWORK, config, 1)) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipnet_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_bilipschitz_ren.py b/tests/test_bilipschitz_ren.py new file mode 100644 index 0000000..b06d7fe --- /dev/null +++ b/tests/test_bilipschitz_ren.py @@ -0,0 +1,71 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `BiLipschitzREN` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "bilipschitz_ren" + + +def _config(nio=3, mu=0.5, nu=4.0, activation="relu"): + return {"input_size": nio, "state_size": 4, "features": 8, "output_size": nio, + "mu": mu, "nu": nu, "activation": activation, "init_method": "long_memory"} + + +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("nio", [1, 3]) +@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) +@pytest.mark.parametrize("activation", ["relu", "tanh"]) +def test_bilipschitz_ren(backend, batches, nio, mu_nu, activation): + mu, nu = mu_nu + config = _config(nio, mu, nu, activation) + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + out = check_contract(net, inputs, {"next_state": (batches, 4), "y": (batches, nio)}) + + if activation == "relu": # the inverse solver is only checked for relu + rec = net.invert(inputs, out) + np.testing.assert_allclose(rec["u"], inputs["u"], atol=1e-3, rtol=1e-3) + + gains = net.gains(300) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + + +@pytest.mark.parametrize("mu,nu", [(2.0, 1.0), (1.0, 1.0)]) +def test_bilipschitz_ren_invalid_bounds(torch_backend, mu, nu): + # TODO(jax): the JAX BiLipschitzREN accepts mu >= nu (neither the constructor, + # `init` nor `check_valid_qsr` raise), unlike PyTorch and the JAX CompositionREN. + # Share this test once JAX validates its bounds. + config = _config(mu=mu, nu=nu) + with pytest.raises(ValueError): + torch_backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +def test_bilipschitz_ren_valid_qsr(jax_backend): + config = _config() + net = jax_backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + net.model.check_valid_qsr() + + +def test_bilipschitz_ren_requires_square(jax_backend): + config = _config() | {"output_size": 2} + with pytest.raises(ValueError): + jax_backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +def test_ren_base_direct_to_explicit_not_implemented(torch_backend): + from robustnn.ren_torch import RENBase + + with pytest.raises(NotImplementedError): + RENBase(2, 3, 4, 2).direct_to_explicit() + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_bilipschitz_ren_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_contracting_ren.py b/tests/test_contracting_ren.py new file mode 100644 index 0000000..6afacb6 --- /dev/null +++ b/tests/test_contracting_ren.py @@ -0,0 +1,113 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `ContractingREN`. + +TODO(torch): ContractingREN is not yet implemented in PyTorch, so the shared +tests are skipped for the torch backend and the JAX-only tests are not collected +for it. +""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "contracting_ren" + + +def _config(nu=3, nx=3, nv=4, ny=2, **kw): + return {"input_size": nu, "state_size": nx, "features": nv, "output_size": ny, **kw} + + +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("activation", ["relu", "tanh", "identity"]) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +@pytest.mark.parametrize("do_polar_param", [True, False]) +@pytest.mark.parametrize("sizes", [(1, 1, 1, 1), (5, 3, 4, 2)]) +def test_contracting_ren(backend, batches, activation, init_method, do_polar_param, sizes): + nu, nx, nv, ny = sizes + config = _config(nu, nx, nv, ny, activation=activation, init_method=init_method, + do_polar_param=do_polar_param) + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"next_state": (batches, nx), "y": (batches, ny)}) + + +def test_contracting_ren_contraction(jax_backend): + # TODO(torch): needs the P matrix of the PyTorch parameterisation. + import jax + import jax.numpy as jnp + from helpers import compute_p_contractingren + + config = _config(5, 3, 4, 2, activation="tanh") + inputs = make_inputs(NETWORK, config, 8) + net = jax_backend.build(NETWORK, config, inputs) + model, params = net.model, net.params + x1, u = jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"]) + x0 = 10 * jax.random.normal(jax.random.key(0), x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingren(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) + + +def test_ren_identity_output(backend): + config = _config(3, 3, 4, 3, identity_output=True) + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + # identity output: y_t = x_t, i.e. the state before the update + np.testing.assert_allclose(net.forward(inputs)["y"], inputs["state"], atol=1e-5) + + +def test_ren_init_output_zero(backend): + config = _config(3, 3, 4, 2, init_output_zero=True) + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + np.testing.assert_allclose(net.forward(inputs)["y"], 0.0, atol=1e-6) + + +def test_ren_identity_output_size_mismatch(backend): + config = _config(3, 3, 4, 2, identity_output=True) + with pytest.raises(ValueError): + backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +def test_ren_bad_init_method(backend): + config = _config(init_method="nope") + with pytest.raises(ValueError): + backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +@pytest.mark.parametrize("horizon", [1, 5]) +def test_simulate_sequence_matches_loop(jax_backend, horizon): + # TODO(torch): `simulate_sequence` has no PyTorch counterpart yet. + import jax + import jax.numpy as jnp + + nu, nx, ny = 3, 4, 2 + config = _config(nu, nx, 5, ny, activation="tanh") + inputs = make_inputs(NETWORK, config, 4) + net = jax_backend.build(NETWORK, config, inputs) + model, params = net.model, net.params + x0 = jnp.asarray(inputs["state"]) + seq = jax.random.normal(jax.random.key(0), (horizon, 4, nu)) + + xT, ys = model.simulate_sequence(params, x0, seq) + assert ys.shape == (horizon, 4, ny) + + x, outs = x0, [] + for t in range(horizon): + x, y = model.apply(params, x, seq[t]) + outs.append(y) + np.testing.assert_allclose(xT, x, atol=1e-4) + np.testing.assert_allclose(ys, jnp.stack(outs), atol=1e-4) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_contracting_ren_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_dyn_orthogonal.py b/tests/test_dyn_orthogonal.py new file mode 100644 index 0000000..8e79fb5 --- /dev/null +++ b/tests/test_dyn_orthogonal.py @@ -0,0 +1,36 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `DynUnitary` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import ATOL, check_contract, check_golden, make_inputs + +NETWORK = "dyn_unitary" + + +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) +@pytest.mark.parametrize("use_bias", [True, False]) +def test_dyn_unitary(backend, batches, io, nx, use_bias): + config = {"input_size": io, "state_size": nx, "use_bias": use_bias} + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + out = check_contract(net, inputs, {"next_state": (batches, nx), "y": (batches, io)}) + + # The stacked state-space matrix is orthogonal. + G = net.orthogonal_matrix() + np.testing.assert_allclose(G @ G.T, np.eye(nx + io), atol=ATOL) + + # The non-causal inverse recovers the previous state and the input. + rec = net.invert(inputs, out) + np.testing.assert_allclose(rec["state"], inputs["state"], atol=1e-4) + np.testing.assert_allclose(rec["u"], inputs["u"], atol=1e-4) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_dyn_unitary_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_general_ren.py b/tests/test_general_ren.py new file mode 100644 index 0000000..4c503d1 --- /dev/null +++ b/tests/test_general_ren.py @@ -0,0 +1,60 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `GeneralREN`. + +TODO(torch): GeneralREN is not yet implemented in PyTorch, so the shared tests +are skipped for the torch backend and the JAX-only tests are not collected for it. +""" + +import numpy as np +import pytest + +import golden +from backends import build_jax_model, check_contract, check_golden, make_inputs, make_qsr + +NETWORK = "general_ren" + + +def _config(nu, nx, nv, ny, **kw): + Q, S, R = make_qsr(nu, ny) + return {"input_size": nu, "state_size": nx, "features": nv, "output_size": ny, + "Q": Q, "S": S, "R": R, **kw} + + +@pytest.mark.parametrize("activation", ["relu", "tanh", "identity"]) +@pytest.mark.parametrize("sizes", [(5, 3, 4, 2), (2, 2, 3, 2)]) +def test_general_ren(backend, activation, sizes): + nu, nx, nv, ny = sizes + config = _config(nu, nx, nv, ny, activation=activation, init_method="long_memory") + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"next_state": (4, nx), "y": (4, ny)}) + + +def test_general_ren_valid_qsr(jax_backend): + # TODO(torch): `check_valid_qsr` has no PyTorch counterpart yet. + config = _config(5, 3, 4, 2) + build_jax_model(NETWORK, config).check_valid_qsr() + + +def test_general_ren_invalid_qsr(jax_backend): + nu, nx, nv, ny = 3, 3, 4, 2 + Q, S, R = make_qsr(nu, ny) + bad = [ + dict(Q=np.eye(ny + 1), S=S, R=R), # wrong Q size + dict(Q=Q, S=np.ones((ny, nu)), R=R), # wrong S size + dict(Q=Q, S=S, R=np.eye(nu + 1)), # wrong R size + dict(Q=np.eye(ny), S=S, R=R), # Q not neg. definite + dict(Q=Q, S=S, R=-np.eye(nu)), # Schur complement not PD + ] + for kw in bad: + config = {"input_size": nu, "state_size": nx, "features": nv, + "output_size": ny, **kw} + with pytest.raises(ValueError): + build_jax_model(NETWORK, config).check_valid_qsr() + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_general_ren_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_lbdn.py b/tests/test_lbdn.py new file mode 100644 index 0000000..1e1bbfd --- /dev/null +++ b/tests/test_lbdn.py @@ -0,0 +1,53 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `LBDN`. + +TODO(torch): LBDN is not yet implemented in PyTorch, so every test here is +skipped for the torch backend. They will run for it once `backends.py` lists +`lbdn` as supported. +""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "lbdn" + + +@pytest.mark.parametrize("activation", ["relu", "tanh", "sigmoid", "identity"]) +@pytest.mark.parametrize("hidden", [[4], [8, 16]]) +@pytest.mark.parametrize("batches", [1, 5]) +def test_lbdn_contract_and_lipschitz(backend, activation, hidden, batches): + nin, nout, gamma = 5, 2, 3.0 + config = {"input_size": nin, "hidden_sizes": hidden, "output_size": nout, + "gamma": gamma, "activation": activation} + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (batches, nout)}) + assert net.gains(200).max() <= gamma + 1e-3 + + +@pytest.mark.parametrize("use_bias", [True, False]) +@pytest.mark.parametrize("trainable_lipschitz", [True, False]) +def test_lbdn_options(backend, use_bias, trainable_lipschitz): + config = {"input_size": 3, "hidden_sizes": [6], "output_size": 2, "gamma": 2.0, + "use_bias": use_bias, "trainable_lipschitz": trainable_lipschitz} + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (4, 2)}) + + +def test_lbdn_init_output_zero(backend): + config = {"input_size": 3, "hidden_sizes": [6], "output_size": 2, + "init_output_zero": True} + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + np.testing.assert_allclose(net.forward(inputs)["y"], 0.0, atol=1e-6) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_lbdn_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_linear_ren.py b/tests/test_linear_ren.py new file mode 100644 index 0000000..298da3c --- /dev/null +++ b/tests/test_linear_ren.py @@ -0,0 +1,64 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the linear RENs (contracting, Lipschitz and general). + +TODO(torch): the linear RENs are not yet implemented in PyTorch, so every test +here is skipped for the torch backend. +""" + +import pytest + +import golden +from backends import build_jax_model, check_contract, check_golden, make_inputs, make_qsr + +NU, NX, NY = 5, 3, 2 + + +def _config(kind, **kw): + cfg = {"input_size": NU, "state_size": NX, "features": 0, "output_size": NY} + if kind == "lipschitz": + cfg["gamma"] = 2.0 + if kind == "general": + Q, S, R = make_qsr(NU, NY) + cfg |= {"Q": Q, "S": S, "R": R, "init_method": "long_memory"} + return cfg | kw + + +@pytest.mark.parametrize("kind", ["contracting", "lipschitz", "general"]) +@pytest.mark.parametrize("batches", [1, 4]) +def test_linear_ren(backend, kind, batches): + network = f"{kind}_linear_ren" + config = _config(kind) + inputs = make_inputs(network, config, batches) + net = backend.build(network, config, inputs) + check_contract(net, inputs, {"next_state": (batches, NX), "y": (batches, NY)}) + + +def test_general_linear_ren_valid_qsr(jax_backend): + # TODO(torch): `check_valid_qsr` has no PyTorch counterpart yet. + build_jax_model("general_linear_ren", _config("general")).check_valid_qsr() + + +def test_linear_ren_lipschitz_bound(backend): + network, gamma = "lipschitz_linear_ren", 2.0 + config = _config("lipschitz", input_size=4, gamma=gamma) + net = backend.build(network, config, make_inputs(network, config, 2)) + assert net.gains(200).max() <= gamma + 1e-3 + + +def test_linear_ren_requires_zero_features(backend): + network = "contracting_linear_ren" + config = _config("contracting", features=4) + with pytest.raises(ValueError): + backend.build(network, config, make_inputs(network, config, 2)) + + +GOLDEN = [(n, c) for n in ("contracting_linear_ren", "lipschitz_linear_ren", + "general_linear_ren") for c in golden.cases(n)] + + +@pytest.mark.parametrize("network,case", GOLDEN, + ids=[f"{n}-{golden.case_id(c)}" for n, c in GOLDEN]) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_linear_ren_matches_golden(backend, network, case, explicit): + check_golden(backend, network, case, explicit) diff --git a/tests/test_lipschitz_ren.py b/tests/test_lipschitz_ren.py new file mode 100644 index 0000000..3b1f05a --- /dev/null +++ b/tests/test_lipschitz_ren.py @@ -0,0 +1,41 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `LipschitzREN`. + +TODO(torch): LipschitzREN is not yet implemented in PyTorch, so every test here +is skipped for the torch backend. +""" + +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "lipschitz_ren" + + +@pytest.mark.parametrize("gamma", [0.5, 1.0, 5.0]) +@pytest.mark.parametrize("activation", ["relu", "tanh", "identity"]) +def test_lipschitz_ren(backend, gamma, activation): + nu, nx, nv, ny = 4, 3, 5, 2 + config = {"input_size": nu, "state_size": nx, "features": nv, "output_size": ny, + "gamma": gamma, "activation": activation} + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"next_state": (4, nx), "y": (4, ny)}) + + # Same initial state => one-step output gain is bounded by gamma. + assert net.gains(200).max() <= gamma + 1e-3 + + +def test_lipschitz_ren_identity_output_unsupported(backend): + config = {"input_size": 3, "state_size": 3, "features": 4, "output_size": 3, + "identity_output": True} + with pytest.raises(NotImplementedError): + backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_lipschitz_ren_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_monlipnet.py b/tests/test_monlipnet.py new file mode 100644 index 0000000..687baa5 --- /dev/null +++ b/tests/test_monlipnet.py @@ -0,0 +1,61 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `MonLipNet` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "monlipnet" + + +@pytest.mark.parametrize("batches", [1, 5]) +@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) +@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) +def test_monlipnet_contract_and_bounds(backend, batches, units, fixed): + size = 3 + config = {"input_size": size, "units": units, "mu": 0.5, "nu": 4.0, "tau": 4.0, + "is_mu_fixed": "mu" in fixed, "is_nu_fixed": "nu" in fixed, + "is_tau_fixed": "tau" in fixed} + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (batches, size)}) + + lo, hi = net.bounds() + gains = net.gains(200) + assert gains.min() >= lo - 1e-3 + assert gains.max() <= hi + 1e-3 + + +@pytest.mark.parametrize("units", [[3], [4, 4]]) +def test_monlipnet_inverse_round_trip(backend, units): + config = {"input_size": 3, "units": units, "mu": 0.5, "nu": 4.0, "tau": 4.0, + "is_tau_fixed": True} + inputs = make_inputs(NETWORK, config, 6) + net = backend.build(NETWORK, config, inputs) + rec = net.invert(inputs, net.forward(inputs)) + np.testing.assert_allclose(rec["x"], inputs["x"], atol=1e-3) + + +def test_monlipnet_fixing_all_raises(jax_backend): + config = {"input_size": 2, "units": [2], "is_mu_fixed": True, + "is_nu_fixed": True, "is_tau_fixed": True} + with pytest.raises(ValueError): + jax_backend.build(NETWORK, config, make_inputs(NETWORK, config, 1)) + + +@pytest.mark.parametrize("kw", [dict(mu=1.0), dict(), dict(nu=2.0)]) +def test_monlipnet_needs_two_bounds(torch_backend, kw): + # JAX raises when all of mu/nu/tau are fixed (above); PyTorch instead + # requires two of them to be specified at construction. + config = {"input_size": 2, "units": [2], **kw} + with pytest.raises(ValueError): + torch_backend.build(NETWORK, config, make_inputs(NETWORK, config, 1)) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_monlipnet_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_orthogonal.py b/tests/test_orthogonal.py new file mode 100644 index 0000000..ca3e8e5 --- /dev/null +++ b/tests/test_orthogonal.py @@ -0,0 +1,54 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `Unitary` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import ATOL, assert_close, check_contract, check_golden, make_inputs + +NETWORK = "unitary" + + +@pytest.mark.parametrize("batches", [1, 7]) +@pytest.mark.parametrize("size", [1, 2, 5]) +def test_unitary_contract_and_orthogonal(backend, batches, size): + config = {"input_size": size, "use_bias": True} + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (batches, size)}) + + R = net.orthogonal_matrix() + np.testing.assert_allclose(R @ R.T, np.eye(size), atol=ATOL) + + +@pytest.mark.parametrize("size", [2, 5]) +def test_unitary_no_bias_preserves_norm(backend, size): + config = {"input_size": size, "use_bias": False} + inputs = make_inputs(NETWORK, config, 8) + net = backend.build(NETWORK, config, inputs) + y = net.forward(inputs)["y"] + assert_close(np.linalg.norm(y, axis=1), np.linalg.norm(inputs["x"], axis=1)) + + +def test_unitary_no_bias_explicit(backend): + config = {"input_size": 3, "use_bias": False} + inputs = make_inputs(NETWORK, config, 4) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (4, 3)}) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +def test_unitary_inverse_round_trip(backend, case): + # The golden parameters have a non-zero bias, which the inverse has to undo. + inputs = case["inputs"] + net = backend.build(NETWORK, case["config"], inputs, case["params"]) + rec = net.invert(inputs, net.forward(inputs)) + np.testing.assert_allclose(rec["x"], inputs["x"], atol=1e-4) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_unitary_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_plnet.py b/tests/test_plnet.py new file mode 100644 index 0000000..6879aaa --- /dev/null +++ b/tests/test_plnet.py @@ -0,0 +1,48 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `PLNet` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "plnet" + +BILIP = {"input_size": 3, "units": [4], "mu": 0.5, "nu": 4.0, "tau": 8.0, + "is_mu_fixed": True, "is_nu_fixed": True, "depth": 2} + + +@pytest.mark.parametrize("batches", [1, 6]) +def test_plnet_contract_and_nonnegative(backend, batches): + config = {"BiLipBlock": BILIP} + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + out = check_contract(net, inputs, {"y": (batches,)}) + assert (out["y"] >= 0).all() + + +def test_plnet_zero_at_optimal_point(backend): + x = make_inputs(NETWORK, {"BiLipBlock": BILIP}, 4)["x"] + config = {"BiLipBlock": BILIP, "optimal_point": x[:1].tolist()} + net = backend.build(NETWORK, config, {"x": x}) + y_opt = net.explicit({"x": x[:1]})["y"] + np.testing.assert_allclose(y_opt, 0.0, atol=1e-6) + assert (net.explicit({"x": x})["y"] >= 0).all() + + +def test_plnet_add_constant(jax_backend): + # TODO(torch): the explicit path with `add_constant=True` currently fails in + # PyTorch (see test_plnet_matches_golden[explicit-add_constant]); share this + # test once that is fixed. + config = {"BiLipBlock": BILIP, "add_constant": True, "c": 1.5} + inputs = make_inputs(NETWORK, config, 4) + net = jax_backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"y": (4,)}) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_plnet_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_r2dn.py b/tests/test_r2dn.py new file mode 100644 index 0000000..7caf3a4 --- /dev/null +++ b/tests/test_r2dn.py @@ -0,0 +1,61 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for the contracting R2DN. + +TODO(torch): R2DN is not yet implemented in PyTorch, so the shared tests are +skipped for the torch backend and the JAX-only tests are not collected for it. +""" + +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "r2dn" + + +def _config(hidden, init_method="long_memory"): + return {"input_size": 5, "state_size": 3, "features": 4, "output_size": 2, + "hidden": hidden, "init_method": init_method} + + +@pytest.mark.parametrize("batches", [1, 4]) +@pytest.mark.parametrize("hidden", [[2], [2, 2], [4, 3, 2]]) +@pytest.mark.parametrize("init_method", ["random", "long_memory"]) +def test_contracting_r2dn(backend, batches, hidden, init_method): + config = _config(hidden, init_method) + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + check_contract(net, inputs, {"next_state": (batches, 3), "y": (batches, 2)}) + + +def test_r2dn_simulate_and_contraction(jax_backend): + import jax + import jax.numpy as jnp + from helpers import compute_p_contractingr2dn + + nu, nx, ny, horizon = 5, 3, 2, 3 + config = _config([2, 2]) + inputs = make_inputs(NETWORK, config, 4) + net = jax_backend.build(NETWORK, config, inputs) + model, params = net.model, net.params + x1, u = jnp.asarray(inputs["state"]), jnp.asarray(inputs["u"]) + + seq = jax.random.normal(jax.random.key(0), (horizon, 4, nu)) + xT, ys = jax.jit(model.simulate_sequence)(params, x1, seq) + assert ys.shape == (horizon, 4, ny) and xT.shape == x1.shape + + x0 = 10 * jax.random.normal(jax.random.key(1), x1.shape) + xn0, _ = model.apply(params, x0, u) + xn1, _ = model.apply(params, x1, u) + P = compute_p_contractingr2dn(model, params) + + def norm2(x): + return jnp.sum((x @ P.T) * x, axis=-1) + assert bool(jnp.all(norm2(xn0 - xn1) - norm2(x0 - x1) <= 1e-4)) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_r2dn_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/test_ren_composition.py b/tests/test_ren_composition.py new file mode 100644 index 0000000..0bf51af --- /dev/null +++ b/tests/test_ren_composition.py @@ -0,0 +1,91 @@ +# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE + +"""Tests for `CompositionREN` (JAX and PyTorch).""" + +import numpy as np +import pytest + +import golden +from backends import check_contract, check_golden, make_inputs + +NETWORK = "ren_composition" + + +def _config(io=2, nx=3, L=2, mu=0.5, nu=5.0, **kw): + return {"input_size": io, "state_size": nx, "features": 6, "num_layers": L, + "mu": mu, "nu": nu, "init_method": "long_memory", + "dyn_state_multiplier": 4, **kw} + + +@pytest.mark.parametrize("batches", [1, 6]) +@pytest.mark.parametrize("L", [1, 3]) +@pytest.mark.parametrize("use_bias", [True, False]) +@pytest.mark.parametrize("activation", ["relu", "tanh"]) +def test_composition_ren(backend, batches, L, use_bias, activation): + io, nx, mu, nu = 2, 3, 0.5, 5.0 + config = _config(io, nx, L, mu, nu, use_bias=use_bias, activation=activation) + inputs = make_inputs(NETWORK, config, batches) + net = backend.build(NETWORK, config, inputs) + out = check_contract(net, inputs, {"y": (batches, io)}) + + carry = out["carry"] + assert carry["rens"].shape == (L, batches, nx) + assert carry["dyn_in"] is None and carry["dyn_out"] is None + + if activation == "relu": + # The inverse solver is only accurate to ~1e-3 for smooth activations + # at the default iteration count, so only check it for relu. + rec = net.invert(inputs, out) + np.testing.assert_allclose(rec["u"], inputs["u"], atol=1e-3, rtol=1e-3) + + gains = net.gains(300) + assert gains.min() >= mu - 1e-3 + assert gains.max() <= nu + 1e-3 + assert net.bounds() == (mu, nu) + + +@pytest.mark.parametrize("at_input,at_output", + [(True, False), (False, True), (True, True)]) +def test_composition_ren_dyn_orth_noncausal_inverse(backend, at_input, at_output): + io, nx = 2, 3 + config = _config(io, nx, dyn_orth_at_input=at_input, dyn_orth_at_output=at_output) + inputs = make_inputs(NETWORK, config, 5) + net = backend.build(NETWORK, config, inputs) + out = net.forward(inputs) + assert out["y"].shape == (5, io) + if at_input: + assert out["carry"]["dyn_in"].shape == (5, 4 * nx) + + rec = net.invert(inputs, out, noncausal=True) + np.testing.assert_allclose(rec["u"], inputs["u"], atol=1e-3, rtol=1e-3) + + +def test_composition_ren_ignore_inverse_with_dyn_in(backend): + config = _config(dyn_orth_at_input=True) + inputs = make_inputs(NETWORK, config, 5) + net = backend.build(NETWORK, config, inputs) + rec = net.invert(inputs, net.forward(inputs)) + # The causal inverse skips the dynamic input layer, so it returns that + # layer's output rather than the network input. + np.testing.assert_allclose(rec["u"], net.dyn_in_output(inputs), atol=1e-3, rtol=1e-3) + + +def test_composition_ren_ignore_inverse_rejects_dyn_out(backend): + config = _config(dyn_orth_at_output=True) + inputs = make_inputs(NETWORK, config, 3) + net = backend.build(NETWORK, config, inputs) + with pytest.raises(ValueError): + net.invert(inputs, net.forward(inputs)) + + +@pytest.mark.parametrize("kw", [dict(L=0), dict(mu=2.0, nu=1.0), dict(mu=1.0, nu=1.0)]) +def test_composition_ren_invalid_args(backend, kw): + config = _config(**kw) + with pytest.raises(ValueError): + backend.build(NETWORK, config, make_inputs(NETWORK, config, 2)) + + +@pytest.mark.parametrize("case", golden.cases(NETWORK), ids=golden.case_id) +@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) +def test_ren_composition_matches_golden(backend, case, explicit): + check_golden(backend, NETWORK, case, explicit) diff --git a/tests/torch_tests/conftest.py b/tests/torch_tests/conftest.py deleted file mode 100644 index 2109078..0000000 --- a/tests/torch_tests/conftest.py +++ /dev/null @@ -1,18 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -import pytest - -pytest.importorskip("torch") - -import torch - - -def pytest_collection_modifyitems(items): - for item in items: - if "torch_tests" in str(item.fspath): - item.add_marker(pytest.mark.torch) - - -@pytest.fixture(autouse=True) -def _seed_torch(): - torch.manual_seed(0) diff --git a/tests/torch_tests/test_bilipnet.py b/tests/torch_tests/test_bilipnet.py deleted file mode 100644 index a4fc46c..0000000 --- a/tests/torch_tests/test_bilipnet.py +++ /dev/null @@ -1,57 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `BiLipNet`.""" - -import numpy as np -import pytest -import torch - -import golden -from robustnn.bilipnet_torch import BiLipNet -from torch_utils import check_golden, empirical_gains, grads_finite - - -relu_np = lambda x: np.maximum(0, x) - - -# ------------------------------- BiLipNet --------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("depth", [1, 2, 3]) -def test_bilipnet(batches, depth): - size, mu, nu = 3, 0.5, 4.0 - net = BiLipNet(size, [4, 4], mu=mu, nu=nu, tau=nu / mu, - is_mu_fixed=True, is_nu_fixed=True, depth=depth) - x = torch.randn(batches, size) - y = net(x) - assert tuple(y.shape) == (batches, size) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - - lo, hi, _ = net.get_bounds() - np.testing.assert_allclose([lo, hi], [mu, nu], rtol=1e-4) - gains = empirical_gains(net, 200, size) - assert gains.min() >= mu - 1e-3 - assert gains.max() <= nu + 1e-3 - - (y ** 2).sum().backward() - grads_finite(net) - - -@pytest.mark.parametrize("depth", [1, 2]) -def test_bilipnet_inverse_round_trip(depth): - net = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, - is_nu_fixed=True, depth=depth) - x = torch.randn(6, 3) - with torch.no_grad(): - y = net(x) - x_rec = net.inverse(y.numpy(), alphas=[0.1] * depth, - inverse_activation_fns=[relu_np] * depth, - iterations=[500] * depth, Lambdas=[1.0] * depth) - np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) - - -@pytest.mark.parametrize("case", golden.cases("bilipnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_bilipnet_matches_golden(case, explicit): - check_golden("bilipnet", case, explicit) diff --git a/tests/torch_tests/test_bilipschitz_ren.py b/tests/torch_tests/test_bilipschitz_ren.py deleted file mode 100644 index b9d6402..0000000 --- a/tests/torch_tests/test_bilipschitz_ren.py +++ /dev/null @@ -1,70 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `BiLipschitzREN`.""" - -import pytest -import torch -import torch.nn as nn - -import golden -from robustnn.ren_torch import BiLipschitzREN, RENBase -from torch_utils import check_golden, grads_finite, rel_err - - -# ------------------------------ BiLipschitzREN ---------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("nio", [1, 3]) -@pytest.mark.parametrize("mu_nu", [(0.5, 4.0), (1.0, 2.0)]) -@pytest.mark.parametrize("activation", [nn.ReLU(), nn.Tanh()]) -def test_bilipschitz_ren(batches, nio, mu_nu, activation): - mu, nu = mu_nu - nx, nv = 4, 8 - model = BiLipschitzREN(nio, nx, nv, mu=mu, nu=nu, activation=activation) - - state = model.initialize_carry(batches) - assert tuple(state.shape) == (batches, nx) - u = torch.randn(batches, nio) - new_state, y = model(state, u) - assert tuple(new_state.shape) == (batches, nx) - assert tuple(y.shape) == (batches, nio) - - with torch.no_grad(): - e = model.direct_to_explicit() - _, y_exp = model.explicit_call(state, u, e) - torch.testing.assert_close(y_exp, y, atol=1e-4, rtol=1e-4) - - if isinstance(activation, nn.ReLU): - with torch.no_grad(): - _, u_rec = model.inverse(state, y) - assert rel_err(u_rec, u) < 1e-3 - - n = 300 - z = torch.zeros(n, nx) - ua, ub = torch.randn(n, nio), torch.randn(n, nio) - with torch.no_grad(): - _, ya = model.explicit_call(z, ua, e) - _, yb = model.explicit_call(z, ub, e) - gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - - (y ** 2).sum().backward() - grads_finite(model) - - -@pytest.mark.parametrize("mu,nu", [(2.0, 1.0), (1.0, 1.0)]) -def test_bilipschitz_ren_invalid_bounds(mu, nu): - with pytest.raises(ValueError): - BiLipschitzREN(2, 3, 4, mu=mu, nu=nu) - - -def test_ren_base_direct_to_explicit_not_implemented(): - base = RENBase(2, 3, 4, 2) - with pytest.raises(NotImplementedError): - base.direct_to_explicit() - - -@pytest.mark.parametrize("case", golden.cases("bilipschitz_ren"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_bilipschitz_ren_matches_golden(case, explicit): - check_golden("bilipschitz_ren", case, explicit) diff --git a/tests/torch_tests/test_dyn_orthogonal.py b/tests/torch_tests/test_dyn_orthogonal.py deleted file mode 100644 index 4f17d91..0000000 --- a/tests/torch_tests/test_dyn_orthogonal.py +++ /dev/null @@ -1,40 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `DynUnitary`.""" - -import pytest -import torch - -import golden -from robustnn.dyn_orthogonal_torch import DynUnitary -from torch_utils import check_golden, grads_finite - - -# ------------------------------- DynUnitary ------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("io,nx", [(1, 1), (3, 4), (2, 6)]) -@pytest.mark.parametrize("bias", [True, False]) -def test_dyn_unitary(batches, io, nx, bias): - layer = DynUnitary(io, nx, bias=bias) - x, u = torch.randn(batches, nx), torch.randn(batches, io) - assert tuple(layer.initialize_carry(batches).shape) == (batches, nx) - - x1, y = layer(x, u) - assert tuple(x1.shape) == (batches, nx) and tuple(y.shape) == (batches, io) - - A, B, C, D = layer._blocks() - G = torch.cat([torch.cat([A, B], 1), torch.cat([C, D], 1)], 0) - torch.testing.assert_close(G @ G.T, torch.eye(nx + io), atol=1e-5, rtol=0) - - x_rec, u_rec = layer.inverse(x1, y) - torch.testing.assert_close(x_rec, x, atol=1e-4, rtol=1e-4) - torch.testing.assert_close(u_rec, u, atol=1e-4, rtol=1e-4) - - (x1 ** 2).sum().add((y ** 2).sum()).backward() - grads_finite(layer) - - -@pytest.mark.parametrize("case", golden.cases("dyn_unitary"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_dyn_unitary_matches_golden(case, explicit): - check_golden("dyn_unitary", case, explicit) diff --git a/tests/torch_tests/test_monlipnet.py b/tests/torch_tests/test_monlipnet.py deleted file mode 100644 index 5311162..0000000 --- a/tests/torch_tests/test_monlipnet.py +++ /dev/null @@ -1,58 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `MonLipNet`.""" - -import numpy as np -import pytest -import torch - -import golden -from robustnn.monlipnet_torch import MonLipNet -from torch_utils import check_golden, empirical_gains, grads_finite - - -# ------------------------------- MonLipNet -------------------------------- -@pytest.mark.parametrize("batches", [1, 5]) -@pytest.mark.parametrize("units", [[3], [4, 4], [3, 3, 3]]) -@pytest.mark.parametrize("fixed", ["mu_nu", "mu_tau", "nu_tau"]) -def test_monlipnet(batches, units, fixed): - size, mu, nu, tau = 3, 0.5, 4.0, 4.0 - net = MonLipNet(size, units, mu=mu, nu=nu, tau=tau, - is_mu_fixed="mu" in fixed, is_nu_fixed="nu" in fixed, - is_tau_fixed="tau" in fixed) - x = torch.randn(batches, size) - y = net(x) - assert tuple(y.shape) == (batches, size) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - - lo, hi, _ = net.get_bounds() - gains = empirical_gains(net, 200, size) - assert gains.min() >= float(lo) - 1e-3 - assert gains.max() <= float(hi) + 1e-3 - - (y ** 2).sum().backward() - grads_finite(net) - - -@pytest.mark.parametrize("units", [[3], [4, 4]]) -def test_monlipnet_inverse_round_trip(units): - net = MonLipNet(3, units, mu=0.5, nu=4.0, tau=4.0, is_tau_fixed=True) - x = torch.randn(6, 3) - with torch.no_grad(): - y = net(x) - x_rec = net.inverse(y.numpy(), alpha=0.1, iterations=500) - np.testing.assert_allclose(x_rec, x.numpy(), atol=1e-3) - - -@pytest.mark.parametrize("kw", [dict(mu=1.0), dict(), dict(nu=2.0)]) -def test_monlipnet_needs_two_bounds(kw): - with pytest.raises(ValueError): - MonLipNet(2, [2], **kw) - - -@pytest.mark.parametrize("case", golden.cases("monlipnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_monlipnet_matches_golden(case, explicit): - check_golden("monlipnet", case, explicit) diff --git a/tests/torch_tests/test_orthogonal.py b/tests/torch_tests/test_orthogonal.py deleted file mode 100644 index f091f78..0000000 --- a/tests/torch_tests/test_orthogonal.py +++ /dev/null @@ -1,50 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `Unitary` layer.""" - -import numpy as np -import pytest -import torch - -import golden -from robustnn.orthogonal_torch import Unitary -from torch_utils import check_golden, grads_finite - - -# -------------------------------- Unitary --------------------------------- -@pytest.mark.parametrize("batches", [1, 7]) -@pytest.mark.parametrize("size", [1, 2, 5]) -def test_unitary(batches, size): - layer = Unitary(size, size, bias=True) - x = torch.randn(batches, size) - y = layer(x) - assert tuple(y.shape) == (batches, size) - - e = layer.direct_to_explicit() - np.testing.assert_allclose(e.Q @ e.Q.T, np.eye(size), atol=1e-5) - np.testing.assert_allclose(layer.explicit_call(x.numpy(), e), - y.detach().numpy(), atol=1e-5) - - with torch.no_grad(): - x_rec = layer.inverse(y) - np.testing.assert_allclose(np.asarray(x_rec), x.numpy(), atol=1e-4) - - (y ** 2).sum().backward() - grads_finite(layer) - - -@pytest.mark.parametrize("size", [2, 5]) -def test_unitary_no_bias_forward_preserves_norm(size): - layer = Unitary(size, size, bias=False) - x = torch.randn(8, size) - np.testing.assert_allclose(layer(x).norm(dim=1).detach(), x.norm(dim=1), rtol=1e-4) - - -def test_unitary_no_bias_explicit(): - Unitary(3, 3, bias=False).direct_to_explicit() - - -@pytest.mark.parametrize("case", golden.cases("unitary"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_unitary_matches_golden(case, explicit): - check_golden("unitary", case, explicit) diff --git a/tests/torch_tests/test_plnet.py b/tests/torch_tests/test_plnet.py deleted file mode 100644 index 4cba659..0000000 --- a/tests/torch_tests/test_plnet.py +++ /dev/null @@ -1,48 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `PLNet`.""" - -import numpy as np -import pytest -import torch - -import golden -from robustnn.bilipnet_torch import BiLipNet -from robustnn.plnet_torch import PLNet -from torch_utils import check_golden, grads_finite - - -# --------------------------------- PLNet ---------------------------------- -def _plnet(**kw): - bilip = BiLipNet(3, [4], mu=0.5, nu=4.0, tau=8.0, is_mu_fixed=True, - is_nu_fixed=True, depth=2) - return PLNet(BiLipBlock=bilip, **kw) - - -@pytest.mark.parametrize("batches", [1, 6]) -def test_plnet(batches): - net = _plnet() - x = torch.randn(batches, 3) - y = net(x) - assert tuple(y.shape) == (batches,) - assert bool((y >= 0).all()) - np.testing.assert_allclose( - net.explicit_call(x.numpy(), net.direct_to_explicit()), - y.detach().numpy(), atol=1e-4) - y.sum().backward() - grads_finite(net) - - -def test_plnet_zero_at_optimal_point(): - net = _plnet() - x = torch.randn(4, 3) - e = net.direct_to_explicit(x_optimal=x[:1].numpy()) - y = net.explicit_call(x[:1].numpy(), e) - np.testing.assert_allclose(y, 0.0, atol=1e-6) - assert bool((net.explicit_call(x.numpy(), e) >= 0).all()) - - -@pytest.mark.parametrize("case", golden.cases("plnet"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_plnet_matches_golden(case, explicit): - check_golden("plnet", case, explicit) diff --git a/tests/torch_tests/test_ren_composition.py b/tests/torch_tests/test_ren_composition.py deleted file mode 100644 index e625c53..0000000 --- a/tests/torch_tests/test_ren_composition.py +++ /dev/null @@ -1,94 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Tests for the PyTorch `CompositionREN`.""" - -import pytest -import torch - -import golden -from robustnn.ren_composition_torch import CompositionREN -from torch_utils import check_golden, grads_finite, rel_err - - -# ------------------------------ CompositionREN ---------------------------- -@pytest.mark.parametrize("batches", [1, 6]) -@pytest.mark.parametrize("L", [1, 3]) -@pytest.mark.parametrize("use_bias", [True, False]) -def test_composition_ren(batches, L, use_bias): - io, nx, nv, mu, nu = 2, 3, 6, 0.5, 5.0 - model = CompositionREN(io, nx, nv, num_layers=L, mu=mu, nu=nu, use_bias=use_bias) - - carry = model.initialize_carry(batches) - u = torch.randn(batches, io) - new_carry, y = model(carry, u) - assert len(new_carry["rens"]) == L - assert tuple(y.shape) == (batches, io) - assert new_carry["dyn_in"] is None and new_carry["dyn_out"] is None - - with torch.no_grad(): - _, u_rec = model.inverse(carry, y) - assert rel_err(u_rec, u) < 1e-3 - - n = 300 - zc = {"rens": [torch.zeros(n, nx) for _ in range(L)], - "dyn_in": None, "dyn_out": None} - ua, ub = torch.randn(n, io), torch.randn(n, io) - with torch.no_grad(): - _, ya = model(zc, ua) - _, yb = model(zc, ub) - gain = (ya - yb).norm(dim=1) / (ua - ub).norm(dim=1) - assert gain.min().item() >= mu - 1e-3 - assert gain.max().item() <= nu + 1e-3 - assert model.get_bounds() == (mu, nu) - - (y ** 2).sum().backward() - grads_finite(model) - - -@pytest.mark.parametrize("at_input,at_output", - [(True, False), (False, True), (True, True)]) -def test_composition_ren_dyn_orth_noncausal_inverse(at_input, at_output): - io, nx, L = 2, 3, 2 - model = CompositionREN(io, nx, 6, num_layers=L, mu=0.5, nu=5.0, - dyn_orth_at_input=at_input, dyn_orth_at_output=at_output, - dyn_state_multiplier=4) - carry = model.initialize_carry(5) - assert (carry["dyn_in"] is not None) == at_input - assert (carry["dyn_out"] is not None) == at_output - - u = torch.randn(5, io) - new_carry, y = model(carry, u) - assert tuple(y.shape) == (5, io) - if at_input: - assert tuple(new_carry["dyn_in"].shape) == (5, 4 * nx) - - with torch.no_grad(): - _, u_rec = model.inverse_noncausal(carry, new_carry, y) - assert rel_err(u_rec, u) < 1e-3 - - -def test_composition_ren_ignore_inverse_with_dyn_in(): - model = CompositionREN(2, 3, 6, num_layers=2, mu=0.5, nu=5.0, - dyn_orth_at_input=True, dyn_state_multiplier=4) - carry = model.initialize_carry(5) - u = torch.randn(5, 2) - _, y = model(carry, u) - with torch.no_grad(): - _, sig_rec = model.inverse(carry, y) - _, dyn_out = model.dyn_in(carry["dyn_in"], u) - assert rel_err(sig_rec, dyn_out) < 1e-3 - - -@pytest.mark.parametrize("kw", [dict(num_layers=0), dict(mu=2.0, nu=1.0), - dict(mu=1.0, nu=1.0)]) -def test_composition_ren_invalid_args(kw): - args = dict(num_layers=2, mu=0.5, nu=5.0) - args.update(kw) - with pytest.raises(ValueError): - CompositionREN(2, 3, 6, **args) - - -@pytest.mark.parametrize("case", golden.cases("ren_composition"), ids=golden.case_id) -@pytest.mark.parametrize("explicit", [False, True], ids=["forward", "explicit"]) -def test_ren_composition_matches_golden(case, explicit): - check_golden("ren_composition", case, explicit) diff --git a/tests/torch_utils.py b/tests/torch_utils.py deleted file mode 100644 index 583516f..0000000 --- a/tests/torch_utils.py +++ /dev/null @@ -1,219 +0,0 @@ -# This file is a part of the RobustNeuralNetworks package. License is MIT: https://github.com/acfr/RobustNeuralNetworks/blob/main/LICENSE - -"""Shared helpers for the PyTorch tests.""" - -import numpy as np -import pytest -import torch -import torch.nn as nn - -import golden - - -def empirical_gains(f, n, size): - ua, ub = torch.randn(n, size), torch.randn(n, size) - with torch.no_grad(): - return (f(ua) - f(ub)).norm(dim=1) / (ua - ub).norm(dim=1) - - -def grads_finite(model): - grads = [p.grad for p in model.parameters() if p.grad is not None] - assert grads - assert all(bool(torch.isfinite(g).all()) for g in grads) - - - -def rel_err(a, b): - return ((a - b).norm() / b.norm()).item() - - -# --------------------------------------------------------------------------- -# Golden-file support (see golden.py) -# --------------------------------------------------------------------------- -_MODULE_ACT = {"relu": nn.ReLU, "tanh": nn.Tanh, "sigmoid": nn.Sigmoid, "identity": nn.Identity} -_FN_ACT = {"relu": torch.relu, "tanh": torch.tanh, "sigmoid": torch.sigmoid, - "identity": lambda x: x} - - -def build_model(network, config): - """Construct the torch module for a golden-file `config` (JAX argument names).""" - from robustnn.bilipnet_torch import BiLipNet - from robustnn.dyn_orthogonal_torch import DynUnitary - from robustnn.monlipnet_torch import MonLipNet - from robustnn.orthogonal_torch import Unitary - from robustnn.plnet_torch import PLNet - from robustnn.ren_composition_torch import CompositionREN - from robustnn.ren_torch import BiLipschitzREN - - c = config - if network == "unitary": - return Unitary(c["input_size"], c["input_size"], bias=c.get("use_bias", True)) - if network in ("monlipnet", "bilipnet"): - kw = dict(features=c["input_size"], unit_features=list(c["units"]), - mu=c.get("mu"), nu=c.get("nu"), tau=c.get("tau"), - is_mu_fixed=c.get("is_mu_fixed", False), - is_nu_fixed=c.get("is_nu_fixed", False), - is_tau_fixed=c.get("is_tau_fixed", False)) - if "act_fn" in c: - kw["act"] = _MODULE_ACT[c["act_fn"]]() - if network == "bilipnet": - return BiLipNet(depth=c.get("depth", 2), **kw) - return MonLipNet(**kw) - if network == "plnet": - op = c.get("optimal_point") - return PLNet( - BiLipBlock=build_model("bilipnet", c["BiLipBlock"]), - add_constant=c.get("add_constant", False), - optimal_point=None if op is None else torch.tensor(op, dtype=torch.float32), - ) - if network == "dyn_unitary": - return DynUnitary(c["input_size"], c["state_size"], bias=c.get("use_bias", True)) - if network == "bilipschitz_ren": - act = _MODULE_ACT[c.get("activation", "relu")]() - return BiLipschitzREN(c["input_size"], c["state_size"], c["features"], - mu=c["mu"], nu=c["nu"], activation=act) - if network == "ren_composition": - return CompositionREN( - c["input_size"], c["state_size"], c["features"], c["num_layers"], - mu=c["mu"], nu=c["nu"], activation=_FN_ACT[c.get("activation", "relu")], - use_bias=c.get("use_bias", True), - dyn_orth_at_input=c.get("dyn_orth_at_input", False), - dyn_orth_at_output=c.get("dyn_orth_at_output", False), - dyn_state_multiplier=c.get("dyn_state_multiplier", 50), - ) - raise KeyError(f"no PyTorch implementation of '{network}'") - - -# --- JAX/Flax parameter layout -> torch state_dict --------------------------- -def _unitary_sd(p, prefix=""): - sd = {f"{prefix}weight": p["W"], f"{prefix}alpha": p["a"]} - if "b" in p: - sd[f"{prefix}bias"] = p["b"] - return sd - - -def _monlip_sd(p, prefix=""): - sd = {f"{prefix}Fq": p["Fq"].T, f"{prefix}fq": p["fq"], f"{prefix}by": p["by"]} - for k in range(sum(1 for name in p if name.startswith("Fab"))): - sd[f"{prefix}Fr.{k}"] = p[f"Fab{k}"].T - sd[f"{prefix}fr.{k}"] = p[f"fab{k}"] - sd[f"{prefix}bs.{k}"] = p[f"b{k}"] - return sd - - -def _bilip_sd(p, prefix=""): - sd = {} - for name, sub in p.items(): - kind, k = name.split("_") - if kind == "uni": - sd.update(_unitary_sd(sub, f"{prefix}orth_layers.{k}.")) - else: - sd.update(_monlip_sd(sub, f"{prefix}mon_layers.{k}.")) - return sd - - -def _flat_sd(p, prefix=""): - return {f"{prefix}{k}": v for k, v in p.items()} - - -def _ren_sd(p, prefix=""): - # The JAX BiLipschitzREN carries a `D22` direct parameter that its QSR - # parameterisation never reads (D22 is derived from Q, S, R); torch has none. - return _flat_sd({k: v for k, v in p.items() if k != "D22"}, prefix) - - -def to_state_dict(network, params): - """Translate golden (JAX-layout) `params` to a torch `state_dict`.""" - if network == "unitary": - return _unitary_sd(params) - if network == "monlipnet": - return _monlip_sd(params) - if network == "bilipnet": - return _bilip_sd(params) - if network == "plnet": - sd = _bilip_sd(params["BiLipBlock"], "bln.") - if "c" in params: - sd["bias"] = params["c"] - return sd - if network == "dyn_unitary": - return _flat_sd(params) - if network == "bilipschitz_ren": - return _ren_sd(params) - if network == "ren_composition": - sd = {} - for name, sub in params.items(): - if name in ("input_orth", "output_orth"): - sd.update(_unitary_sd(sub, f"{name}.")) - elif name.startswith("orths_"): - sd.update(_unitary_sd(sub, f"orths.{name.split('_')[1]}.")) - elif name.startswith("rens_"): - sd.update(_ren_sd(sub, f"rens.{name.split('_')[1]}.")) - else: # dyn_in / dyn_out - sd.update(_flat_sd(sub, f"{name}.")) - return sd - raise KeyError(network) - - -# Bounds are constructor arguments (buffers/parameters named mu, nu, tau): they -# come from the config, not from the golden parameters. -_CONFIG_KEYS = ("mu", "nu", "tau") - - -def load_params(network, model, params): - sd = {k: torch.tensor(np.asarray(v), dtype=torch.float32) - for k, v in to_state_dict(network, params).items()} - missing, unexpected = model.load_state_dict(sd, strict=False) - missing = [k for k in missing if k.split(".")[-1] not in _CONFIG_KEYS] - assert not unexpected, f"unexpected keys: {unexpected}" - assert not missing, f"keys not set from golden params: {missing}" - - -def _t(a): - return None if a is None else torch.tensor(np.asarray(a), dtype=torch.float32) - - -def _n(t): - return None if t is None else t.detach().numpy() - - -def run_model(network, model, inputs, explicit=False): - """Evaluate `model` on golden `inputs` (forward pass or explicit path). - - Returns None if the explicit path is not available for this network. - """ - with torch.no_grad(): - if network in ("unitary", "monlipnet", "bilipnet", "plnet"): - if explicit: - x = np.asarray(inputs["x"]) - return {"y": np.asarray(model.explicit_call(x, model.direct_to_explicit()))} - return {"y": _n(model(_t(inputs["x"])))} - - if network == "ren_composition": - if explicit: - return None - c = inputs["carry"] - carry = {"rens": [_t(r) for r in c["rens"]], - "dyn_in": _t(c["dyn_in"]), "dyn_out": _t(c["dyn_out"])} - new, y = model(carry, _t(inputs["u"])) - return {"carry": {"rens": np.stack([_n(r) for r in new["rens"]]), - "dyn_in": _n(new["dyn_in"]), "dyn_out": _n(new["dyn_out"])}, - "y": _n(y)} - - x, u = _t(inputs["state"]), _t(inputs["u"]) - if explicit: - if network != "bilipschitz_ren": - return None - xn, y = model.explicit_call(x, u, model.direct_to_explicit()) - else: - xn, y = model(x, u) - return {"next_state": _n(xn), "y": _n(y)} - - -def check_golden(network, case, explicit=False): - """Check a golden case against the torch model (forward or explicit path).""" - model = build_model(network, case["config"]) - load_params(network, model, case["params"]) - out = run_model(network, model, case["inputs"], explicit=explicit) - if out is None: - pytest.skip(f"no explicit path for torch '{network}'") - golden.assert_close(out, case["outputs"])