diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb874366..d8b38d36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,18 @@ name: CI on: - # Pushes to main or master and pull requests, each run once. + # Pushes to the main branches and pull requests, each run once. push: - branches: [main, master] + branches: [main, master, integration] pull_request: + # Manual runs choose the conformance cells: the CI profile or the full matrix (to record + # the expected failures of the runner's hardware; see tests/python/conformance/known.py). workflow_dispatch: + inputs: + conformance: + description: Conformance cells to run + type: choice + options: [ci, full] + default: ci permissions: contents: read concurrency: @@ -18,12 +26,13 @@ jobs: steps: # Check out repository sources for static analysis and formatting checks. - uses: actions/checkout@v7.0.1 - # Set up Python 3.10 (the minimum supported version and basedpyright target). + # Set up Python 3.12: basedpyright targets 3.10 (pyproject.toml) but needs the type stubs + # of current NumPy and JAX, which Python 3.10 cannot install. - uses: actions/setup-python@v7.1.0 with: - python-version: '3.10' + python-version: '3.12' # Install pinned linters/formatters (ruff, clang-format, clang-tidy, basedpyright, lizard) and scientific stubs. - - run: python -m pip install -r requirements-dev.txt numpy scipy mpmath scikit-learn jax + - run: python -m pip install -r requirements-dev.txt numpy scipy mpmath matplotlib scikit-learn jax # Install CPU PyTorch so basedpyright can resolve torch types across the package and tests. - run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu # Run ruff, clang-format, basedpyright, version consistency, complexity ceiling, and clang-tidy. @@ -118,6 +127,9 @@ jobs: OPENBLAS_NUM_THREADS: '1' OMP_NUM_THREADS: '1' CXX: clang++-18 + CONFORMANCE: ${{ inputs.conformance || 'ci' }} + # Each conformance cell with failures appends a record here, uploaded when a step fails. + SDCA_CONFORMANCE_RECORD: ${{ github.workspace }}/build/conformance steps: # Check out repository sources for Python, PyTorch, JAX, and scikit-learn test suites. - uses: actions/checkout@v7.0.1 @@ -135,14 +147,24 @@ jobs: - run: python tools/check_extension_builds.py --compiler g++ --compiler clang++-18 # Build the native C ABI shared library against the external BLAS backend. - run: python python/build.py --blas-backend EXTERNAL - # Run all parallel Python test groups (core, torch, jax), skipping multi-minute stress tests. - - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group all -m 'not long' + # Run all parallel Python test groups (core, torch, jax), skipping multi-minute stress tests, + # with the CI profile of the conformance matrix unless a manual run asks for the full one. + - run: >- + PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python + --group all -m 'not long' --conformance "$CONFORMANCE" # Run thread-scaling and timing tests that require an unloaded machine serially. - run: PYTHONPATH=python python -m pytest -p no:cacheprovider tests/python --group all -m serial # Compile and test the JIT-built PyTorch CPU C++ extension (SDCA_CPU_EXTENSION=on) with GCC. - run: >- CXX=g++ SDCA_CPU_EXTENSION=on PYTHONPATH=python python -m pytest -p no:cacheprovider tests/python/test_softmax_limits.py::SoftmaxLimitTests tests/python/test_torch.py::TorchTests + # The conformance records of a failed run, for review against the known-failure manifest. + - uses: actions/upload-artifact@v7.0.2 + if: failure() + with: + name: conformance-linux-${{ github.sha }} + path: build/conformance + if-no-files-found: ignore python-versions: name: Python ${{ matrix.python }} (core) @@ -151,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.11', '3.13'] + python: ['3.11', '3.13', '3.14'] env: OPENBLAS_NUM_THREADS: '1' OMP_NUM_THREADS: '1' @@ -167,11 +189,14 @@ jobs: # Build the native C ABI library with the built-in BLAS, as in the wheels. - run: python python/build.py --blas-backend BUILTIN # Run the core test group in parallel, skipping multi-minute stress tests. - - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core -m 'not long' + - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core -m 'not long' --conformance ci macos: name: macOS (arm64) runs-on: macos-15 + env: + CONFORMANCE: ${{ inputs.conformance || 'ci' }} + SDCA_CONFORMANCE_RECORD: ${{ github.workspace }}/build/conformance steps: # Check out repository sources to test Apple Silicon (arm64), Apple Clang, and Accelerate BLAS. - uses: actions/checkout@v7.0.1 @@ -192,11 +217,16 @@ jobs: # Build the Python C ABI library linked against macOS Accelerate. - run: CXX=c++ python python/build.py --blas-backend EXTERNAL # Run the core NumPy and scikit-learn test suite in parallel across workers. - - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core - # Run the serial thread-scaling tests on macOS arm64. - - run: PYTHONPATH=python python -m pytest -p no:cacheprovider tests/python --group core -m serial + - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core --conformance "$CONFORMANCE" # Run the PyTorch CPU test suite (including the Apple Clang CPU extension) in parallel. - - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group torch -m 'not long' + - run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group torch -m 'not long' --conformance "$CONFORMANCE" + # The conformance records of a failed run, for review against the known-failure manifest. + - uses: actions/upload-artifact@v7.0.2 + if: failure() + with: + name: conformance-macos-${{ github.sha }} + path: build/conformance + if-no-files-found: ignore rust: name: Rust @@ -279,4 +309,4 @@ jobs: py="$RUNNER_TEMP/env/bin/python" src="$RUNNER_TEMP/sdist" SDCA_BLAS_BACKEND=BUILTIN "$py" -m pip install dist/*.tar.gz pytest scipy mpmath scikit-learn mkdir "$src" && tar -xzf dist/*.tar.gz -C "$src" --strip-components=1 - cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed + cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed --conformance ci diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 306d8458..dd2042ac 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -2,9 +2,8 @@ name: Packages # Builds the sdist, the wheels and the crate archive. A manual run can upload # the sdist and wheels to TestPyPI; a v* tag uploads them to PyPI, then the # crate to crates.io. Uploads use Trusted Publishing in the GitHub environments -# testpypi, pypi and crates-io; the one-time setup is in CONTRIBUTING.md -# (Releases). Pull requests only build and test the artifacts: every upload -# job skips them. +# testpypi, pypi and crates-io. Pull requests only build and test the +# artifacts: every upload job skips them. on: pull_request: workflow_dispatch: @@ -27,24 +26,25 @@ jobs: runs-on: ubuntu-24.04 steps: # Check out repository sources to build and validate the release source distribution. - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Set up Python 3.12 for building and testing the sdist. - uses: actions/setup-python@v7.1.0 with: python-version: '3.12' - # On tag pushes, verify that the git tag v matches the version in pyproject.toml. + # On tag pushes, verify that the git tag is v and that every version copy agrees. - if: github.ref_type == 'tag' - name: Check that the tag is v + name: Check that the tag is v env: TAG: ${{ github.ref_name }} - run: >- - python -c "import os, sys, tomllib; - v = 'v' + tomllib.load(open('pyproject.toml', 'rb'))['project']['version']; - sys.exit(None if os.environ['TAG'] == v else 'tag ' + os.environ['TAG'] + ' is not ' + v)" + run: | + test "$TAG" = "v$(cat VERSION)" || { echo "tag $TAG is not v$(cat VERSION)"; exit 1; } + python -c "import runpy; runpy.run_path('tools/quality.py')['check_versions']()" # Install pinned PEP 517 build frontend and Twine metadata checker. - run: python -m pip install build==1.6.1 twine==7.0.0 # Build the source distribution archive in dist/. - run: python -m build --sdist + - name: Check shared CPU sources in the sdist + run: python tools/check_install_archives.py --source . dist/*.tar.gz # Verify sdist metadata and long_description rendering for PyPI. - run: python -m twine check --strict dist/* # Install the sdist into an isolated venv outside the repo and run the core test suite. @@ -54,7 +54,7 @@ jobs: py="$RUNNER_TEMP/env/bin/python" src="$RUNNER_TEMP/sdist" SDCA_BLAS_BACKEND=BUILTIN "$py" -m pip install dist/*.tar.gz pytest scipy mpmath scikit-learn mkdir "$src" && tar -xzf dist/*.tar.gz -C "$src" --strip-components=1 - cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed + cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed --conformance ci # Verify that the C++ sources shipped inside the sdist can JIT-compile the PyTorch CPU extension. - name: Build the CPU extension from the sdist sources run: | @@ -78,9 +78,33 @@ jobs: runs-on: ${{ matrix.os }} steps: # Check out repository sources to build and test portable binary wheels. - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@v7.1.0 + with: + python-version: '3.12' # Build py3-none platform wheels with built-in BLAS, repair with auditwheel/delocate, and run core tests. - uses: pypa/cibuildwheel@v4.3.5 + - name: Check shared CPU headers in the wheels + run: python tools/check_install_archives.py --source . wheelhouse/*.whl + # Install only each documented extra into a fresh environment and run README examples. + - name: Fresh wheel installs and literal README examples + run: | + python -m pip install uv + python tools/readme_smoke.py --wheel wheelhouse/libsdca-*.whl --output "$RUNNER_TEMP/readme-smoke" --cleanup + - name: Save fresh-install records + if: always() + uses: actions/upload-artifact@v7.0.2 + with: + name: readme-smoke-${{ matrix.os }} + path: | + ${{ runner.temp }}/readme-smoke/results.json + ${{ runner.temp }}/readme-smoke/*/commands.log + ${{ runner.temp }}/readme-smoke/*/packages.txt + ${{ runner.temp }}/readme-smoke/*/example.py + ${{ runner.temp }}/readme-smoke/*/precision.py + ${{ runner.temp }}/readme-smoke/*/precision.json + ${{ runner.temp }}/readme-smoke/*/cpu-dispatch.py + ${{ runner.temp }}/readme-smoke/*/cpu-dispatch.json # Validate wheel metadata and README formatting with Twine. - run: pipx run --spec twine==7.0.0 twine check --strict wheelhouse/* # Upload the platform wheel artifact for the TestPyPI / PyPI publish jobs. @@ -94,7 +118,7 @@ jobs: runs-on: ubuntu-24.04 steps: # Check out repository sources to package and test the standalone Rust crate archive. - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Install stable Rust toolchain to package and test the crate. - run: rustup toolchain install stable --profile minimal && rustup default stable # Vendor the C++ headers and C ABI sources into rust/native/ and build the .crate archive. @@ -123,13 +147,13 @@ jobs: id-token: write steps: # Download the verified sdist and platform wheels built by earlier jobs. - - uses: actions/download-artifact@v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: dist-* path: dist merge-multiple: true # Publish sdist and wheels to TestPyPI via OIDC Trusted Publishing. - - uses: pypa/gh-action-pypi-publish@v1.14.3 + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: repository-url: https://test.pypi.org/legacy/ @@ -145,13 +169,13 @@ jobs: id-token: write steps: # Download the verified sdist and platform wheels built by earlier jobs. - - uses: actions/download-artifact@v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: dist-* path: dist merge-multiple: true # Publish sdist and wheels to PyPI via OIDC Trusted Publishing. - - uses: pypa/gh-action-pypi-publish@v1.14.3 + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 crates-io: name: Upload (crates.io) @@ -165,18 +189,18 @@ jobs: id-token: write steps: # Check out the tagged sources the verified crate archive was built from. - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Install stable Rust to package and publish the crate. - run: rustup toolchain install stable --profile minimal && rustup default stable # Download the crate archive that the crate job tested outside the checkout. - - uses: actions/download-artifact@v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: crate path: verified # Repackage and check that the archive to publish is byte-identical to the tested one. - run: rust/package.sh --no-verify && cmp verified/*.crate rust/target/package/libsdca-*.crate # Exchange the GitHub OIDC token for a short-lived crates.io token, revoked after the job. - - uses: rust-lang/crates-io-auth-action@v1.0.6 + - uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 id: auth # Publish the crate to crates.io via Trusted Publishing. - run: cargo publish --allow-dirty diff --git a/.gitignore b/.gitignore index bf4cb589..c037730f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build* +!build_*.py !python/build.py !rust/build.rs matlab/*.mex* diff --git a/AGENTS.md b/AGENTS.md index 2ad03f64..287a6e3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,8 +7,9 @@ examples are in the [README](README.md#quick-start) and the linked guides. ```bash pip install libsdca # NumPy solver and losses (numpy>=1.23) -pip install 'libsdca[torch]' # PyTorch losses (torch>=2.1, installs ninja) -pip install 'libsdca[jax]' # JAX losses (jax>=0.4.35) +pip install 'libsdca[torch]' # PyTorch losses (torch>=2.1) +pip install 'libsdca[jax]' # JAX CPU losses (jax>=0.4.35) +pip install 'libsdca[jax-cuda]' # JAX CUDA losses (Linux, NVIDIA driver 580+) pip install 'libsdca[sklearn]' # SDCAClassifier (scikit-learn>=1.6) cargo add libsdca # Rust crate (Rust>=1.74 and a C++11 compiler) ``` @@ -17,28 +18,33 @@ cargo add libsdca # Rust crate (Rust>=1.74 and a C++11 compiler) - PyTorch and JAX compile small C++ extensions on first use when a compiler and Python headers are present; otherwise they fall back with a one-time warning ([TORCH.md](python/TORCH.md#build-tools)). +- Check the selected path: `libsdca.torch.explain_backend(cross_entropy, logits, target)` + returns `(backend, reason)` for the call `cross_entropy(logits, target)`. ## Which API to use | User goal | API | Guide | | --- | --- | --- | -| Replace `torch.nn.CrossEntropyLoss` (integer class labels only, no label smoothing) | `libsdca.CrossEntropyLoss`, `cross_entropy` | [TORCH.md](python/TORCH.md#replace-pytorch-cross-entropy) | +| Replace `torch.nn.CrossEntropyLoss` (labels or probabilities, weights, label smoothing) | `libsdca.torch.CrossEntropyLoss`, `cross_entropy` | [TORCH.md](python/TORCH.md#replace-pytorch-cross-entropy) | | Large-vocabulary head + cross-entropy without `(N, V)` logits | `LinearCrossEntropyLoss`, `linear_cross_entropy` | [TORCH.md](python/TORCH.md#linear-layer-and-cross-entropy-without-the-logits) | | Top-*k*, hinge and multilabel PyTorch losses | `TopKCrossEntropyLoss`, `TopKHingeLoss`, `MulticlassHingeLoss`, `MultilabelHingeLoss`, `MultilabelCrossEntropyLoss` | [TORCH.md](python/TORCH.md) | | Logit distillation in PyTorch | `DistillationLoss`, `distillation_loss` | [TORCH.md](python/TORCH.md#teacher-logits-and-distillation) | | Losses in JAX (`jit`, `grad`, `vmap`) | `libsdca.jax` | [JAX.md](python/JAX.md) | -| Linear or kernel SVM / softmax classifier (scikit-learn) | `libsdca.SDCAClassifier` | [python/README.md](python/README.md#classifier) | -| Same, with duality-gap history and warm starts | `libsdca.fit` | [python/README.md](python/README.md#training) | +| Linear or kernel SVM / softmax classifier (scikit-learn) | `libsdca.sklearn.SDCAClassifier` | [python/README.md](python/README.md#classifier) | +| Same, with duality-gap history and warm starts | `libsdca.fit(X, y, loss)` with loss objects such as `libsdca.TopKHingeBeta(k=2)` | [python/README.md](python/README.md#training) | +| Losses and gradients on NumPy arrays | `libsdca.loss_and_grad` | [python/README.md](python/README.md#losses-and-gradients) | | Rust, C++, C, MATLAB | crate `libsdca`, headers, `libsdca_c`, `matsdca_*` | [Rust](rust/README.md), [C++](docs/CPP.md), [C](docs/C.md), [MATLAB](docs/MATLAB.md) | +Loss names, options, defaults and errors are shared across languages: +[docs/API.md](docs/API.md). + ## Pitfalls - **Class axis:** `CrossEntropyLoss` takes classes on `dim=1` like PyTorch; every other loss defaults to `dim=-1`. Pass `dim=1` for `(N, C, H, W)` inputs to the others ([TORCH.md](python/TORCH.md#argument-parity)). -- **Label smoothing and probability targets:** use `distillation_loss` with - log-probabilities as the teacher - ([TORCH.md](python/TORCH.md#label-smoothing-and-probability-targets)). +- **Hyperparameters are keyword-only** and belong to their loss: `TopKHingeLoss(k=2)`, + `libsdca.TopKHingeBeta(k=2)`; passing a parameter another loss uses is an error. - **Distillation** takes raw logits for both inputs and applies no $T^2$ factor ([TORCH.md](python/TORCH.md#teacher-logits-and-distillation)). - **Memory:** CUDA losses can reuse the logits buffer for the gradient when nothing else @@ -46,11 +52,26 @@ cargo add libsdca # Rust crate (Rust>=1.74 and a C++11 compiler) - **Solver objective:** $\frac12\Vert W \Vert_2^2 + C\sum_i \ell_i$ with no intercept; standardize features; for $d \gg n$ pass a Gram matrix ([python/README.md](python/README.md#training), [API.md](docs/API.md#options)). -- **Convergence:** only `status == "solved"` certifies the duality gap +- **Convergence:** only `status == "solved"` guarantees the duality gap ([API.md](docs/API.md#results)). +- **Precision:** `precision=None` follows the storage dtype (16-bit inputs use `float32` + work); pass `precision="float64"` for double work ([API.md](docs/API.md#precision), + [known limits](docs/API.md#known-limits)). + +## Working in this repository + +| Path | Contents | +| --- | --- | +| `include/sdca/` | Header-only C++11 core (`solver.h`, `loss.h`, `prox.h`, `c_api.h`) | +| `src/c_api.cpp` | C ABI used by Python and Rust | +| `python/libsdca/` | Python package (NumPy, scikit-learn, PyTorch, JAX) | +| `rust/`, `matlab/`, `src/matlab/` | Rust crate and MATLAB/Octave gateways | +| `tests/`, `examples/` | Test suites and runnable examples | -## Further documentation +```bash +pip install -e '.[test-torch,test-jax]' +python -m pytest -n auto tests/python --group core # core, torch, jax or all +python tools/quality.py --skip-tidy +``` -- Loss formulas, options and environment variables: [docs/API.md](docs/API.md) -- Numerics and known limits: [docs/numerics.md](docs/numerics.md) -- Performance: [docs/performance.md](docs/performance.md) +[CONTRIBUTING.md](CONTRIBUTING.md) lists the full checks; `VERSION` holds the version. diff --git a/CHANGELOG.md b/CHANGELOG.md index 49230f5d..09679c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,87 @@ # Changelog -## 0.3.1 (2026-11-12) +## 0.4.0 (2026-11-16) -- **CUDA hinges:** For `C <= 128`, the fused multiclass, smooth and top-*k* hinge kernels pack - several rows per CTA, as the top-*k* cross-entropy kernels already do. Forward plus backward - on an A100 (`bfloat16`): `multiclass_hinge` at `262144 × 10` takes `0.20` ms (was `0.52`), - the smooth hinge `0.27` ms (was `2.21`), and `topk_hinge_alpha` ($k = 5$) `0.17` ms (was - `2.76`). Values and gradients are unchanged, and wider rows keep the existing kernels. -- **Name and description:** The project is described everywhere by the technical report's - title, *libsdca: Accurate and Fast Classification Losses for SDCA and Neural Networks* - (README, PyPI, crates.io, module docs), and report links go to - . -- **Citation:** `CITATION.cff` and the README cite the technical report and the TPAMI 2018 - paper; softmax collapse is credited to Prieto et al. (2025). -- **Docs:** Fixed GitHub math rendering, wide tables and line wrapping. -- **Packaging:** The crate is published through crates.io Trusted Publishing, and - `tools/quality.py` checks the `CITATION.cff` version and date and the release-pinned links - of the PyPI and crates.io pages. +- **JAX on CUDA:** `libsdca.jax` runs the same fused Triton kernels as PyTorch for every + loss (Linux wheels include the handler), falling back to `jax.numpy` with a warning. +- **Shared CPU distillation:** PyTorch and JAX use the same fused CPU row kernels for + distillation in all four dtypes; JAX proximal distillation uses the shared native solver. + An overflowing native proximal-distillation loss now returns `+inf` with a finite + gradient instead of `nan`. +- **`float32` work by default:** NumPy, scikit-learn, Rust and the C++ loss helpers default + to `float32` work for `float32` inputs; `precision="float64"` keeps double work. Proximal + cross-entropy and CUDA top-1 hinges use `float32` work for 16- and 32-bit inputs, promoting + rows that need double ([API.md](docs/API.md#precision)). +- **Hinge ties:** Nonsmooth top-1 hinges select the largest represented score before forming + the margin, so tiny score gaps no longer round to ties. +- **Aggregate tails:** CUDA distillation, proximal distillation and multilabel + cross-entropy, and the JAX array losses, keep sums of individually underflowing + exponentials. +- **Faster CUDA paths:** Adaptive distillation kernels and packed short hinge rows. +- **JAX CPU fusion:** `SDCA_JAX_CPU_FUSION=auto` times fused and fenced array programs once + per machine and caches the choice ([JAX.md](python/JAX.md#cpu-pass-fusion)). +- **Packaging:** JAX-only installs no longer need PyTorch; the `torch` extra works with + PyTorch 2.1; transient build failures no longer suppress retries for 24 hours. The + version is read from one `VERSION` file. +- **Clearer errors:** PyTorch losses other than `cross_entropy` raise `ValueError` inside + `torch.func` transforms, and all losses raise `ValueError` for tensors other than CPU or + CUDA (MPS failed with a `TypeError`). `gamma` and `alpha` reject booleans. Fallback + warnings point at the calling line. +- **Fixes:** `cross_entropy(..., backend="torch")` accepts nonfinite logits in ignored rows, + as the other backends do. A solve whose final evaluation throws ends as `failed` instead + of raising. The JAX CPU build falls back to portable flags when the native-ISA probe fails. + A cross-entropy coordinate step whose label probability is 1 to within `exp(-200)` no + longer fails a fit ("Entropy normalization did not converge"). + `float64` proximal cross-entropy compensates its row sums: on 4,097–32,768 classes, values + and gradients were up to `95×` the error bound and are now within it. +- **API redesign (breaking, every interface):** One vocabulary of losses, each taking only + its own parameters (`k`, `gamma`, `alpha`, `temperature`); a parameter of another loss is + an error, and string aliases (`topk_hinge`, `softmax`, `fp32`, ...) are gone. One target + kind per loss: class labels, label sets / boolean masks, teacher logits, and target + probabilities for training `cross_entropy` (the former `distillation` objective). + `eval_epoch = 0` evaluates only at the end everywhere; a new `seed` option sets the order + of the examples; a fit stopped by `max_epoch` reports `solved` when its last evaluation + meets `tol`; statuses are `solved`, `no_progress`, `max_epoch`, `failed`. + - **NumPy:** loss objects (`fit(X, y, libsdca.TopKHingeBeta(k=2))`, + `loss_and_grad(logits, target, loss)`, `coordinate_update`), `libsdca.Kernel(K)` and + `libsdca.Probabilities(P)` inputs, `num_threads` and `seed` on `fit`; 16-bit and + `float64` data with `precision="float32"` are cast; the distillation functions and the + `Solver` class are gone, and `libsdca` no longer re-exports the PyTorch names. + - **scikit-learn:** `libsdca.sklearn.SDCAClassifier(loss, *, C, k, gamma, tol, max_iter, + eval_epoch, shrinking, kernel="linear"|"precomputed", precision, n_jobs, random_state, + warm_start)` passes `check_estimator`. + - **PyTorch and JAX:** one function and module per loss with keyword-only, validated + options; `SDCALoss`/`sdca_loss` and the alpha/beta hinge aliases are gone + (`topk_hinge_loss(variant=...)`). Cross-entropy and linear cross-entropy take label + smoothing and probability targets, the single-label losses `weight` and + `ignore_index`, any integer label dtype; `torch.func` `grad`/`vmap`/`jacrev` work for + every loss; traced JAX calls return NaN rows instead of raising; `explain_backend` + takes the function or module and its arguments. + - **Rust:** `Loss` variants carry their parameters (`Loss::topk_hinge(2)`), a `Solver` + builder trains and takes coordinate steps, `Settings` hold the precision, `Threads` + and projection solver, `Input::Kernel` and `Targets` (labels, sets, probabilities, + teacher logits) are explicit, and `Model::history` returns `Evaluation` structs. + - **C ABI (version 6):** `sdca_loss_evaluate`, `sdca_dual_update`, `sdca_model_fit` and + `sdca_history_rows` over separate `sdca_loss`, `sdca_execution`, `sdca_targets`, + `sdca_fit_options` and `sdca_input` structs; every call validates before it writes, and + the fit checks warm starts and builds `W` itself. + - **C++:** objectives use the shared names with a `gamma` parameter (default `0`), outputs + take the number of classes, labels are 0-based without the 1-based guess, + `criteria.tol` and `criteria.seed`, a row `loss_and_gradient` taking the target, and + self-contained headers (`SDCA_LOG_*` macros, no `logging.cpp`, CBLAS names in + `sdca`). + - **MATLAB:** shared names only, 1-based labels, `precision` follows `X`, `num_classes`, + `num_threads` and `seed`; `matsdca_loss` evaluates the distillation losses. +- **Stale builds:** Loading a native library built in the source tree before its sources + changed warns, and the test suite refuses to run against it. +- **CI:** `pytest --conformance ci` runs a profile of the conformance matrix (4,362 of + 46,400 cells); the default still runs it all. +- **Docs and citation:** The README starts with the installation and shows the same two losses + in every interface; [TORCH.md](python/TORCH.md#benchmarks) lists CUDA benchmarks; + `CITATION.cff` prefers the TPAMI 2018 paper. +- **Known limits:** [API.md](docs/API.md#known-limits) lists the recorded `float64` + cancellation errors (below `5e-17` absolute) and the behavior of finite rows at + `±finfo.max`. ## 0.3.0 (2026-11-09) @@ -28,11 +93,10 @@ gateways sharing one set of objective, option and status names ([API.md](docs/API.md)). - **Numerical stability:** Losses and gradients preserve tiny probability tails down to the subnormal limit (avoiding softmax collapse), and distillation gradients are exact zero at - equal logits ([numerics](docs/numerics.md), - [known limits](docs/numerics.md#aggregate-tail-limitations)). + equal logits ([known limits](docs/API.md#known-limits)). - **Fused CUDA and CPU kernels:** On an A100 (`8192 × 32000` and `8192 × 128256`), cross-entropy takes `0.93–0.98×` `torch.compile` time (`0.33–0.41×` eager), top-5 - cross-entropy `0.23–0.26×`, and distillation `0.62–0.81×` ([performance](docs/performance.md)). + cross-entropy `0.23–0.26×`, and distillation `0.62–0.81×`. - **Certified convergence:** `status == "solved"` requires the duality gap of the returned weights and the feasible projection of the returned duals, evaluated in `double` with a first-order rounding estimate, to meet @@ -78,11 +142,11 @@ - **Single rounding:** 16-bit gradients are scaled by the upstream cotangent and reduction factor in working precision before a single rounding to the output dtype, and hinge / top-*k* target gradients are formed from the unrounded competitor sum. -- **Double iterate for `float32` fits (`precision="float64"` default):** Keeps internal +- **Optional double iterate for `float32` fits (`precision="float64"`):** Keeps internal solver weights and duals in `double` (`8(d + n)m` bytes) and rounds once at each evaluation so `float32` fits reach `float64` gap tolerances down to `float32` output rounding (`3.5e-7` on `digits`; pass `precision="float32"` for pure `float32` speed and - memory; [numerics.md](docs/numerics.md#what-each-precision-guarantees)). + memory). - **Compensated reductions & hardened projections:** Compensated sums in multilabel cross-entropy and distillation, warm-start residual preservation at large score offsets, and checked `float64` / extended-precision `multilabel_hinge` projections diff --git a/CITATION.cff b/CITATION.cff index 6a4cd83c..8b15f97e 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,7 +1,7 @@ cff-version: 1.2.0 message: >- - If libsdca is useful in your research, please cite the paper for the losses you use - and the technical report for the library. + If libsdca is useful in your research, please cite the TPAMI paper; the earlier papers + introduce the top-k losses, and the technical report describes the library. type: software title: "libsdca: Accurate and Fast Classification Losses for SDCA and Neural Networks" abstract: >- @@ -10,8 +10,8 @@ abstract: >- authors: - family-names: Lapin given-names: Max -version: 0.3.1 -date-released: 2026-11-12 +version: 0.4.0 +date-released: 2026-11-16 license: MIT url: https://mlapin.github.io/libsdca/ repository-code: https://github.com/mlapin/libsdca @@ -25,14 +25,21 @@ keywords: - pytorch - jax preferred-citation: - type: report - title: "libsdca: Accurate and Fast Classification Losses for SDCA and Neural Networks" + type: article + title: Analysis and Optimization of Loss Functions for Multiclass, Top-k, and Multilabel Classification authors: - family-names: Lapin - given-names: Max - year: 2026 - month: 11 - url: https://mlapin.github.io/libsdca/report/ + given-names: Maksim + - family-names: Hein + given-names: Matthias + - family-names: Schiele + given-names: Bernt + journal: IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI) + volume: 40 + issue: 7 + start: 1533 + end: 1554 + year: 2018 references: - type: conference-paper title: Top-k Multiclass SVM @@ -56,18 +63,11 @@ references: given-names: Bernt collection-title: IEEE Conference on Computer Vision and Pattern Recognition (CVPR) year: 2016 - - type: article - title: Analysis and Optimization of Loss Functions for Multiclass, Top-k, and Multilabel Classification + - type: report + title: "libsdca: Accurate and Fast Classification Losses for SDCA and Neural Networks" authors: - family-names: Lapin - given-names: Maksim - - family-names: Hein - given-names: Matthias - - family-names: Schiele - given-names: Bernt - journal: IEEE Transactions on Pattern Analysis and Machine Intelligence - volume: 40 - issue: 7 - start: 1533 - end: 1554 - year: 2018 + given-names: Max + year: 2026 + month: 11 + url: https://mlapin.github.io/libsdca/report/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dbb7cd1..2af06a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,7 @@ cmake_minimum_required(VERSION 3.12) -project(libsdca) - -set(libsdca_MAJOR_VERSION 0) -set(libsdca_MINOR_VERSION 3) -set(libsdca_PATCH_VERSION 1) -set(libsdca_VERSION - ${libsdca_MAJOR_VERSION}.${libsdca_MINOR_VERSION}.${libsdca_PATCH_VERSION} - ) +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" libsdca_VERSION LIMIT_COUNT 1) +project(libsdca VERSION ${libsdca_VERSION}) +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS VERSION) add_definitions(-DSDCA_VERSION="${libsdca_VERSION}") # Whether to use @@ -231,6 +226,7 @@ set(libsdca_PROX_SOURCES set(libsdca_SOLVER_SOURCES ${libsdca_INCLUDE_PATH}/loss.h + ${libsdca_INCLUDE_PATH}/loss_kind.h ${libsdca_INCLUDE_PATH}/solver.h ${libsdca_INCLUDE_PATH}/solver/solverdef.h ${libsdca_INCLUDE_PATH}/solver/context.h @@ -248,12 +244,12 @@ set(libsdca_SOLVER_SOURCES ${libsdca_INCLUDE_PATH}/solver/eval/types.h ${libsdca_INCLUDE_PATH}/solver/eval.h ${libsdca_INCLUDE_PATH}/solver/objective/feasible_dual.h - ${libsdca_INCLUDE_PATH}/solver/objective/l2_entropy.h - ${libsdca_INCLUDE_PATH}/solver/objective/l2_entropy_topk.h - ${libsdca_INCLUDE_PATH}/solver/objective/l2_hinge_topk.h - ${libsdca_INCLUDE_PATH}/solver/objective/l2_multilabel_entropy.h + ${libsdca_INCLUDE_PATH}/solver/objective/l2_cross_entropy.h + ${libsdca_INCLUDE_PATH}/solver/objective/l2_topk_cross_entropy.h + ${libsdca_INCLUDE_PATH}/solver/objective/l2_topk_hinge_alpha.h + ${libsdca_INCLUDE_PATH}/solver/objective/l2_multilabel_cross_entropy.h ${libsdca_INCLUDE_PATH}/solver/objective/l2_multilabel_hinge.h - ${libsdca_INCLUDE_PATH}/solver/objective/l2_topk_hinge.h + ${libsdca_INCLUDE_PATH}/solver/objective/l2_topk_hinge_beta.h ${libsdca_INCLUDE_PATH}/solver/objective/objective_base.h ${libsdca_INCLUDE_PATH}/solver/objective.h ${libsdca_INCLUDE_PATH}/solver/reporting.h @@ -300,8 +296,7 @@ target_compile_features(libsdca INTERFACE cxx_std_11) target_include_directories(libsdca INTERFACE $ $ ${SDCA_BLAS_INCLUDE_DIRS}) -target_compile_definitions(libsdca INTERFACE ${SDCA_BLAS_DEFINITIONS} - SDCA_VERSION="${libsdca_VERSION}") +target_compile_definitions(libsdca INTERFACE ${SDCA_BLAS_DEFINITIONS}) if(USE_CHRONO) target_compile_definitions(libsdca INTERFACE SDCA_USE_CHRONO) else() @@ -322,7 +317,7 @@ if(SDCA_BUILD_C_API) target_include_directories(sdca_c INTERFACE $ $) # The shared library version follows SDCA_ABI_VERSION. - set_target_properties(sdca_c PROPERTIES OUTPUT_NAME sdca_c VERSION 5 SOVERSION 5 + set_target_properties(sdca_c PROPERTIES OUTPUT_NAME sdca_c VERSION 6 SOVERSION 6 CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) install(TARGETS sdca_c EXPORT libsdcaTargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76f327dd..3895cf87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,117 +1,105 @@ # Contributing -`libsdca` is a header-only C++11 library with a C ABI and Python (NumPy, -scikit-learn, PyTorch, JAX), Rust and MATLAB/Octave interfaces. This guide lists -the development workflow, repository layout, quality gates and coding rules. +`libsdca` is a header-only C++11 library with a C ABI and Python (NumPy, scikit-learn, +PyTorch, JAX), Rust and MATLAB/Octave interfaces. ## Development setup +On Debian/Ubuntu: + ```bash +sudo apt-get install -y build-essential python3-dev python3-venv cmake ninja-build clang \ + libomp-dev libblas-dev pkg-config python3 -m venv build/dev && source build/dev/bin/activate pip install -r requirements-dev.txt numpy scipy mpmath scikit-learn jax ninja pip install torch --index-url https://download.pytorch.org/whl/cpu pip install -e . # rerun after changing include/ or src/ +``` + +## Tests and checks + +```bash python -m pytest -n auto --dist worksteal tests/python --group all # or core, torch, jax -python -m pytest tests/python --group all -m serial +python -m pytest tests/python --group all -m serial # timing-sensitive, unloaded host cmake -S . -B build/cpp -DBUILD_TESTING=ON -DSDCA_BUILD_C_API=ON -DSDCA_WARNINGS_AS_ERRORS=ON -cmake --build build/cpp -j4 -ctest --test-dir build/cpp --output-on-failure --parallel 4 -python tools/quality.py +cmake --build build/cpp -j4 && ctest --test-dir build/cpp --output-on-failure --parallel 4 +python tools/quality.py # --fix formats first; --skip-tidy skips clang-tidy python tools/coverage.py --cpp-only cargo fmt --check --manifest-path rust/Cargo.toml cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings cargo test --manifest-path rust/Cargo.toml +OCTAVE_HOME= tests/matlab/run_octave_tests.sh ``` -- `tools/quality.py --fix` formats C++ and Python sources before running the checks. -- If you run `python/build.py`, delete `build/python` (or set `SDCA_LIBRARY_DIR`) when - switching back to `pip install -e .`, as `build/python/libsdca_c.so` takes precedence. -- Pytest markers (`tests/python/conftest.py`): `slow` and `long` tests are scheduled first - under `-n auto --dist worksteal` (CI deselects `long` with `-m "not long"`); `serial` - tests require an unloaded machine and run via `-m serial` without `-n`. -- For repeated local JAX test runs, set `JAX_COMPILATION_CACHE_DIR=` and +- **Markers:** `slow` and `long` tests are scheduled first under `--dist worksteal` (CI + deselects `long`); `serial` tests run without `-n` on an unloaded machine. +- **Conformance matrix:** `--conformance ci` runs about a tenth of the conformance cells + (every loss on the default backends, cross-entropy and the multiclass hinge on every + backend, dtype and precision), as CI does; the default runs the full matrix. A manual CI + run (*Run workflow*, `conformance: full`) runs the full matrix on the hosted runners and, + when cells fail, uploads their records (`SDCA_CONFORMANCE_RECORD`) as an artifact; a + reviewed failure set of other hardware is added to `known_failures/` as a variant. +- **CUDA:** On a machine with an NVIDIA GPU, install a CUDA build of PyTorch (and + of JAX); the same pytest command then runs the CUDA tests, which skip otherwise. + `python tests/python/release_numerics.py --device cuda --output ` also checks the + CUDA entropy and distillation losses against `mpmath` references and exits 1 on a failure. +- **Runtime:** `tests/python/test_work_counts.py` bounds the solver's epochs on seeded + problems, so convergence regressions fail without a timer. On a CUDA machine, + `python tools/perf_gate.py --output ` times the losses against `torch.compile` on + 32,000- and 128,256-class rows and exits 1 when a speedup falls below its floor or libsdca + needs more peak memory. +- **Installed package:** `python -m pytest tests/python --installed` tests the installed + `libsdca` instead of the source tree. +- **`python/build.py`** builds `build/python/libsdca_c.so`, which takes precedence over an + editable install; delete it (or set `SDCA_LIBRARY_DIR`) when switching back. +- **JAX:** For repeated runs, set `JAX_COMPILATION_CACHE_DIR=` and `JAX_PERSISTENT_CACHE_MIN_COMPILE_TIME_SECS=0`. +| Tool | Rule | +| --- | --- | +| `clang-format`, `clang-tidy` (v18) | `.clang-format` (Google style, C++11, 100 columns) and `.clang-tidy`, warnings as errors | +| `ruff`, `basedpyright` | `pyproject.toml`: `E`, `F`, `I`, `B`, `UP`, `C90`; standard mode, warnings as errors | +| `lizard` | Cyclomatic complexity $\le 20$, length $\le 100$ lines (`tools/complexity_baseline.txt` may only shrink) | +| `gcovr`, `coverage.py` | $\ge 90\%$ line coverage of `include/sdca`, `src/c_api.cpp` and `python/libsdca` | +| `rustfmt`, `clippy` | `-D warnings`, all public items documented | +| C++ compilers | `-pedantic -Werror -W -Wall -Wextra` under GCC and Clang | + ## Layout | Location | Contents | | --- | --- | -| `include/sdca/math/`, `include/sdca/prox/` | Compensated math, entropy roots, BLAS, projections and proximal operators | -| `include/sdca/solver/`, `include/sdca/loss.h` | Data views, contexts, objectives, dual updates, stopping criteria, evaluation and loss dispatch | +| `include/sdca/math/`, `include/sdca/prox/` | Compensated math, BLAS, projections and proximal operators | +| `include/sdca/solver/`, `include/sdca/loss.h` | Data views, objectives, dual updates, stopping criteria and losses | | `include/sdca/c_api.h`, `src/c_api.cpp` | Exception-safe C ABI used by Python and Rust | | `python/libsdca/_core.py`, `sklearn.py` | NumPy API and `SDCAClassifier` | -| `python/libsdca/torch.py`, `_cpu.py`, `_torch_*` | PyTorch autograd layers, tensor losses, and ATen/OpenMP CPU extension | -| `python/libsdca/_triton_*.py`, `_cuda_launch.py` | Fused Triton CUDA kernels and C++ `cuLaunchKernel` launcher | -| `python/libsdca/jax.py`, `_jax_*` | JAX losses and XLA FFI custom-call handler | -| `rust/`, `matlab/`, `src/matlab/` | Rust crate (`rust/consumer` tests BLAS links) and MATLAB/Octave gateways | -| `tests/`, `examples/`, `docs/`, `tools/` | C++/Python/MATLAB test suites, runnable examples, docs, and quality/coverage scripts | - -## Checks - -| Tool | Scope | Rule | -| --- | --- | --- | -| `clang-format` | `include/sdca`, `src`, `tests`, `tools`, `python/libsdca` | `.clang-format` (v18): Google style, C++11, 100 columns | -| `clang-tidy` | `src/c_api.cpp`, `tests/cpp/regression.cpp` (both BLAS backends) | `.clang-tidy` (v18): correctness and performance checks, warnings as errors | -| `lizard` | `include/sdca`, `src`, `python/libsdca` | Cyclomatic complexity $\le 20$, length $\le 100$ lines (`tools/complexity_baseline.txt` may only shrink) | -| `ruff` | `python`, `tests/python`, `examples/python`, `tools`, build scripts | `pyproject.toml`: `E`, `F`, `I`, `B`, `UP`, `C90` (complexity $\le 15$) | -| `basedpyright` | Python package, tests, examples, build scripts, tools | `pyproject.toml`: standard mode, warnings as errors, rule-scoped `# pyright: ignore[rule]` | -| `gcovr`, `coverage.py` | `include/sdca`, `src/c_api.cpp`, `python/libsdca/` | $\ge 90\%$ line coverage across C++ and Python host code (only `@triton.jit` device bodies excluded) | -| `rustfmt`, `clippy` | `rust/` | Standard style, `-D warnings`, all public items documented | -| C++ compilers | All C++ targets | `-pedantic -Werror -W -Wall -Wextra` under GCC and Clang | - -To combine GPU and CPU Python host coverage from the same source checkout: - -```bash -python tools/coverage.py --write-python-manifest build/gpu.coverage -PYTHONPATH=python COVERAGE_FILE=build/gpu.coverage SDCA_CUDA_LAUNCHER=on \ - python -m coverage run --source=libsdca -m pytest tests/python --group torch -PYTHONPATH=python COVERAGE_FILE=build/gpu.coverage SDCA_CUDA_LAUNCHER=off \ - python -m coverage run --append --source=libsdca -m pytest tests/python --group torch -python tools/coverage.py --python-data build/gpu.coverage -``` - -## Coding guidelines - -- **C++:** C++11, header-only, strict IEEE semantics (`-fno-fast-math -ffp-contract=off`), - no `new` (use `std::vector` or `detail::entropy_buffer`). Public functions validate inputs - and throw `std::invalid_argument` or `std::runtime_error`; `detail` helpers document their - preconditions. -- **Python:** Annotate all functions; keep `import libsdca` free of `torch`, `jax`, - `sklearn` and native-library loads; raise `ValueError` or `TypeError` at the API boundary - and never copy tensors across devices implicitly. +| `python/libsdca/torch.py`, `_cpu.py`, `_torch_*` | PyTorch layers, tensor losses and the CPU extension | +| `python/libsdca/_triton_*.py`, `_cuda_launch.py` | Triton CUDA kernels and the C++ launcher | +| `python/libsdca/jax.py`, `_jax_*` | JAX losses and XLA FFI handlers | +| `rust/`, `matlab/`, `src/matlab/` | Rust crate and MATLAB/Octave gateways | +| `tests/`, `examples/`, `docs/`, `tools/` | Test suites, examples, guides and developer scripts | + +## Guidelines + +- **C++:** C++11, header-only, IEEE semantics (`-fno-fast-math -ffp-contract=off`), no `new`. + Public functions validate inputs and throw `std::invalid_argument` or `std::runtime_error`. +- **Python:** Annotate all functions; keep `import libsdca` free of `torch`, `jax`, `sklearn` + and native-library loads; raise `ValueError` or `TypeError` at the API boundary; never copy + tensors across devices implicitly. - **Rust:** No `unsafe` outside FFI calls, each with a `// SAFETY:` comment. -- **Cross-language vocabulary:** Every new objective, option or status uses the same name - across C++, the C ABI, Python, Rust and MATLAB ([docs/API.md](docs/API.md)), pinned by - `tests/python/test_api_contracts.py` and `rust/tests/api.rs`. -- **Numerical changes:** Every fix includes a regression test against an independent - analytic, `mpmath`, or KKT reference. Refactorings of `include/sdca` must leave results - bitwise unchanged, and performance-sensitive changes must report before/after timings - ([docs/performance.md](docs/performance.md#where-libsdca-is-slower)). +- **Names:** A new objective, option or status uses the same name in every interface + ([docs/API.md](docs/API.md)), pinned by `tests/python/test_api_contracts.py` and + `rust/tests/api.rs`. +- **Numerical changes:** Every fix comes with a regression test against an independent + analytic, `mpmath` or KKT reference. Do not loosen a tolerance to make a test pass. ## Releases -The version string is checked across 11 files by `tools/quality.py`: `CMakeLists.txt`, -`pyproject.toml`, `setup.py`, `python/build.py`, `python/libsdca/__init__.py`, -`matlab/make.m`, `rust/Cargo.toml`, `include/sdca/c_api.h`, `docs/C.md`, `CITATION.cff` -(whose `date-released` must match the changelog date), and `CHANGELOG.md`. The release-pinned -GitHub links (`blob/v`, `--tag v`) in `pyproject.toml`, `python/README.md` -and `rust/README.md` must name the same version. - -The project is described everywhere by the technical report's title, *libsdca: Accurate and -Fast Classification Losses for SDCA and Neural Networks*. The PyPI summary, the crates.io -description, `CITATION.cff` and the README openings start from it and then name the losses by -their search terms (cross-entropy, SVM, softmax, distillation). - -`.github/workflows/packages.yml` builds and tests the sdist, `manylinux_2_28` `x86_64` and -macOS 11+ `arm64` wheels (`cibuildwheel`), and the `.crate` archive (`rust/package.sh`), -publishing to TestPyPI (manual dispatch) or, on `v*` tags, to PyPI and then crates.io via OIDC -Trusted Publishing: +```bash +python tools/bump_version.py 0.3.2 # VERSION and its copies; "## Unreleased" gets the date +python tools/quality.py --skip-tidy # checks that every copy agrees +git tag v0.3.2 && git push origin v0.3.2 +``` -1. In GitHub *Settings → Environments*, create `testpypi`, `pypi` and `crates-io` (restrict - `pypi` and `crates-io` to `v*` tags, and `pypi` to required reviewers: approving it releases - both, since the crates.io job waits for the PyPI upload). -2. On [TestPyPI](https://test.pypi.org/manage/account/publishing/) and - [PyPI](https://pypi.org/manage/account/publishing/), register `mlapin/libsdca`, workflow - `packages.yml`, and environment `testpypi` / `pypi` as a trusted publisher. -3. In the [crates.io](https://crates.io/crates/libsdca) crate settings, add a GitHub trusted - publisher for `mlapin/libsdca`, workflow `packages.yml`, and environment `crates-io`. +`VERSION` is read by CMake, `setup.py`, `pyproject.toml`, `python/build.py` and +`matlab/make.m`. On a `v*` tag, `.github/workflows/packages.yml` builds the sdist, the wheels +and the crate and publishes them to PyPI and crates.io with trusted publishing. diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 269e66db..a9e2fd3d 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -1,46 +1,62 @@ -# Dependencies and optional builds +# Dependencies and builds -`libsdca` requires a C++11 compiler. An external BLAS library is optional: a built-in -backend lets the solver and loss functions run with no third-party runtime dependencies. -CMake 3.12+ is required only for the CMake packages (`libsdca::libsdca` and -`libsdca::sdca_c`). +`libsdca` needs only a C++11 compiler. An external CBLAS is optional (a built-in backend is +included), and CMake 3.12+ is needed only for the CMake packages `libsdca::libsdca` and +`libsdca::sdca_c`. ## Tested configurations -| Component | Validated environments | +| Component | Environments | | --- | --- | -| C++ & C ABI | Ubuntu 22.04 / 24.04 (`x86_64`) with GCC 11.4 / 13 and Clang 18 (`BUILTIN` and `EXTERNAL` BLAS, C and C++ consumers, GCC ASan/UBSan, Clang TSan); macOS (`arm64`) with Apple Clang 21 (`BUILTIN` and Accelerate) | -| Python (CPU) | Python 3.10–3.14, NumPy 1.23–2.5, SciPy 1.10–1.18, scikit-learn 1.6–1.9, PyTorch 2.1–2.15 (CPU extension across AVX-512, AVX2 and generic/arm64), JAX 0.4.35–0.12 (with and without XLA FFI) | -| CUDA | NVIDIA A100-SXM4-80GB and H100 (PyTorch 2.14–2.15, Triton 3.8–3.9, both C++ launcher and Triton launch paths; JAX 0.12 with CUDA plugin) | -| Packaging | Portable `py3-none` wheels (`SDCA_BLAS_BACKEND=BUILTIN`) via `cibuildwheel` on `manylinux_2_28` `x86_64` and macOS 11+ `arm64`, sdist builds, and standalone `.crate` archives | -| Rust | Rust 1.74 (MSRV) and stable on Linux `x86_64` (`BUILTIN`, reference BLAS, OpenBLAS and BLIS, shared and static) | -| Octave | GNU Octave 10.3 (`mkoctfile` MEX gateways) | +| C++ and C ABI | Ubuntu 22.04 / 24.04 `x86_64` with GCC 11 and 13 and Clang 18 (built-in and external BLAS, ASan/UBSan, TSan); macOS `arm64` with Apple Clang (built-in BLAS and Accelerate) | +| Python | Python 3.10–3.14, NumPy 1.26–2.5, SciPy 1.15–1.18, scikit-learn 1.7–1.9, PyTorch 2.1–2.15, JAX 0.4.35–0.12 | +| CUDA | NVIDIA A100 with PyTorch 2.14–2.15, Triton 3.8–3.9 and JAX 0.12 (hosted CI has no GPU) | +| Packaging | `py3-none` wheels via `cibuildwheel` on `manylinux_2_28` `x86_64` and macOS 11+ `arm64`, sdist, `.crate` | +| Rust | Rust 1.74 (MSRV) and stable on Linux `x86_64` (built-in BLAS, reference BLAS, OpenBLAS, BLIS) | +| Octave | GNU Octave 10 (`mkoctfile`) | -- **PyTorch & JAX version floors:** Requires `python >= 3.10`, `numpy >= 1.23`, - `torch >= 2.1` (use `numpy < 2` with PyTorch 2.1 wheels), `jax >= 0.4.35`, and - `scikit-learn >= 1.6`. -- **CUDA fused kernels:** Require `torch >= 2.7` and `triton >= 3.3` (older versions fall - back to composed PyTorch tensor ops). The C++ `cuLaunchKernel` launcher supports Triton - 3.3–3.9 (`SDCA_CUDA_LAUNCHER` overrides selection; `libsdca.torch.explain_backend` - reports the active backend). +## Python -## C++ configuration +| Extra | Packages | Purpose | +| --- | --- | --- | +| *(default)* | `numpy>=1.23` | NumPy solver and losses | +| `sklearn` | `scikit-learn>=1.6` | `SDCAClassifier` | +| `torch` | `torch>=2.1`, `ninja`, `setuptools>=77,<81`, `ziglang>=0.13,<0.18` (Linux) | PyTorch losses and their C++ extensions | +| `jax` | `jax>=0.4.35` | JAX losses (`libsdca.jax`) | +| `jax-cuda` | `jax[cuda13]>=0.7.2`, `triton>=3.3` (Linux) | JAX losses on CUDA | +| `test`, `test-torch`, `test-jax` | `pytest`, `pytest-xdist`, `scipy`, `mpmath`, ... | Test suites (`--group core\|torch\|jax\|all`) | + +- **Versions:** PyTorch 2.1 wheels need `numpy<2`. The fused CUDA kernels need `triton>=3.3`, + which PyTorch 2.7 and later install (older versions use composed PyTorch operations); the C++ + launcher supports Triton 3.3–3.9. `libsdca.torch.explain_backend` reports the active path. +- **Runtime extensions:** The PyTorch CPU extension and CUDA launcher compile on first use + with `torch.utils.cpp_extension` (a C++17 compiler with OpenMP, Python headers, and CUDA + headers for the launcher). JAX compiles a small XLA FFI handler against + `jax.ffi.include_dir()`. Without build tools both fall back with a one-time warning. + +```bash +python -m pip install . # system CBLAS if found, else built-in +SDCA_BLAS_BACKEND=BUILTIN python -m pip install . # no external BLAS +SDCA_BLAS_BACKEND=EXTERNAL python -m pip install . # require an external CBLAS +SDCA_BLAS="-L/opt/blas/lib -lopenblas" python -m pip install . # explicit linker flags +SDCA_BUILD_NATIVE=OFF python -m pip install '.[torch]' # tensor losses only, no compiler +``` + +## C++ and CMake | CMake option | Default | Description | | --- | --- | --- | -| `SDCA_BLAS_BACKEND` | `AUTO` | `AUTO` (prefer external CBLAS, else built-in), `BUILTIN` (no external BLAS/OpenMP), or `EXTERNAL` (require external CBLAS) | -| `SDCA_BUILD_C_API` | `OFF` | Build and install the C ABI library `libsdca::sdca_c` (links `Threads`) | -| `SDCA_WARNINGS_AS_ERRORS` | `OFF` | Treat compiler warnings as errors (`-Werror`) | -| `BUILD_TESTING` | `OFF` | Build bundled GoogleTest and C++ test suites | -| `SDCA_BUILD_MATLAB` | `OFF` | Build MATLAB MEX gateways | -| `USE_CHRONO` | `ON` | Use `` wall clock (`OFF` selects POSIX `gettimeofday`) | -| `USE_SEQUENTIAL`, `USE_ILP64`, `USE_THREADING_SUFFIX` | `ON`, `ON`, `OFF` | Intel MKL threading layer, 64-bit integer ABI, and MEX binary suffix | - -Minimal installation and CMake usage: +| `SDCA_BLAS_BACKEND` | `AUTO` | `AUTO` (external CBLAS if found, else built-in), `BUILTIN`, or `EXTERNAL` | +| `SDCA_BUILD_C_API` | `OFF` | Build and install the C ABI library `libsdca::sdca_c` | +| `SDCA_BUILD_MATLAB` | `OFF` | Build the MATLAB MEX gateways | +| `BUILD_TESTING` | `OFF` | Build the bundled GoogleTest suites | +| `SDCA_WARNINGS_AS_ERRORS` | `OFF` | Add `-Werror` | +| `USE_CHRONO` | `ON` | `` wall clock (`OFF`: POSIX `gettimeofday`) | +| `USE_SEQUENTIAL`, `USE_ILP64`, `USE_THREADING_SUFFIX` | `ON`, `ON`, `OFF` | Intel MKL threading layer, 64-bit integers, MEX suffix | ```bash -cmake -S . -B build/minimal -DSDCA_BLAS_BACKEND=BUILTIN -DCMAKE_INSTALL_PREFIX=/your/prefix -cmake --build build/minimal --target install +cmake -S . -B build -DSDCA_BLAS_BACKEND=BUILTIN -DCMAKE_INSTALL_PREFIX="$PWD/build/install" +cmake --build build --target install ``` ```cmake @@ -48,44 +64,12 @@ find_package(libsdca CONFIG REQUIRED) target_link_libraries(my_program PRIVATE libsdca::libsdca) ``` -Include `sdca/utility/logging.cpp` in exactly one translation unit. Direct header users -without CMake default to `BLAS_BUILTIN` and `` (or define `BLAS_DEFAULT`, -`BLAS_DEFAULT_LOCAL_HEADER`, `BLAS_MKL`, `BLAS_ACCELERATE`, or `BLAS_MATLAB` and link the -corresponding CBLAS library). - -## Python configuration - -The PyTorch CPU extension and CUDA launcher compile on first use via -`torch.utils.cpp_extension` (requiring `ninja`, a C++17/C++20 compiler with OpenMP, Python -headers, and CUDA headers for the launcher); native JAX CPU losses likewise compile a small -XLA FFI handler against `jax.ffi.include_dir()` ([environment variables](docs/API.md#environment-variables)). -Without build tools, both fall back automatically with a one-time warning. - -```bash -# Default source install (uses system CBLAS if found, else built-in): -python -m pip install . - -# Force built-in BLAS (no external BLAS dependency): -SDCA_BLAS_BACKEND=BUILTIN python -m pip install . - -# Require external BLAS: -SDCA_BLAS_BACKEND=EXTERNAL python -m pip install . - -# Pure-Python wheel for PyTorch/JAX tensor losses only (no C++ compiler needed): -SDCA_BUILD_NATIVE=OFF python -m pip install '.[torch]' -``` - -| Optional extra | Packages | Purpose | -| --- | --- | --- | -| *(default)* | `numpy>=1.23` | NumPy solver and loss interface | -| `torch` | `torch>=2.1`, `ninja` | PyTorch loss layers and JIT extensions | -| `jax` | `jax>=0.4.35` | JAX loss functions (`libsdca.jax`) | -| `sklearn` | `scikit-learn>=1.6` | `SDCAClassifier` estimator | -| `test`, `test-torch`, `test-jax` | `pytest>=9`, `pytest-xdist>=3.6`, `scipy`, `mpmath`, ... | Test suites (`--group core\|torch\|jax\|all`) | -| `coverage` | `coverage[toml]>=7` | Python line-coverage instrumentation | +Without CMake, define one of `BLAS_BUILTIN` (default), `BLAS_DEFAULT`, +`BLAS_DEFAULT_LOCAL_HEADER`, `BLAS_MKL`, `BLAS_ACCELERATE` or `BLAS_MATLAB`, link the matching +CBLAS; the headers need no separate source file. -## Rust configuration +## Rust -The `libsdca` crate has zero Rust dependencies and requires Rust 1.74+. Its `build.rs` -compiles `src/c_api.cpp` with `CXX` (default `c++`) and the built-in BLAS, or links an -external CBLAS via `SDCA_BLAS` ([rust/README.md](rust/README.md#building)). +The crate has no Rust dependencies and needs Rust 1.74+. `build.rs` compiles `src/c_api.cpp` +with `CXX` (default `c++`) and the built-in BLAS, or links an external CBLAS given by +`SDCA_BLAS` ([rust/README.md](rust/README.md#building)). diff --git a/MANIFEST.in b/MANIFEST.in index 08a4ba3c..af865075 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,9 @@ recursive-include include *.h -include src/c_api.cpp src/tensor_losses.h -include sdca_build.py +include src/c_api.cpp src/c_api_internal.h src/tensor_losses.h +include sdca_build.py VERSION include python/README.md include python/TORCH.md python/JAX.md -include python/libsdca/_torch_entropy.cpp python/libsdca/_torch_*.h python/libsdca/_jax_ffi.cpp -include include/sdca/utility/logging.cpp +include python/libsdca/_cpu_*.h +include python/libsdca/_torch_entropy.cpp python/libsdca/_torch_*.h python/libsdca/_jax_ffi.cpp python/libsdca/_jax_cuda.cpp include CHANGELOG.md DEPENDENCIES.md AGENTS.md CITATION.cff -include tests/python/*.py +recursive-include tests/python *.py *.json diff --git a/README.md b/README.md index 8188b1e1..cc0ff39f 100644 --- a/README.md +++ b/README.md @@ -6,90 +6,97 @@ [![Crates.io](https://img.shields.io/crates/v/libsdca.svg)](https://crates.io/crates/libsdca) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -**libsdca** provides accurate and fast classification losses for SDCA and neural -networks. It implements multiclass, top-*k*, multilabel, and soft-target -classification losses in two forms: fused CPU and CUDA loss layers for PyTorch and -JAX (and NumPy functions), and stochastic dual coordinate ascent (SDCA) solvers for -linear and kernel models. It covers cross-entropy, the multiclass hinge, knowledge -distillation, the multilabel losses, and the top-*k* hinge and entropy losses of -[Lapin et al.](#citation). - -Every loss and gradient is accurate to the limit of its floating-point format: -confident predictions keep nonzero losses and target gradients down to subnormal -underflow instead of collapsing to zero (softmax collapse [7], fixed in [6]), and -distillation gradients are exact zero at equal student and teacher logits (softmax -cancellation [6]), across `float16`, `bfloat16`, `float32` and `float64` on CPU -and CUDA ([known limits](docs/numerics.md#known-limits)). +**libsdca** is a fast and accurate implementation of classification losses: cross-entropy, +multiclass and multilabel SVM hinges, top-*k* losses and knowledge distillation. The losses +are available as PyTorch and JAX layers on CPU and CUDA and as NumPy, Rust, C++, C and MATLAB +functions, and a stochastic dual coordinate ascent (SDCA) solver trains linear and kernel +classifiers with them. + +- **Fast:** The losses are fused CPU and CUDA kernels, not chains of tensor operations. On an + A100 with 32k–128k classes, cross-entropy matches `torch.compile` and is 2.4–3× faster than + eager PyTorch, and the multiclass hinge is 4–8× faster than compiled and 8–14× faster than + eager PyTorch implementations ([benchmarks](python/TORCH.md#benchmarks)). +- **Accurate:** Losses and gradients of confident predictions are kept down to the subnormal + range of the output dtype: a naive `float32` cross-entropy rounds to zero from a margin of + about 17 (softmax collapse [7]), and `libsdca` stays accurate up to about 104. Distillation + gradients are exactly zero at equal student and teacher logits [6]. The entropy and + distillation losses are tested on CPU and CUDA against 100-digit `mpmath` references + ([known limits](docs/API.md#known-limits)). +- **Many losses:** Cross-entropy, the multiclass hinge (Crammer–Singer SVM [1]), the top-*k* + hinge and entropy losses [4, 5], the multilabel hinge [2] and cross-entropy, logit + distillation and proximal variants, with nonsmooth or smoothed hinges, and + `linear_cross_entropy` for a language-model head without its `(N, V)` logits + ([losses](docs/API.md#loss-definitions)). +- **Many interfaces:** PyTorch modules and functions (with `torch.compile` support), JAX + functions for `jit`, `grad` and `vmap`, NumPy, a scikit-learn estimator, Rust, header-only + C++, C and MATLAB/Octave, with the same loss and option names in each + ([API reference](docs/API.md)). +- **SDCA solver with a duality-gap guarantee:** SDCA trains linear and kernel classifiers + (multiclass, top-*k* and multilabel SVMs, softmax regression). The duality gap bounds the + distance to the optimum, so `status == "solved"` guarantees that the objective of the + returned model is within the relative tolerance `tol` of the optimal value + ([results](docs/API.md#results)). Its steps use efficient projections onto the simplex and + the top-*k* cone [3, 4], which C++ and MATLAB also provide directly. +## Installation + +Create a fresh environment with [uv](https://docs.astral.sh/uv/) and install `libsdca` with +PyTorch support: + + +```bash +uv venv --seed --managed-python --python 3.12 .venv +source .venv/bin/activate +pip install 'libsdca[torch]' +``` + +Then use the losses in place of the PyTorch ones: + + ```python -from libsdca import CrossEntropyLoss +import torch +from libsdca.torch import CrossEntropyLoss, TopKHingeLoss -criterion = CrossEntropyLoss() # drop-in for torch.nn.CrossEntropyLoss(), except integer - # class labels only and no label smoothing -loss = criterion(logits, targets) -loss.backward() +device = "cuda" if torch.cuda.is_available() else "cpu" +logits = torch.randn(32, 5, device=device, requires_grad=True) +targets = torch.randint(5, (32,), device=device) + +ce = CrossEntropyLoss()(logits, targets) # replaces torch.nn.CrossEntropyLoss() +hinge = TopKHingeLoss(k=2)(logits, targets) # top-2 SVM loss +(ce + hinge).backward() ``` -- **Accurate to the limit of the format:** Every loss value and gradient entry is - verified within `64` output-dtype eps against 100-digit `mpmath` references across - `2` to `32,769` classes, subnormal tails, cancellation offsets and `±finfo.max` - ([verification](docs/numerics.md#how-it-is-verified)). -- **Fast fused loss kernels:** On an A100 at `8192 × 32000` and `8192 × 128256` - (`bfloat16` / `float32`), `CrossEntropyLoss` runs in `0.33–0.41×` eager - `F.cross_entropy` time (`0.93–0.98×` `torch.compile`) and saves a few scalars - per row instead of an `(N, C)` tensor ([performance](docs/performance.md)). -- **Certified SDCA:** Training evaluates the duality gap of the returned - weights and the feasible projection of the returned duals in `double`, with a - first-order rounding estimate $e$; - `status == "solved"` certifies $P(W) - D(A) + e \le \text{tol} \cdot \max(|P|, |D|)$ - ([numerics](docs/numerics.md#the-solved-certificate)). -- **Decoupled storage and compute precision:** `precision="float32"` or `"float64"` - selects working arithmetic independently of array/tensor storage - ([support matrix](docs/API.md#precision)). -- **Zero-overhead imports & optional BLAS:** `import libsdca` loads neither - PyTorch, JAX nor scikit-learn until used, and the C++ core includes a built-in - BLAS fallback when no external CBLAS is installed. - -## Objectives - -| Name | Loss | Targets | -| --- | --- | --- | -| `cross_entropy` | Softmax cross-entropy | single class index | -| `topk_cross_entropy` | Top-*k* entropy [5] | single class index | -| `multiclass_hinge` | Multiclass hinge (Crammer–Singer SVM) [1] | single class index | -| `topk_hinge_alpha`, `topk_hinge_beta` | Top-*k* hinge $\alpha$ [4] and $\beta$ [5] | single class index | -| `multilabel_hinge` | Multilabel hinge (multilabel SVM) [2] | label sets / boolean mask | -| `multilabel_cross_entropy` | Multilabel cross-entropy | label sets / boolean mask | - -Hinge losses accept `gamma >= 0` (`0` = nonsmooth hinge, `> 0` = Moreau-smoothed -hinge [5]). The loss layers additionally provide `proximal_cross_entropy`, -`distillation_loss` ([TORCH.md](python/TORCH.md#teacher-logits-and-distillation)), -`proximal_distillation_loss`, and `linear_cross_entropy`, which evaluates a -language-model linear head + cross-entropy in chunks without materializing `(N, V)` -logits ([TORCH.md](python/TORCH.md#linear-layer-and-cross-entropy-without-the-logits)). +### Other interfaces -## Installation +Every interface installs with one command or CMake target, and its guide documents all +options: | Interface | Install | Guide | | --- | --- | --- | | Python (NumPy) | `pip install libsdca` | [python/README.md](python/README.md) | | scikit-learn estimator | `pip install 'libsdca[sklearn]'` | [python/README.md](python/README.md#classifier) | | PyTorch losses | `pip install 'libsdca[torch]'` | [python/TORCH.md](python/TORCH.md) | -| JAX losses | `pip install 'libsdca[jax]'` | [python/JAX.md](python/JAX.md) | +| JAX losses (CPU) | `pip install 'libsdca[jax]'` | [python/JAX.md](python/JAX.md) | +| JAX losses (Linux NVIDIA GPU) | `pip install 'libsdca[jax-cuda]'` | [python/JAX.md](python/JAX.md) | | Rust | `cargo add libsdca` | [rust/README.md](rust/README.md) | | C++ (header-only) | CMake target `libsdca::libsdca` | [docs/CPP.md](docs/CPP.md) | | C | CMake target `libsdca::sdca_c` (`-DSDCA_BUILD_C_API=ON`) | [docs/C.md](docs/C.md) | | MATLAB / Octave | `run matlab/make.m` / `tests/matlab/run_octave_tests.sh` | [docs/MATLAB.md](docs/MATLAB.md) | -- **Python wheels:** Prebuilt wheels for Linux `x86_64` (`manylinux_2_28`) and - macOS 11+ (`arm64`) are self-contained; other platforms build from the sdist with - a C++11 compiler. -- **Runtime C++ extensions:** PyTorch and JAX compile small C++ extensions on first - use when a C++17/C++20 compiler and Python headers are installed (`build-essential` - and `python3-dev` on Debian/Ubuntu; `xcode-select --install` on macOS). Without a - compiler, losses fall back automatically with a one-time warning - ([build tools](python/TORCH.md#build-tools), [environment variables](docs/API.md#environment-variables)). -- **CMake and BLAS:** See [DEPENDENCIES.md](DEPENDENCIES.md) for CMake and BLAS options. +
+conda environment + + +```bash +conda create --yes --override-channels --channel conda-forge --prefix .venv python=3.12 pip +conda activate ./.venv +pip install 'libsdca[torch]' +``` + +
+ +
+C++ and C from source ```bash git clone https://github.com/mlapin/libsdca && cd libsdca @@ -97,10 +104,54 @@ cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/usr/local # add -DSDCA_BUILD_C_API cmake --build build --target install ``` +CMake and BLAS options are in [DEPENDENCIES.md](DEPENDENCIES.md). + +
+ +
+Wheels, GPU and compiler requirements + +- **Wheels:** Prebuilt for Linux `x86_64` (`manylinux_2_28`) and macOS 11+ `arm64`; other + platforms build from the sdist with a C++11 compiler. +- **GPU:** CUDA needs Linux, an NVIDIA GPU and a compatible driver. The PyTorch examples + use CUDA when available; `jax-cuda` installs the CUDA 13 runtime and Triton (Python 3.11+, + driver 580+). NumPy, scikit-learn, Rust, C, C++ and MATLAB run on CPU. +- **Compilers:** PyTorch and JAX compile small C++ extensions on first use when a C++17 + compiler and Python headers are present (`build-essential python3-dev` on Debian/Ubuntu, + `xcode-select --install` on macOS); otherwise they fall back with a one-time warning + ([build tools](python/TORCH.md#build-tools)). uv-managed and conda Pythons include headers. + +
+ ## Quick start -### Python (scikit-learn and NumPy) +Each example uses cross-entropy and the top-2 hinge (top-2 SVM) loss with the default +options; the PyTorch example is [above](#installation). The [API reference](docs/API.md) and +the interface guides document every parameter and option. + +### JAX +The losses are plain functions that compose with `jax.grad`, `jit` and `vmap`: + + +```python +import jax +import jax.numpy as jnp +from libsdca.jax import cross_entropy, topk_hinge_loss + +logits = jnp.array([[1.0, 2.0, 0.5], [0.0, 1.0, 3.0]]) +targets = jnp.array([0, 2]) + +value, gradient = jax.value_and_grad(cross_entropy)(logits, targets) +value, gradient = jax.value_and_grad(topk_hinge_loss)(logits, targets, k=2) +``` + +### Python (NumPy) + +`fit` trains a linear classifier with the SDCA solver and returns the model with its +convergence status: + + ```python import numpy as np import libsdca @@ -109,91 +160,167 @@ rng = np.random.default_rng(0) X = rng.normal(size=(200, 10)) y = (X @ rng.normal(size=(10, 3))).argmax(axis=1) -clf = libsdca.SDCAClassifier("topk_cross_entropy", k=2, C=1.0).fit(X, y) -model = libsdca.fit(X, y, "topk_cross_entropy", k=2, C=1.0) -values, gradients = libsdca.loss_and_grad(X @ model.weights, y, "topk_hinge_alpha", k=2, gamma=1.0) +model = libsdca.fit(X, y) # cross-entropy (softmax regression) +print(model.status, model.score(X, y)) +model = libsdca.fit(X, y, libsdca.TopKHingeBeta(k=2)) # top-2 SVM +print(model.status, model.score(X, y)) ``` -### PyTorch +### Python (scikit-learn) -```python -import torch -from libsdca import CrossEntropyLoss, TopKHingeLoss - -net = torch.nn.Linear(10, 5) -criterion = CrossEntropyLoss() # or TopKHingeLoss(k=2, variant="beta", gamma=0.5) -loss = criterion(net(torch.randn(32, 10)), torch.randint(0, 5, (32,))) -loss.backward() -``` - -### JAX +`SDCAClassifier` wraps the solver as a scikit-learn estimator: + ```python -import jax -import libsdca.jax as sjax +import numpy as np +from libsdca.sklearn import SDCAClassifier + +rng = np.random.default_rng(0) +X = rng.normal(size=(200, 10)) +y = (X @ rng.normal(size=(10, 3))).argmax(axis=1) -loss_and_grad = jax.jit(jax.value_and_grad( - lambda w, x, y: sjax.sdca_loss(x @ w, y, "topk_hinge_beta", k=3, gamma=0.5) -)) +clf = SDCAClassifier().fit(X, y) # cross-entropy +print(clf.status_, clf.score(X, y)) +clf = SDCAClassifier("topk_hinge_beta", k=2).fit(X, y) # top-2 SVM +print(clf.status_, clf.score(X, y)) ``` ### Rust +In a Cargo project (Rust 1.74+ and a C++11 compiler), run `cargo add libsdca`, save this as +`src/main.rs`, and run `cargo run`: + + ```rust -use libsdca::{fit, loss_and_grad, FitOptions, Loss, LossOptions, Matrix, Objective}; - -let x = Matrix::new(&[1.0, 0.0, 0.0, 1.0, -1.0, -1.0], 3, 2)?; -let fit_opts = FitOptions { c: 10.0, objective: Objective::CrossEntropy, ..Default::default() }; -let model = fit(x, &[0, 1, 2], &fit_opts)?; -let scores = Matrix::new(&[0.3, -0.8, 1.2], 1, 3)?; -let loss_opts = LossOptions { loss: Loss::TopKHingeBeta, k: 2, gamma: 1.0, ..Default::default() }; -let (values, gradients) = loss_and_grad(scores, &[0], &loss_opts)?; +use libsdca::{Loss, Matrix, Solver}; + +fn main() -> libsdca::Result<()> { + let x = Matrix::new(&[1.0, 0.0, 0.0, 1.0, -1.0, -1.0], 3, 2)?; // 3 examples, 2 features + let y = [0, 1, 2]; + + let model = Solver::new(Loss::cross_entropy()).fit(x, &y)?; + println!("{} {:?}", model.status(), model.predict(x)?); + + let model = Solver::new(Loss::topk_hinge(2)).fit(x, &y)?; // top-2 SVM + println!("{} {:?}", model.status(), model.predict(x)?); + Ok(()) +} ``` ### C++ +After the [CMake installation](#other-interfaces) (*C++ and C from source*), compile this as +a C++11 program with the installed include directory and +`-DBLAS_BUILTIN -pthread -fno-fast-math -ffp-contract=off` (a CMake consumer is in +[examples/cpp](examples/cpp)): + + ```cpp +#include +#include #include "sdca/solver.h" -#include "sdca/utility/logging.cpp" // in exactly one translation unit -auto context = sdca::make_context( - sdca::make_input_feature(d, n, features), sdca::multiclass_output(m, labels), - sdca::l2_entropy(/* C */ 1.0), dual, weights); -sdca::make_solver(context).solve(); +// Trains the objective it is given; solved is whether the gap reached tol. +struct train { + bool& solved; + template + void operator()(Objective objective) const { + double features[] = {1, 0, 0, 1, -1, -1}; // 3 examples, 2 features + double weights[6] = {}, dual[9] = {}; // W (2, 3), A (3, 3) + std::vector labels = {0, 1, 2}; + auto context = sdca::make_context( + sdca::make_input_feature(2, 3, features), + sdca::make_output_multiclass(labels.begin(), labels.end(), 3), + std::move(objective), weights, dual); + sdca::make_solver(context).solve(); + solved = context.status == sdca::solver_status::solved; + } +}; + +int main() { + bool entropy = false, hinge = false; + sdca::visit_multiclass_objective( + sdca::cross_entropy_objective(), train{entropy}); // cross-entropy + sdca::visit_multiclass_objective( + sdca::topk_hinge_beta_objective(1.0, 2), train{hinge}); // top-2 hinge + return entropy && hinge ? 0 : 1; +} +``` + +### C + +Build with `-DSDCA_BUILD_C_API=ON` as for C++ and link `libsdca_c` (a CMake consumer is in +[examples/c](examples/c)); this program evaluates both losses and their gradients: + + +```c +#include +#include "sdca/c_api.h" + +int main(void) { + const double scores[] = {0.3, -0.8, 1.2}; + const int64_t label[] = {0}; + double values[1], gradients[3]; + struct sdca_loss loss = SDCA_LOSS_INIT; /* cross-entropy */ + struct sdca_execution execution = SDCA_EXECUTION_INIT; + struct sdca_targets targets = SDCA_TARGETS_INIT; /* class labels */ + targets.labels = label; + int status = sdca_loss_evaluate(&loss, &execution, SDCA_DTYPE_FLOAT64, 1, 3, scores, + &targets, values, gradients); + loss.kind = SDCA_LOSS_TOPK_HINGE_BETA; /* top-2 SVM loss */ + loss.k = 2; + status |= sdca_loss_evaluate(&loss, &execution, SDCA_DTYPE_FLOAT64, 1, 3, scores, + &targets, values, gradients); + return status != SDCA_OK; +} ``` -### MATLAB +### MATLAB / Octave +In MATLAB, `run matlab/make.m; addpath matlab` builds the gateways (with a configured MEX +C++ compiler). In Octave, run `tests/matlab/run_octave_tests.sh`, then `addpath build/octave`. + + ```matlab -model = matsdca_fit(X, Y, struct('objective', 'cross_entropy', 'C', 1)); % X: d-by-n -[~, predicted] = max(model.W' * Xtest); +X = [1 0 -1; 0 1 -1]; % 2 features, 3 examples +Y = [1 2 3]; % labels in 1:3 +model = matsdca_fit(X, Y); % cross-entropy +model = matsdca_fit(X, Y, struct('loss', 'topk_hinge_beta', 'k', 2)); % top-2 hinge +[~, predicted] = max(model.W' * X); ``` -Runnable examples are in [examples/](examples), [rust/examples](rust/examples) and +More examples: [examples/](examples), [rust/examples](rust/examples) and [matlab/demo.m](matlab/demo.m). ## Documentation -- [API reference](docs/API.md): loss formulas, cross-language names, options, - precision, results and environment variables +An index of the guides is in [docs/](docs/README.md). + +- [API reference](docs/API.md): loss formulas, names across languages, options, precision, + results, threads, environment variables and known limits - Interface guides: [Python](python/README.md), [PyTorch](python/TORCH.md), [JAX](python/JAX.md), [Rust](rust/README.md), [C++](docs/CPP.md), [C](docs/C.md), [MATLAB](docs/MATLAB.md) -- [Numerical notes](docs/numerics.md): precision guarantees, verification and known limits -- [Performance](docs/performance.md): CUDA, CPU and solver benchmarks -- [Softmax collapse and cancellation](docs/softmax-collapse.md): formulations and - reproduction of Barron et al. [6] - [Dependencies](DEPENDENCIES.md), [contributing](CONTRIBUTING.md), - [AI coding assistant guide](AGENTS.md) and [changelog](CHANGELOG.md) -- [Project page](https://mlapin.github.io/libsdca/) and technical report, - [*libsdca: Accurate and Fast Classification Losses for SDCA and Neural Networks*](https://mlapin.github.io/libsdca/report/) + [guide for AI coding assistants](AGENTS.md) and [changelog](CHANGELOG.md) +- [Project page](https://mlapin.github.io/libsdca/) and technical report ## Citation -If `libsdca` is useful in your research, please cite the paper for the losses you use -and the technical report for the library: +If `libsdca` is useful in your research, please cite the TPAMI paper; the earlier papers +introduce the top-*k* losses, and the technical report describes the library: ```bibtex +@article{lapin2018analysis, + title = {Analysis and Optimization of Loss Functions for Multiclass, Top-k, and Multilabel Classification}, + author = {Lapin, Maksim and Hein, Matthias and Schiele, Bernt}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + volume = {40}, + number = {7}, + pages = {1533--1554}, + year = {2018} +} + @inproceedings{lapin2015topk, title = {Top-k Multiclass {SVM}}, author = {Lapin, Maksim and Hein, Matthias and Schiele, Bernt}, @@ -208,16 +335,6 @@ and the technical report for the library: year = {2016} } -@article{lapin2018analysis, - title = {Analysis and Optimization of Loss Functions for Multiclass, Top-k, and Multilabel Classification}, - author = {Lapin, Maksim and Hein, Matthias and Schiele, Bernt}, - journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, - volume = {40}, - number = {7}, - pages = {1533--1554}, - year = {2018} -} - @misc{lapin2026libsdca, title = {libsdca: Accurate and Fast Classification Losses for {SDCA} and Neural Networks}, author = {Lapin, Max}, @@ -230,6 +347,8 @@ GitHub's *Cite this repository* button uses [CITATION.cff](CITATION.cff). ## References +The numbered citations above refer to: + 1. K. Crammer and Y. Singer. On the algorithmic implementation of multiclass kernel-based vector machines. *JMLR*, 2001. 2. K. Crammer and Y. Singer. A family of additive online algorithms for @@ -240,7 +359,7 @@ GitHub's *Cite this repository* button uses [CITATION.cff](CITATION.cff). 5. M. Lapin, M. Hein and B. Schiele. Loss functions for top-k error: analysis and insights. *CVPR*, 2016. 6. J. T. Barron, D. Gnanapragasam and M. Tschannen. A fix for softmax and - cross-entropy. arXiv:2610.31642, 2026 ([reproduction](docs/softmax-collapse.md#reproduction)). + cross-entropy. arXiv:2610.31642, 2026. 7. L. Prieto, M. Barsbey, P. A. M. Mediano and T. Birdal. Grokking at the edge of numerical stability. *ICLR*, 2025. diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..1d0ba9ea --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.4.0 diff --git a/contracts.json b/contracts.json new file mode 100644 index 00000000..e4d3dab7 --- /dev/null +++ b/contracts.json @@ -0,0 +1,446 @@ +[ + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -29, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float32", + "power": -60, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -29, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "float64", + "power": -60, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -29, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "multiclass_hinge", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "multiclass_hinge", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "topk_hinge_alpha", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "topk_hinge_beta", + "precision": null, + "route": "auto", + "status": "PASS" + }, + { + "contract": "hinge_score_order", + "dtype": "bfloat16", + "power": -60, + "loss": "topk_hinge_beta", + "precision": "float64", + "route": "auto", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "auto", + "precision": null, + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "auto", + "precision": "float32", + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "auto", + "precision": "float64", + "resolved_precision": "float64", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "native", + "precision": null, + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "native", + "precision": "float32", + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "float32", + "backend": "native", + "precision": "float64", + "resolved_precision": "float64", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "auto", + "precision": null, + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "auto", + "precision": "float32", + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "auto", + "precision": "float64", + "resolved_precision": "float64", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "native", + "precision": null, + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "native", + "precision": "float32", + "resolved_precision": "float32", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + }, + { + "contract": "jax_proximal_ce_work", + "dtype": "bfloat16", + "backend": "native", + "precision": "float64", + "resolved_precision": "float64", + "native_calls": 1, + "route": "native FFI", + "status": "PASS" + } +] diff --git a/docs/API.md b/docs/API.md index c6adba42..d2e68dfc 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,9 +1,8 @@ # API reference -`libsdca` shares one set of loss names, option names, status codes and defaults -across C++, the C ABI (`include/sdca/c_api.h`), Python, PyTorch, JAX, Rust and -MATLAB. Interface guides: [Python](../python/README.md), -[PyTorch](../python/TORCH.md), [JAX](../python/JAX.md), +One set of loss names, option names, status codes and defaults is shared by C++, the C ABI +(`include/sdca/c_api.h`), Python, PyTorch, JAX, Rust and MATLAB. Interface guides: +[Python](../python/README.md), [PyTorch](../python/TORCH.md), [JAX](../python/JAX.md), [Rust](../rust/README.md), [C++](CPP.md), [C](C.md) and [MATLAB](MATLAB.md). ## Loss definitions @@ -34,23 +33,26 @@ $h_j = 1 + z_j - z_y$ ($j \neq y$) sorted as $h_{[1]} \ge h_{[2]} \ge \dots$ where $H(p, q) = -\sum_j p_j \log q_j$ is the cross-entropy and $p = \mathrm{softmax}(t/T)$. - **Hinge smoothing (`gamma`):** `gamma = 0` (default) is the nonsmooth hinge; `gamma > 0` is its Moreau envelope $\max_{p \in S} \langle h, p \rangle - \frac{\gamma}{2}\Vert p \Vert_2^2$, - which is within $\gamma / 2$ of the nonsmooth loss and has a $(1/\gamma)$-Lipschitz gradient [4]. + which has a $(1/\gamma)$-Lipschitz gradient [4] and lies below the nonsmooth loss by at most + $\gamma / (2k)$ for the single-label hinges ($\gamma / 2$ for `multiclass_hinge`) and + $\gamma$ for `multilabel_hinge`. - **Top-*k* exclusion:** Top-*k* hinges exclude the target class $y$ when selecting the $k$ - largest competitor margins $h_{[1]}, \dots, h_{[k]}$ [4, Prop. 11]. At $k = 1$, + largest competitor margins $h_{[1]}, \dots, h_{[k]}$ [4]. At $k = 1$, `topk_hinge_alpha` and `topk_hinge_beta` coincide with `multiclass_hinge`. ## Objectives and losses ### Core entry points -| Interface | Train classifier | Loss & gradient | Coordinate step | +| Interface | Train a classifier | Loss and gradient | Coordinate step | | --- | --- | --- | --- | -| Python (NumPy / sklearn) | `fit`, `SDCAClassifier` | `loss_and_grad` | `coordinate_update` | -| PyTorch | — | `SDCALoss`, `sdca_loss` | — | -| JAX | — | `libsdca.jax.sdca_loss` | — | -| Rust | `fit` | `loss_and_grad` | `coordinate_update` | -| C++ | `make_solver(ctx).solve()` | `loss_and_gradient` | `objective.update_dual_variables` | -| C | `sdca_fit` | `sdca_loss_gradient` | `sdca_update_objective` | +| NumPy | `fit(X, y, loss)` | `loss_and_grad(logits, target, loss)` | `coordinate_update(logits, target, loss)` | +| scikit-learn | `libsdca.sklearn.SDCAClassifier` | — | — | +| PyTorch | — | one function and module per loss (`libsdca.torch`) | — | +| JAX | — | one function per loss (`libsdca.jax`) | — | +| Rust | `Solver::fit` | `Loss::loss_and_grad` | `Solver::coordinate_update` | +| C++ | `make_solver(ctx).solve()` | `loss_and_gradient`, `distillation_loss` | `update_dual_variables` | +| C | `sdca_model_fit` | `sdca_loss_evaluate` | `sdca_dual_update` | | MATLAB | `matsdca_fit` | `matsdca_loss` | — | Public C++ headers: `sdca/solver.h`, `sdca/prox.h` and `sdca/loss.h` @@ -58,59 +60,69 @@ Public C++ headers: `sdca/solver.h`, `sdca/prox.h` and `sdca/loss.h` ### Names across interfaces -| Canonical name | PyTorch module, PyTorch / JAX function | Rust | C++ | -| --- | --- | --- | --- | -| `cross_entropy` | `CrossEntropyLoss`, `cross_entropy` | `CrossEntropy` | `l2_entropy(c)` | -| `topk_cross_entropy` | `TopKCrossEntropyLoss`, `topk_cross_entropy` | `TopKCrossEntropy` | `l2_entropy_topk(c, k)` | -| `multiclass_hinge` | `MulticlassHingeLoss`, `multiclass_hinge_loss` | `MulticlassHinge` | `l2_hinge_topk[_smooth]` | -| `topk_hinge_alpha` | `TopKHingeAlphaLoss`, `topk_hinge_alpha_loss` | `TopKHingeAlpha` | `l2_hinge_topk[_smooth]` | -| `topk_hinge_beta` | `TopKHingeLoss`, `topk_hinge_loss` (`*Beta*`) | `TopKHingeBeta` | `l2_topk_hinge[_smooth]` | -| `multilabel_hinge` | `MultilabelHingeLoss`, `multilabel_hinge_loss` | `MultilabelHinge` | `l2_multilabel_hinge[_smooth]` | -| `multilabel_cross_entropy` | `MultilabelCrossEntropyLoss`, `multilabel_cross_entropy` | `MultilabelCrossEntropy` | `l2_multilabel_entropy(c)` | -| `proximal_cross_entropy` | `ProximalCrossEntropyLoss`, `proximal_cross_entropy` | `Loss::ProximalCrossEntropy` | `loss_and_gradient` | -| `distillation` | `DistillationLoss`, `distillation_loss` | `fit_distillation` | `l2_distillation`, `distillation_loss` | -| `proximal_distillation` | `ProximalDistillationLoss`, `proximal_distillation_loss` | `proximal_distillation_*` | `proximal_distillation_loss` | -| `linear_cross_entropy` | `LinearCrossEntropyLoss`, `linear_cross_entropy` (PyTorch) | — | — | - -- **Aliases and C ABI names:** `TopKHingeBetaLoss` / `topk_hinge_beta_loss` alias - `TopKHingeLoss` / `topk_hinge_loss` (`variant="beta"` default). Python and Rust evaluate - distillation via `distillation_loss_and_grad` and `proximal_distillation_loss_and_grad` - (`distillation_loss_and_gradient` and `proximal_distillation_loss_and_gradient` in C++). - In C, `enum sdca_loss` constants are `SDCA_` (e.g. `SDCA_CROSS_ENTROPY`), - plus `sdca_distillation_*`, `sdca_fit_distillation` and `sdca_proximal_distillation_rows`. -- **Python targets:** Passing `objective="distillation"` to `fit` or `coordinate_update` - trains on a target probability matrix `(n, num_classes)`; passing label sets or a boolean - mask with `cross_entropy` selects `multilabel_cross_entropy`. +Every interface uses the canonical names and takes, for each loss, only its own parameters +(`k`, `gamma`, `alpha`, `temperature`); a parameter of another loss is an error. + +| Canonical name | NumPy | PyTorch module, PyTorch / JAX function | Rust | C | +| --- | --- | --- | --- | --- | +| `cross_entropy` | `CrossEntropy()` | `CrossEntropyLoss`, `cross_entropy` | `Loss::CrossEntropy` | `SDCA_LOSS_CROSS_ENTROPY` | +| `topk_cross_entropy` | `TopKCrossEntropy(k)` | `TopKCrossEntropyLoss`, `topk_cross_entropy` | `Loss::TopKCrossEntropy { k }` | `SDCA_LOSS_TOPK_CROSS_ENTROPY` | +| `multiclass_hinge` | `MulticlassHinge(gamma)` | `MulticlassHingeLoss`, `multiclass_hinge_loss` | `Loss::MulticlassHinge { gamma }` | `SDCA_LOSS_MULTICLASS_HINGE` | +| `topk_hinge_alpha` | `TopKHingeAlpha(k, gamma)` | `TopKHingeLoss`, `topk_hinge_loss` (`variant="alpha"`) | `Loss::TopKHingeAlpha { k, gamma }` | `SDCA_LOSS_TOPK_HINGE_ALPHA` | +| `topk_hinge_beta` | `TopKHingeBeta(k, gamma)` | `TopKHingeLoss`, `topk_hinge_loss` (`variant="beta"`) | `Loss::TopKHingeBeta { k, gamma }` | `SDCA_LOSS_TOPK_HINGE_BETA` | +| `multilabel_hinge` | `MultilabelHinge(gamma)` | `MultilabelHingeLoss`, `multilabel_hinge_loss` | `Loss::MultilabelHinge { gamma }` | `SDCA_LOSS_MULTILABEL_HINGE` | +| `multilabel_cross_entropy` | `MultilabelCrossEntropy()` | `MultilabelCrossEntropyLoss`, `multilabel_cross_entropy` | `Loss::MultilabelCrossEntropy` | `SDCA_LOSS_MULTILABEL_CROSS_ENTROPY` | +| `proximal_cross_entropy` | `ProximalCrossEntropy(alpha)` | `ProximalCrossEntropyLoss`, `proximal_cross_entropy` | `Loss::ProximalCrossEntropy { alpha }` | `SDCA_LOSS_PROXIMAL_CROSS_ENTROPY` | +| `distillation` | `Distillation(temperature)` | `DistillationLoss`, `distillation_loss` | `Loss::Distillation { temperature }` | `SDCA_LOSS_DISTILLATION` | +| `proximal_distillation` | `ProximalDistillation(temperature, alpha)` | `ProximalDistillationLoss`, `proximal_distillation_loss` | `Loss::ProximalDistillation { temperature, alpha }` | `SDCA_LOSS_PROXIMAL_DISTILLATION` | +| `linear_cross_entropy` | — | `LinearCrossEntropyLoss`, `linear_cross_entropy` (PyTorch) | — | — | + +C++ objectives use the canonical names with an `l2_` prefix (`l2_topk_hinge_beta`; +`*_objective(c, ...)` factories take `gamma`, default `0`), and MATLAB takes the canonical +names as `opts.objective` / `opts.loss`. + +- **Targets:** The single-label losses take class labels (0-based; 1-based in MATLAB), the + multilabel losses label sets or a boolean mask, and the distillation losses teacher + logits. The training objective `cross_entropy` also takes target probabilities, rows of an + `(n, num_classes)` matrix such as a teacher's softmax (`libsdca.Probabilities` in NumPy, + `Targets::Probabilities` in Rust, `SDCA_TARGET_PROBABILITIES` in C). The proximal and + distillation losses are not training objectives. ## Options -Python, MATLAB and Rust share the option names below (`c` in Rust and C): +The solver options below have the same names and defaults in every interface (Rust: `Solver` +setters such as `with_tol`; C: `struct sdca_fit_options`; C++: `context.criteria`; the +scikit-learn estimator names `max_epoch` `max_iter` and `seed` `random_state`): -| Option | C++ | Default | Meaning | -| --- | --- | --- | --- | -| `C` (`c` in Rust / C) | objective constructor | `1.0` | Loss weight in $0.5\Vert W \Vert_2^2 + C \sum_i \ell_i$ | -| `k` | objective constructor | `1` | Integer in `[1, classes - 1]` for top-*k* losses | -| `gamma` | `_smooth` objective | `0.0` | `0`: nonsmooth hinge; `> 0`: Moreau smoothing step | -| `tol` | `criteria.epsilon` | `1e-3` | Relative duality-gap stopping threshold | -| `max_epoch` | `criteria.max_epoch` | `1000` | Maximum passes over the training set | -| `eval_epoch` | `criteria.eval_epoch` | `10` | Epochs between duality-gap evaluations | -| `eval_on_start` | `criteria.eval_on_start` | `false` | Evaluate objective and gap before epoch 1 | -| `recompute_primal` | `criteria.recompute_primal` | `true` | Rebuild $W = W_0 + X^\top A$ at each evaluation | -| `shrinking` | `criteria.shrinking` | `"auto"` | Hinge shrinking and screening ([below](#shrinking)) | -| `projection_precision` | `criteria.projection` | `"float64"` | Multilabel hinge projection solver ([below](#projection-precision)) | -| `kernel` | `make_input_kernel` | `false` | Input $X$ is a symmetric PSD $(n, n)$ Gram matrix | -| `center` | `context.primal_initial` | `None` | Regularize $0.5\Vert W - W_0 \Vert_2^2$ instead of $0.5\Vert W \Vert_2^2$ | -| `initial_dual` | dual buffer | `None` | Initial dual variables $A$ for warm starts | -| `num_threads` | `criteria.num_threads` | `auto` | Worker threads for losses, steps and evaluations | -| `precision` | `` | `"float64"` | Working and iterate precision ([below](#precision)) | +| Option | Default | Meaning | +| --- | --- | --- | +| `C` (`c` in Rust / C) | `1.0` | Loss weight in $0.5\Vert W \Vert_2^2 + C \sum_i \ell_i$ | +| `tol` | `1e-3` | Relative duality-gap stopping threshold | +| `max_epoch` | `1000` | Maximum passes over the training set | +| `eval_epoch` | `10` | Epochs between duality-gap evaluations; `0` evaluates only at the end | +| `eval_on_start` | `false` | Evaluate objective and gap before epoch 1 | +| `recompute_primal` | `true` | Rebuild $W = W_0 + X^\top A$ at each evaluation | +| `shrinking` | `"auto"` | Hinge shrinking and screening ([below](#shrinking)); the entropy objectives take only `"auto"` | +| `seed` | `0` | The order of the examples in the epochs | +| `num_classes` | largest label plus one | The number of classes | +| `center` | none | Regularize $0.5\Vert W - W_0 \Vert_2^2$ instead of $0.5\Vert W \Vert_2^2$ (features only) | +| `initial_dual` | none | Initial dual variables $A$ for warm starts; the library checks them and builds $W$ | + +Kernel input is explicit: `libsdca.Kernel(K)` in NumPy, `kernel="precomputed"` in +scikit-learn, `Input::Kernel` in Rust, `SDCA_INPUT_KERNEL` in C, `make_input_kernel` in C++. +Every call also takes the execution settings `precision` ([below](#precision)) and +`num_threads` ([threads](#threads)); `projection_precision` belongs to `multilabel_hinge` +([below](#projection-precision)). - **Evaluations (`eval_epoch`, `recompute_primal`):** Besides checking every `eval_epoch` epochs, a fit with `tol > 0` schedules up to two interpolated evaluations when the gap trend projects crossing `tol` early. With `recompute_primal=True` (default), feature fits rebuild $W = W_0 + X^\top A$ before evaluating the primal objective so incremental update drift does not - pollute the certificate. -- **Kernel validation (`kernel=True`):** Every interface verifies a nonnegative + enter the gap (not for `float32` data with `precision="float64"`, whose `double` $W$ is + evaluated as stored). A fit stopped by `max_epoch` reports `solved` when its last + evaluation meets `tol`. +- **Kernel validation:** Every interface verifies a nonnegative diagonal, symmetry within $10^{-5}\sqrt{K_{ii}K_{jj}}$, and $\lvert K_{ij} \rvert \le (1 + 10^{-3})\sqrt{K_{ii}K_{jj}}$. - **Feature scaling:** Because each coordinate step scales with $1/(C\Vert x_i \Vert_2^2)$, @@ -126,14 +138,15 @@ bounds (entropy objectives have strictly positive probabilities and ignore the o shrinking [7]. Epochs between full passes skip examples whose dual block did not change and restrict single-label updates to the target and nonzero-dual classes; full passes and evaluations run after `eval_epoch` full-pass equivalents of work (at most 50 epochs). -- `"safe"` (`Shrinking::Safe`, `SDCA_SHRINKING_SAFE`): Gap-safe screening [8] on feature - inputs. Uses strong convexity $\Vert W - W^\star \Vert_2 \le \sqrt{2\text{ gap}}$ to permanently - prune dual variables proven to be zero at the optimum. +- `"safe"` (`Shrinking::Safe`, `SDCA_SHRINKING_SAFE`): Gap-safe screening [8] for the + single-label hinges on feature inputs (no effect for `multilabel_hinge` or kernels). Uses + strong convexity $\Vert W - W^\star \Vert_2 \le \sqrt{2\text{ gap}}$ to permanently prune + dual variables proven to be zero at the optimum. - `"both"` / `"auto"` (default): Combines `"heuristic"` and `"safe"`. - `"off"`: Updates every class of every example on every epoch. -All modes evaluate the duality gap across all examples and classes (`solved` -carries the same certificate; see [performance](performance.md#solver)). +All modes evaluate the duality gap across all examples and classes, so `solved` carries the +same guarantee. ### Projection precision @@ -143,69 +156,75 @@ carries the same certificate; see [performance](performance.md#solver)). in `double` and verifies that every entry lies clear of its threshold error bound, falling back to the extended-precision solver only when a near-tie or extreme scale fails the check. - `"extended"` (`ProjectionPrecision::Extended`, `SDCA_PROJECTION_EXTENDED`): Always runs - the hardened `long double` solver (`0–12%` slower at 1,000 classes, bitwise reproducible - on extreme spreads; see [numerics.md](numerics.md#projection-precision)). + the `long double` solver (slightly slower; reproducible on extreme spreads). ## Precision -Storage and working arithmetic are decoupled across all interfaces: +Storage and working precision are separate. `precision=None` (default) follows the storage +dtype, with 16-bit storage computed in `float32`; `"float32"` or `"float64"` selects the +working precision explicitly: -| Storage dtype | `precision` | Native (NumPy, Rust, C) | PyTorch | JAX | -| --- | --- | --- | --- | --- | -| `float16`, `bfloat16` | `None` (default), `"float32"` | unsupported (cast to `float32`) | `float32` (`float64` CE stats) | `float32` work | -| `float16`, `bfloat16` | `"float64"` | unsupported | `float64` work | `float64` (needs `jax_enable_x64`) | -| `float32` | `None` (default), `"float64"` | `float64` work & `double` iterate | `float32` in CE (`float64` in hinges or `"float64"`) | `float32` in CE (`float64` with `"float64"`, x64) | -| `float32` | `"float32"` | `float32` work & `float` iterate | `float32` work | `float32` work | -| `float64` | `None` (default), `"float64"` | `float64` (`"float32"` invalid) | `float64` (`"float32"` invalid) | `float64` (`"float32"` invalid) | - -- **Outputs:** Returned weights, duals and gradients keep the input storage dtype. Loss - values are `double` in the native APIs and `promote(logits.dtype, float32)` in PyTorch - and JAX (`float32` for 16-/32-bit, `float64` for `float64`, as in `torch.autocast`). -- **Default `precision="float64"` for `float32` fits:** Keeping `W` and the duals in - `float32` loses updates below $\frac{1}{2}\mathrm{ulp}$ near the optimum (`4.7e-6` gap on - `digits`). The default `"float64"` (`` in C++) keeps `double` iterates - (`8(d + n)m` extra bytes) and certifies the rounded `float32` arrays (`3.5e-7` gap; - [numerics.md](numerics.md#what-each-precision-guarantees)). Pass `"float32"` for - `1.6–1.9×` faster cross-entropy fits. -- **Compiler flags:** Build `libsdca` with IEEE semantics (`-fno-fast-math -ffp-contract=off`); - `-ffast-math` or `-Ofast` breaks nonfinite-input checks and compensated reductions. +| Storage dtype | `None` (default), `"float32"` | `"float64"` | +| --- | --- | --- | +| `float16`, `bfloat16` | `float32` work (NumPy casts 16-bit arrays to `float32`; C takes `float32` or `float64`) | `float64` work (JAX: see [backends](../python/JAX.md#backends)) | +| `float32` | `float32` work | `float64` work | +| `float64` | `float64` work (`"float32"`: NumPy casts to `float32` and returns `float32` arrays; PyTorch and JAX raise, since gradients keep the logits' dtype) | `float64` work | + +MATLAB follows the class of `X` (`'single'` or `'double'`). In C the storage type is the +`dtype` argument and the work is `struct sdca_execution`'s `precision` +(`SDCA_PRECISION_DEFAULT`, `SDCA_PRECISION_FLOAT32` or `SDCA_PRECISION_FLOAT64`). + +- **Accuracy contract:** `float32` work bounds the accuracy of the results; individual + operations may run in `float64`. Row statistics of the entropy losses are accumulated in + `float64`, and kernels promote rows or parameters that `float32` cannot represent. Some paths + (single-label cross-entropy and smoothed hinges in the native library, proximal distillation + on CPU) keep `float64` arithmetic throughout. +- **Outputs:** Weights, duals and gradients keep the storage dtype. Loss values are `double` + in the native APIs and `promote(logits.dtype, float32)` in PyTorch and JAX. +- **`float32` fits:** Keeping $W$ and the duals in `float32` can stall near the optimum + (updates below half an ulp are lost). `precision="float64"` (`` in C++) keeps + `double` iterates for tighter convergence at `8(d + n)m` extra bytes. Both modes check the + gap of the returned arrays. +- **Compiler flags:** Build with IEEE semantics (`-fno-fast-math -ffp-contract=off`); + `-ffast-math` or `-Ofast` breaks the nonfinite-input checks and compensated sums. ### Parameter domains and numerical limits All loss and training entry points require `classes >= 2`, integer `1 <= k < classes`, finite scores/features, and valid targets (0-based class indices, or 1-based in MATLAB; boolean masks require at least one positive label, plus one negative label for -`multilabel_hinge` and `proximal_cross_entropy`). +`multilabel_hinge` and `proximal_cross_entropy`; label sets in `fit` and `coordinate_update` +require a negative label for every objective). | Parameter | Valid domain | Notes | | --- | --- | --- | -| `C`, `alpha`, `temperature` | finite, strictly positive | `C` and `C * norm2` must be normal in working precision; `alpha` or `temperature` outside normal `float32` promotes to `float64` | +| `C`, `alpha`, `temperature` | finite, strictly positive | `C`, `norm2` and `C * norm2` must be normal in the storage dtype, and `C` in working precision; `alpha` or `temperature` outside normal `float32` promotes to `float64` | | `gamma` | finite, `>= 0` | `0` selects the nonsmooth hinge; positive values follow the rules of `alpha` | -| `tol` | finite, `>= 0` | `0` runs until `max_epoch` (or time limits in MATLAB); tolerances below ~`1e-12` in `float64` cannot be certified | +| `tol` | finite, `>= 0` | `0` disables the `solved` stop: the fit ends with `max_epoch`, `no_progress` or `failed` (or a time limit in MATLAB) | | `ignore_index` (PyTorch) | 64-bit integer | Matching targets are excluded from the loss and gradient | | `weight` (PyTorch CE) | finite, `>= 0`, shape `(C,)` | Matches logits device and dtype (`float32` accepted for 16-bit logits under autocast) | -Tested accuracy bounds and representation limits are documented in -[numerics.md](numerics.md#known-limits). +See [known limits](#known-limits) for accuracy bounds. ## Results -Training returns a status (`none`, `solving`, `solved`, `no_progress`, `max_epoch`, -`max_cpu_time`, `max_wall_time`, `failed`) and an evaluation `history` (`epoch`, -`primal`, `dual`, `primal_loss`, `dual_loss`, `regularizer`, `relative_gap`, -`accuracy` / `rank_loss`, `cpu_time`, `wall_time`). +Training returns a status (`solved`, `no_progress`, `max_epoch` or `failed`) and an +evaluation `history` with the same fields in every interface: `epoch`, `primal`, `dual`, +`primal_loss`, `dual_loss`, `regularizer`, `relative_gap`, `accuracy` (the rank loss for the +multilabel losses), `cpu_time` and `wall_time`. -Only `status == "solved"` certifies convergence: +Only `status == "solved"` guarantees convergence: $$P(W) - D(A) + e \le \text{tol} \cdot \max(|P(W)|, |D(A)|),$$ where $P(W)$ is evaluated in `double` at the returned weights, $D(A)$ at the feasible -projection of the returned duals, and $e$ is a first-order estimate of evaluation rounding ([numerics.md](numerics.md#the-solved-certificate)). +projection of the returned duals, and $e$ is a first-order estimate of evaluation rounding. If a numerical step fails or the evaluated gap falls below $-e$, training stops with `failed` at the last consistent iterate. With a regularizer center $W_0$, $P(W)$ and $D(A)$ evaluate the full centered objective directly from $W - W_0$ without cancelling $\frac{1}{2}\Vert W_0 \Vert_2^2$. Array layouts for $n$ examples, $d$ features and $m$ classes: Python returns `weights (d, m)` -column-major and `dual (n, m)` row-major; Rust returns both row-major; MATLAB takes +column-major and `dual (n, m)` row-major; Rust returns both row-major (`Model::weights`, +`Model::dual`) and the history as `Evaluation` structs; MATLAB takes features `(d, n)` and returns `W (d, m)` and `A (m, n)`. ## Distillation losses @@ -215,8 +234,7 @@ same shape and dtype; the teacher receives no gradient. With $p = \mathrm{softma $q = \mathrm{softmax}(z/T)$, the student gradient is $(q - p)/T$ (or $(\pi - p)/T$ at the Moreau-envelope optimum $\pi = \mathrm{softmax}(u^\star)$ for `proximal_distillation_loss`), with no $T^2$ multiplier. Because each entry is formed from a centered log-ratio via `expm1`, -identical student and teacher logits produce an **exact zero gradient** in every dtype -([softmax cancellation](softmax-collapse.md#softmax-cancellation)). +identical student and teacher logits produce an **exact zero gradient** in every dtype. ```python import torch @@ -230,33 +248,39 @@ prox = ProximalDistillationLoss(temperature=4.0, alpha=0.5)(student, teacher) ``` In PyTorch, both run on fused Triton kernels on CUDA and the C++ extension on CPU -(first-order derivatives only). In JAX (`libsdca.jax`), both run in `jax.numpy` on any -device (`jvp`/`jacfwd` raise, while reverse-over-reverse and `jax.hessian` are supported). +(first-order derivatives only). In JAX (`libsdca.jax`), both run on the same Triton +kernels on CUDA and use the shared CPU kernels where available, with array +fallbacks. Direct `jvp`/`jacfwd` of these custom-VJP losses raise. Ordinary +distillation supports reverse-over-reverse and `jax.hessian`; proximal distillation +supports `jax.hessian` (forward-over-reverse), but reverse-over-reverse is unsupported +because the array reference solve's dynamic loop cannot be transposed. ## Threads Loss evaluations and coordinate steps process rows independently across up to `num_threads` worker threads with **bitwise-identical results** for every thread count: -- **Default selection:** When `num_threads` is omitted (`None` in Python/Rust, - `SDCA_THREADS_DEFAULT` in C), the library reads `SDCA_NUM_THREADS` once and otherwise uses - automatic threading (`0`), capped by the active OpenMP team (`torch.set_num_threads`) or - `OMP_NUM_THREADS` and divided across concurrent calls. Passing `1` runs on the calling thread. +- **Default selection:** When `num_threads` is omitted (`None` in Python, `Threads::FromEnv` + in Rust, `0` in C), the library reads `SDCA_NUM_THREADS` once and + otherwise uses automatic threading, capped by the active + OpenMP team (`torch.set_num_threads`) or `OMP_NUM_THREADS` and divided across concurrent + calls. Passing `1` runs on the calling thread. - **Work threshold:** A call times its first rows on the calling thread and spawns workers only when remaining work is estimated at $\ge 1\text{ ms}$ ($\ge 0.2\text{ ms}$ on an OpenMP team). -- **Training (`fit`):** Coordinate-ascent epochs are sequential; only duality-gap evaluations - and kernel validation run in parallel. +- **Training (`fit`):** Coordinate-ascent epochs are sequential; only duality-gap evaluations, + gap-safe screening passes and kernel validation run in parallel. - **Floating-point environment:** Every C ABI call executes with IEEE subnormals enabled and round-to-nearest (even if the caller set flush-to-zero, as XLA:CPU does) and restores the caller's environment on return. ## Environment variables -For the runtime-compiled extension switches (`SDCA_CPU_EXTENSION`, `SDCA_CUDA_LAUNCHER`, -`SDCA_JAX_FFI`), `auto` (default) compiles and caches the extension on first use when build -tools are available, or falls back with a one-time warning (`*.build-failed.txt` suppresses -retries across processes for 24 h); `on` (`1`) requires the extension and raises -`RuntimeError` on failure; `off` (`0`) disables it silently. +The runtime-compiled extension switches (`SDCA_CPU_EXTENSION`, `SDCA_CUDA_LAUNCHER`, +`SDCA_JAX_FFI`, `SDCA_JAX_CUDA`) take `auto` (default: compile and cache on first use, or +fall back with a one-time warning), `on` (`1`: raise `RuntimeError` if unavailable) or `off` +(`0`). A compiler error leaves `*.build-failed.txt` in the cache, which suppresses +retries for 24 hours (transient failures are not recorded); delete it to retry. The CUDA +switches fall back without a warning when Triton is newer than the verified range. | Variable | Default | Purpose | | --- | --- | --- | @@ -264,11 +288,13 @@ retries across processes for 24 h); `on` (`1`) requires the extension and raises | `SDCA_CPU_EXTENSION` | `auto` | `auto \| on \| off`: JIT-compiled PyTorch CPU C++ extension (`libsdca.torch.explain_backend` reports status) | | `SDCA_CUDA_LAUNCHER` | `auto` | `auto \| on \| off`: C++ `cuLaunchKernel` launcher for CUDA kernels (Triton 3.3–3.9) | | `SDCA_JAX_FFI` | `auto` | `auto \| on \| off`: XLA FFI custom-call handler for native JAX CPU losses | -| `SDCA_CACHE_DIR` | `~/.cache/libsdca` | Cache directory for compiled JAX FFI handlers | +| `SDCA_JAX_CUDA` | `auto` | `auto \| on \| off`: fused Triton kernels of JAX losses on CUDA GPUs (Triton 3.3–3.9; `off` keeps `jax.numpy`) | +| `SDCA_JAX_CPU_FUSION` | `auto` | `auto \| fenced \| unfenced`: fusion of the JAX entropy and distillation CPU passes, read when a loss is traced; `auto` times both once per machine ([JAX.md](../python/JAX.md#cpu-pass-fusion)) | +| `SDCA_CACHE_DIR` | `$XDG_CACHE_HOME/libsdca` (`~/.cache/libsdca`) | Cache directory for compiled JAX FFI handlers and `SDCA_JAX_CPU_FUSION=auto` decisions | | `TORCH_EXTENSIONS_DIR` | PyTorch cache | Cache directory for compiled PyTorch CPU and CUDA extensions | | `OMP_NUM_THREADS` | unset | Caps automatic threads when no OpenMP runtime is loaded | | `SDCA_LIBRARY_DIR` | unset | Override directory from which Python loads `libsdca_c.so` | -| `SDCA_BLAS_BACKEND` | `AUTO` | Build-time BLAS selection (`AUTO`, `EXTERNAL`, `BUILTIN`) for `pip install`, `python/build.py`, `rust/build.rs`, and CMake | +| `SDCA_BLAS_BACKEND` | `AUTO` | Build-time BLAS selection (`AUTO`, `EXTERNAL`, `BUILTIN`) for `pip install`, `python/build.py` and `rust/build.rs` (CMake: `-DSDCA_BLAS_BACKEND`) | | `SDCA_BLAS` | unset | Explicit external BLAS linker flags (e.g. `-lopenblas` or `-L/opt/blas/lib -lblas`) | | `SDCA_BUILD_NATIVE` | `ON` | Set to `OFF` during `pip install` to build a pure-Python wheel for tensor losses only | | `CXX`, `AR`, `CUDA_HOME` | system default | Compiler, archiver, and CUDA toolkit path for native and extension builds | @@ -282,12 +308,40 @@ CMake options (`cmake -D