Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
58 changes: 44 additions & 14 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
name: CI
on:
# Pushes to main or master and pull requests, each run once.
# Pushes to the main branches and pull requests, each run once.
push:
branches: [main, master]
branches: [main, master, integration]
pull_request:
# Manual runs choose the conformance cells: the CI profile or the full matrix (to record
# the expected failures of the runner's hardware; see tests/python/conformance/known.py).
workflow_dispatch:
inputs:
conformance:
description: Conformance cells to run
type: choice
options: [ci, full]
default: ci
permissions:
contents: read
concurrency:
Expand All @@ -18,12 +26,13 @@ jobs:
steps:
# Check out repository sources for static analysis and formatting checks.
- uses: actions/checkout@v7.0.1
# Set up Python 3.10 (the minimum supported version and basedpyright target).
# Set up Python 3.12: basedpyright targets 3.10 (pyproject.toml) but needs the type stubs
# of current NumPy and JAX, which Python 3.10 cannot install.
- uses: actions/setup-python@v7.1.0
with:
python-version: '3.10'
python-version: '3.12'
# Install pinned linters/formatters (ruff, clang-format, clang-tidy, basedpyright, lizard) and scientific stubs.
- run: python -m pip install -r requirements-dev.txt numpy scipy mpmath scikit-learn jax
- run: python -m pip install -r requirements-dev.txt numpy scipy mpmath matplotlib scikit-learn jax
# Install CPU PyTorch so basedpyright can resolve torch types across the package and tests.
- run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
# Run ruff, clang-format, basedpyright, version consistency, complexity ceiling, and clang-tidy.
Expand Down Expand Up @@ -118,6 +127,9 @@ jobs:
OPENBLAS_NUM_THREADS: '1'
OMP_NUM_THREADS: '1'
CXX: clang++-18
CONFORMANCE: ${{ inputs.conformance || 'ci' }}
# Each conformance cell with failures appends a record here, uploaded when a step fails.
SDCA_CONFORMANCE_RECORD: ${{ github.workspace }}/build/conformance
steps:
# Check out repository sources for Python, PyTorch, JAX, and scikit-learn test suites.
- uses: actions/checkout@v7.0.1
Expand All @@ -135,14 +147,24 @@ jobs:
- run: python tools/check_extension_builds.py --compiler g++ --compiler clang++-18
# Build the native C ABI shared library against the external BLAS backend.
- run: python python/build.py --blas-backend EXTERNAL
# Run all parallel Python test groups (core, torch, jax), skipping multi-minute stress tests.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group all -m 'not long'
# Run all parallel Python test groups (core, torch, jax), skipping multi-minute stress tests,
# with the CI profile of the conformance matrix unless a manual run asks for the full one.
- run: >-
PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python
--group all -m 'not long' --conformance "$CONFORMANCE"
# Run thread-scaling and timing tests that require an unloaded machine serially.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider tests/python --group all -m serial
# Compile and test the JIT-built PyTorch CPU C++ extension (SDCA_CPU_EXTENSION=on) with GCC.
- run: >-
CXX=g++ SDCA_CPU_EXTENSION=on PYTHONPATH=python python -m pytest -p no:cacheprovider
tests/python/test_softmax_limits.py::SoftmaxLimitTests tests/python/test_torch.py::TorchTests
# The conformance records of a failed run, for review against the known-failure manifest.
- uses: actions/upload-artifact@v7.0.2
if: failure()
with:
name: conformance-linux-${{ github.sha }}
path: build/conformance
if-no-files-found: ignore

python-versions:
name: Python ${{ matrix.python }} (core)
Expand All @@ -151,7 +173,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python: ['3.11', '3.13']
python: ['3.11', '3.13', '3.14']
env:
OPENBLAS_NUM_THREADS: '1'
OMP_NUM_THREADS: '1'
Expand All @@ -167,11 +189,14 @@ jobs:
# Build the native C ABI library with the built-in BLAS, as in the wheels.
- run: python python/build.py --blas-backend BUILTIN
# Run the core test group in parallel, skipping multi-minute stress tests.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core -m 'not long'
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core -m 'not long' --conformance ci

macos:
name: macOS (arm64)
runs-on: macos-15
env:
CONFORMANCE: ${{ inputs.conformance || 'ci' }}
SDCA_CONFORMANCE_RECORD: ${{ github.workspace }}/build/conformance
steps:
# Check out repository sources to test Apple Silicon (arm64), Apple Clang, and Accelerate BLAS.
- uses: actions/checkout@v7.0.1
Expand All @@ -192,11 +217,16 @@ jobs:
# Build the Python C ABI library linked against macOS Accelerate.
- run: CXX=c++ python python/build.py --blas-backend EXTERNAL
# Run the core NumPy and scikit-learn test suite in parallel across workers.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core
# Run the serial thread-scaling tests on macOS arm64.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider tests/python --group core -m serial
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group core --conformance "$CONFORMANCE"
# Run the PyTorch CPU test suite (including the Apple Clang CPU extension) in parallel.
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group torch -m 'not long'
- run: PYTHONPATH=python python -m pytest -p no:cacheprovider -n auto --dist worksteal tests/python --group torch -m 'not long' --conformance "$CONFORMANCE"
# The conformance records of a failed run, for review against the known-failure manifest.
- uses: actions/upload-artifact@v7.0.2
if: failure()
with:
name: conformance-macos-${{ github.sha }}
path: build/conformance
if-no-files-found: ignore

rust:
name: Rust
Expand Down Expand Up @@ -279,4 +309,4 @@ jobs:
py="$RUNNER_TEMP/env/bin/python" src="$RUNNER_TEMP/sdist"
SDCA_BLAS_BACKEND=BUILTIN "$py" -m pip install dist/*.tar.gz pytest scipy mpmath scikit-learn
mkdir "$src" && tar -xzf dist/*.tar.gz -C "$src" --strip-components=1
cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed
cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed --conformance ci
64 changes: 44 additions & 20 deletions .github/workflows/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ name: Packages
# Builds the sdist, the wheels and the crate archive. A manual run can upload
# the sdist and wheels to TestPyPI; a v* tag uploads them to PyPI, then the
# crate to crates.io. Uploads use Trusted Publishing in the GitHub environments
# testpypi, pypi and crates-io; the one-time setup is in CONTRIBUTING.md
# (Releases). Pull requests only build and test the artifacts: every upload
# job skips them.
# testpypi, pypi and crates-io. Pull requests only build and test the
# artifacts: every upload job skips them.
on:
pull_request:
workflow_dispatch:
Expand All @@ -27,24 +26,25 @@ jobs:
runs-on: ubuntu-24.04
steps:
# Check out repository sources to build and validate the release source distribution.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Set up Python 3.12 for building and testing the sdist.
- uses: actions/setup-python@v7.1.0
with:
python-version: '3.12'
# On tag pushes, verify that the git tag v<version> matches the version in pyproject.toml.
# On tag pushes, verify that the git tag is v<VERSION> and that every version copy agrees.
- if: github.ref_type == 'tag'
name: Check that the tag is v<package version>
name: Check that the tag is v<VERSION>
env:
TAG: ${{ github.ref_name }}
run: >-
python -c "import os, sys, tomllib;
v = 'v' + tomllib.load(open('pyproject.toml', 'rb'))['project']['version'];
sys.exit(None if os.environ['TAG'] == v else 'tag ' + os.environ['TAG'] + ' is not ' + v)"
run: |
test "$TAG" = "v$(cat VERSION)" || { echo "tag $TAG is not v$(cat VERSION)"; exit 1; }
python -c "import runpy; runpy.run_path('tools/quality.py')['check_versions']()"
# Install pinned PEP 517 build frontend and Twine metadata checker.
- run: python -m pip install build==1.6.1 twine==7.0.0
# Build the source distribution archive in dist/.
- run: python -m build --sdist
- name: Check shared CPU sources in the sdist
run: python tools/check_install_archives.py --source . dist/*.tar.gz
# Verify sdist metadata and long_description rendering for PyPI.
- run: python -m twine check --strict dist/*
# Install the sdist into an isolated venv outside the repo and run the core test suite.
Expand All @@ -54,7 +54,7 @@ jobs:
py="$RUNNER_TEMP/env/bin/python" src="$RUNNER_TEMP/sdist"
SDCA_BLAS_BACKEND=BUILTIN "$py" -m pip install dist/*.tar.gz pytest scipy mpmath scikit-learn
mkdir "$src" && tar -xzf dist/*.tar.gz -C "$src" --strip-components=1
cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed
cd "$RUNNER_TEMP" && "$py" -m pytest -p no:cacheprovider "$src/tests/python" --group core --installed --conformance ci
# Verify that the C++ sources shipped inside the sdist can JIT-compile the PyTorch CPU extension.
- name: Build the CPU extension from the sdist sources
run: |
Expand All @@ -78,9 +78,33 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
# Check out repository sources to build and test portable binary wheels.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-python@v7.1.0
with:
python-version: '3.12'
# Build py3-none platform wheels with built-in BLAS, repair with auditwheel/delocate, and run core tests.
- uses: pypa/cibuildwheel@v4.3.5
- name: Check shared CPU headers in the wheels
run: python tools/check_install_archives.py --source . wheelhouse/*.whl
# Install only each documented extra into a fresh environment and run README examples.
- name: Fresh wheel installs and literal README examples
run: |
python -m pip install uv
python tools/readme_smoke.py --wheel wheelhouse/libsdca-*.whl --output "$RUNNER_TEMP/readme-smoke" --cleanup
- name: Save fresh-install records
if: always()
uses: actions/upload-artifact@v7.0.2
with:
name: readme-smoke-${{ matrix.os }}
path: |
${{ runner.temp }}/readme-smoke/results.json
${{ runner.temp }}/readme-smoke/*/commands.log
${{ runner.temp }}/readme-smoke/*/packages.txt
${{ runner.temp }}/readme-smoke/*/example.py
${{ runner.temp }}/readme-smoke/*/precision.py
${{ runner.temp }}/readme-smoke/*/precision.json
${{ runner.temp }}/readme-smoke/*/cpu-dispatch.py
${{ runner.temp }}/readme-smoke/*/cpu-dispatch.json
# Validate wheel metadata and README formatting with Twine.
- run: pipx run --spec twine==7.0.0 twine check --strict wheelhouse/*
# Upload the platform wheel artifact for the TestPyPI / PyPI publish jobs.
Expand All @@ -94,7 +118,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
# Check out repository sources to package and test the standalone Rust crate archive.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Install stable Rust toolchain to package and test the crate.
- run: rustup toolchain install stable --profile minimal && rustup default stable
# Vendor the C++ headers and C ABI sources into rust/native/ and build the .crate archive.
Expand Down Expand Up @@ -123,13 +147,13 @@ jobs:
id-token: write
steps:
# Download the verified sdist and platform wheels built by earlier jobs.
- uses: actions/download-artifact@v8.0.2
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: dist-*
path: dist
merge-multiple: true
# Publish sdist and wheels to TestPyPI via OIDC Trusted Publishing.
- uses: pypa/gh-action-pypi-publish@v1.14.3
- uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
with:
repository-url: https://test.pypi.org/legacy/

Expand All @@ -145,13 +169,13 @@ jobs:
id-token: write
steps:
# Download the verified sdist and platform wheels built by earlier jobs.
- uses: actions/download-artifact@v8.0.2
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: dist-*
path: dist
merge-multiple: true
# Publish sdist and wheels to PyPI via OIDC Trusted Publishing.
- uses: pypa/gh-action-pypi-publish@v1.14.3
- uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3

crates-io:
name: Upload (crates.io)
Expand All @@ -165,18 +189,18 @@ jobs:
id-token: write
steps:
# Check out the tagged sources the verified crate archive was built from.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Install stable Rust to package and publish the crate.
- run: rustup toolchain install stable --profile minimal && rustup default stable
# Download the crate archive that the crate job tested outside the checkout.
- uses: actions/download-artifact@v8.0.2
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: crate
path: verified
# Repackage and check that the archive to publish is byte-identical to the tested one.
- run: rust/package.sh --no-verify && cmp verified/*.crate rust/target/package/libsdca-*.crate
# Exchange the GitHub OIDC token for a short-lived crates.io token, revoked after the job.
- uses: rust-lang/crates-io-auth-action@v1.0.6
- uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6
id: auth
# Publish the crate to crates.io via Trusted Publishing.
- run: cargo publish --allow-dirty
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
build*
!build_*.py
!python/build.py
!rust/build.rs
matlab/*.mex*
Expand Down
Loading
Loading