From 058b30a81979b3caf00b1a13a3330d6bcbe36a0b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 08:17:34 +0000 Subject: [PATCH 1/4] Release: the version is set once, in CMakeLists.txt; tags must match it tests/test_version.c compares nx_version() with numpy2x_VERSION from CMake instead of a literal. The wheels workflow gets a first job that fails a tag run unless the tag is v; wheels and sdist wait for it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JT2yHm85oFjgf1AcgGLUux --- .github/workflows/wheels.yml | 18 ++++++++++++++++++ tests/CMakeLists.txt | 2 ++ tests/test_version.c | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0ae9597..14f47dc 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -15,7 +15,24 @@ on: workflow_dispatch: jobs: + # A release tag must name the version that CMakeLists.txt sets (project()), + # which the wheels, the sdist and numpy2x.__version__ all take. + version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Tag matches the version in CMakeLists.txt + if: startsWith(github.ref, 'refs/tags/v') + shell: bash + run: | + v=$(sed -nE 's/^project\(numpy2x VERSION ([^ ]+).*/\1/p' CMakeLists.txt) + if [ "v$v" != "$GITHUB_REF_NAME" ]; then + echo "::error::tag $GITHUB_REF_NAME does not match version $v in CMakeLists.txt" + exit 1 + fi + wheels: + needs: version strategy: fail-fast: false matrix: @@ -36,6 +53,7 @@ jobs: path: wheelhouse/*.whl sdist: + needs: version runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a5800d9..6bbdcdb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,7 @@ add_executable(test_version test_version.c) target_link_libraries(test_version PRIVATE numpy2x) +# The version is set once, in project() of the top-level CMakeLists.txt. +target_compile_definitions(test_version PRIVATE NX_EXPECTED_VERSION="${numpy2x_VERSION}") add_test(NAME version COMMAND test_version) # Verifies that NumPy's inner loops can be called from worker threads without the GIL. diff --git a/tests/test_version.c b/tests/test_version.c index a39dc02..f66807d 100644 --- a/tests/test_version.c +++ b/tests/test_version.c @@ -5,7 +5,7 @@ int main(void) { const char* v = nx_version(); - if (v == NULL || strcmp(v, "1.0.0") != 0) { + if (v == NULL || strcmp(v, NX_EXPECTED_VERSION) != 0) { fprintf(stderr, "unexpected version: %s\n", v ? v : "(null)"); return 1; } From 6fb6e30d8a2a10ef55caf2b20f07786b07e8ddf2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 08:20:38 +0000 Subject: [PATCH 2/4] docs/development.md: layout, rules for changes, releases Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JT2yHm85oFjgf1AcgGLUux --- docs/development.md | 72 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/docs/development.md b/docs/development.md index ac19136..7107725 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,22 @@ # Development +How to build, test, change and release numpy2x. + +## Layout + +``` +include/numpy2x/ public C ABI (numpy2x.h), the only interface the Python bridge uses +src/runtime/ runtime core: arrays, tasks, stage events, workers, element pool +src/kernels/ adapters that call NumPy's loops (ufuncs, reductions, matmul, gufuncs, sort) +src/capi/ implementation of the C ABI +python/numpy2x/ Python package (_numpy2x.c: per-op hot path; Python fallbacks and wrappers) +python/tests/ pytest, every result compared with NumPy +python/benchmarks/ benchmark scripts and suite +tests/ C and C++ tests +docker/ Linux, Windows and Wine images +.github/workflows/ ci.yml (build and test on every push and PR), wheels.yml (wheels, releases) +``` + ## Build and test `dev.cmd` (Windows, also by double-click) and `./dev.sh` (Linux) pull, build into `.venv` and @@ -43,3 +60,58 @@ docker build -f docker/linux/Dockerfile -t numpy2x:linux . docker run --rm numpy2x:linux # all test suites docker run --rm -v "$PWD:/work" numpy2x:linux python prog.py # run a program with numpy2x ``` + +## Rules for changes + +- **Results stay bit-identical to NumPy**: values, dtypes, shapes, strides, flags, warnings and + errors. numpy2x is a drop-in: no numpy2x-specific errors or results, no own kernels, no + reordered sums, no fusion. What cannot run asynchronously with NumPy's semantics falls back + to NumPy. +- **Two worker threads** (`worker_threads()` in `src/runtime/workers.cpp`), fixed: no option, + environment variable or API argument for it. +- Every new operation gets a test against NumPy and a test that it runs asynchronously + (`python/tests/async_path.py`). +- Changes to kernels, the issue path or the bridge's per-op code are performance changes: + measure them against `main` and give the numbers in the pull request. +- Linux and Windows behave identically and run the same tests. +- The Python bridge reaches the runtime only through `include/numpy2x/numpy2x.h` (C ABI, `nx_` + prefix); C++ lives in `namespace numpy2x`; Python is included via `src/numpy2x_python.h`. +- English for code, comments, docs and commit messages; format C/C++ with `clang-format -i` + (`.clang-format`). Keep `README.md` short and its numbers measured + (`python/benchmarks/readme_examples.py`). +- Build and run the tests before every commit; changes reach `main` through pull requests, + merged when `ci` is green. + +## Releases + +The version is set in one place: `project(numpy2x VERSION x.y.z ...)` in `CMakeLists.txt`. +`pyproject.toml`, `numpy2x.__version__`, `nx_version()` and the version test take it from there. +Numbers follow semantic versioning: `x.y.z+1` for fixes, `x.y+1.0` for new features, `x+1.0.0` +for incompatible changes. + +1. Change the version in `CMakeLists.txt` in a pull request and merge it once `ci` is green. +2. Tag the merge commit with `v` and the version, and push the tag: + + ```bash + git checkout main && git pull + git tag -a v1.0.1 -m "numpy2x 1.0.1" + git push origin v1.0.1 + ``` + +3. The tag starts the `wheels` workflow (`.github/workflows/wheels.yml`): + - it stops at once unless the tag matches the version in `CMakeLists.txt`; + - it builds the wheels (CPython 3.12 and 3.13; Windows x64, Linux x86_64 manylinux) and the + sdist, and tests every wheel with the full pytest suite; + - it creates the GitHub Release for the tag, with the wheels and the sdist attached; + - it uploads them to PyPI (trusted publishing, environment `pypi`; only while the repository + variable `PYPI_PUBLISH` is `true`). +4. Add a title and release notes to the GitHub Release. + +PyPI accepts each version only once: a broken release is fixed with a new version, never by +uploading the same one again. If the PyPI step failed, run the workflow again on the tag +(Actions, `wheels`, Run workflow, "Use workflow from": Tags, the tag); a run from a branch +does not upload. Every tagged release is a Released Version under the patent license +([PATENTS](../PATENTS)); branches and commits between releases are not. + +A new Python version needs an entry in `[tool.cibuildwheel] build` in `pyproject.toml` and in +the matrix of `ci.yml`; until then, pip builds numpy2x from the sdist on it. From b05c2072448b801a011b08d5de9e7570ba66cc00 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 08:22:03 +0000 Subject: [PATCH 3/4] docs/development.md: how it works, CI, licenses; releasing in more detail Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JT2yHm85oFjgf1AcgGLUux --- docs/development.md | 169 +++++++++++++++++++++++++++----------------- 1 file changed, 105 insertions(+), 64 deletions(-) diff --git a/docs/development.md b/docs/development.md index 7107725..20a9b6f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,20 +1,38 @@ # Development -How to build, test, change and release numpy2x. +How numpy2x is built, tested, measured and released. For using numpy2x, see the +[README](../README.md) and [testing.md](testing.md). -## Layout +## How it works + +numpy2x intercepts NumPy operations (through NumPy's array protocols and, with +`python -m numpy2x`, its own module) and issues each one as a task. The Python thread only +issues tasks and runs ahead; two worker threads run each task as soon as its inputs are +computed. Every task calls the inner loops of the installed NumPy (ufunc, reduction, gufunc and +sort loops), exactly as NumPy calls them, so results are bit-identical to NumPy. Whatever numpy2x +does not run itself falls back to NumPy, after waiting for the data it reads. + +## Repository layout ``` -include/numpy2x/ public C ABI (numpy2x.h), the only interface the Python bridge uses -src/runtime/ runtime core: arrays, tasks, stage events, workers, element pool -src/kernels/ adapters that call NumPy's loops (ufuncs, reductions, matmul, gufuncs, sort) -src/capi/ implementation of the C ABI -python/numpy2x/ Python package (_numpy2x.c: per-op hot path; Python fallbacks and wrappers) -python/tests/ pytest, every result compared with NumPy -python/benchmarks/ benchmark scripts and suite -tests/ C and C++ tests -docker/ Linux, Windows and Wine images -.github/workflows/ ci.yml (build and test on every push and PR), wheels.yml (wheels, releases) +include/numpy2x/numpy2x.h C interface of the runtime; the Python bridge uses only this +src/runtime/ runtime: arrays, tasks, worker pool, element memory pool, + copy-on-write, layouts; no Python dependency + (workers.cpp: the number of worker threads) +src/kernels/ adapters that call NumPy's loops (element-wise, reductions, + matmul, gufuncs, sort) +src/capi/capi.cpp implementation of the C interface +python/numpy2x/ Python package: _numpy2x.c (C extension, the per-operation hot + path), _ndarray.py (array class, fallbacks), _ufuncs.py, _linalg.py, + _fft.py, _join.py, _indexing.py, _compose.py, _sort.py +python/_numpy2x_hook.py `import numpy` gives numpy2x in user code (python -m numpy2x, + NUMPY2X=1 through numpy2x.pth) +python/tests/ pytest suite: every test compares with NumPy +python/benchmarks/ README examples, benchmark suite, shape maps +tests/ C/C++ tests (CTest) +docker/ Linux, Windows and Wine images +tools/dev.ps1, dev.cmd, dev.sh one-command build and test +.github/workflows/ ci.yml (build + tests), wheels.yml (wheels, releases, PyPI) ``` ## Build and test @@ -25,6 +43,10 @@ run the tests; run programs with `.venv\Scripts\python -m numpy2x prog.py` (Linu `dev.cmd -NoPull -NoTest -Python `, `./dev.sh --no-pull --no-test`, `PYTHON=python3.13 ./dev.sh`. +Requirements: Python >= 3.12 (64 bit), NumPy >= 2.5, CMake >= 3.24 and a C++20 compiler +(Visual Studio 2022 with the C++ workload on Windows, GCC or Clang on Linux); pip fetches CMake +and Ninja itself. + With CMake directly: ```bash @@ -37,8 +59,29 @@ PYTHONPATH=build/python python3 -m pytest -q python/tests `pip wheel .` builds a wheel (`numpy2x-*.whl`) to install on other machines with the same platform, Python version and architecture. -Every test compares numpy2x with NumPy running the same code: results must be bit-identical -(values, dtypes, shapes, strides, warnings, errors). +## Rules for changes + +- **Bit-identical to NumPy.** Every result must equal plain NumPy's: values, dtypes, shapes, + strides, warnings and errors. Never trade this for speed: no own kernels, no reordered sums. +- **Test against NumPy.** Every new operation needs a test that runs the same code with numpy2x + and with NumPy and compares the results, and a test that it actually runs asynchronously + (`python/tests/async_path.py`: `assert_async`). +- **Measure performance changes.** Changes to kernels, the issue path or the per-operation code + are performance changes: compare with `main` (benchmark suite, cases the change targets) + before merging. +- **Linux and Windows behave the same.** Platform-specific code stays behind small functions + with one implementation per platform; CI tests both. +- **Two worker threads, single-stage execution.** The number of worker threads + (`src/runtime/workers.cpp`) and the execution model (a task starts when its inputs are + completely computed) are conditions of the patent license ([PATENTS](../PATENTS)). Changes to + either are not covered by that license; see "Licenses" below. +- Formatting: `.clang-format` (C/C++). Code, comments and commit messages in English. + +## Continuous integration + +`ci.yml` runs on every push to `main` and on pull requests: pip build and the pytest suite on +Linux (GCC) and Windows (MSVC) with Python 3.12 and 3.13, and `tools/dev.ps1` as started from +Visual Studio's developer prompt. Pull requests are merged only with green CI. ## Benchmarks @@ -48,7 +91,9 @@ PYTHONPATH=build/python python3 python/benchmarks/suite.py [--quick] # ~100 pr PYTHONPATH=build/python python3 python/benchmarks/array_pipelining.py ``` -Use a Release build. Reports go to `./reports` (untracked). +Use a Release build and an otherwise idle machine. Reports go to `./reports` (untracked). +`readme_examples.py` replaces the committed results in `python/benchmarks/results/readme` and +the charts in `docs/images`; commit them only from a run whose results are all bit-identical. ## Docker @@ -61,57 +106,53 @@ docker run --rm numpy2x:linux # all test suites docker run --rm -v "$PWD:/work" numpy2x:linux python prog.py # run a program with numpy2x ``` -## Rules for changes +## Releasing + +The version is set in one place: `project(numpy2x VERSION x.y.z ...)` in the top-level +`CMakeLists.txt`. The package version (pyproject reads it), `numpy2x.__version__` and the version +test all take it from there. -- **Results stay bit-identical to NumPy**: values, dtypes, shapes, strides, flags, warnings and - errors. numpy2x is a drop-in: no numpy2x-specific errors or results, no own kernels, no - reordered sums, no fusion. What cannot run asynchronously with NumPy's semantics falls back - to NumPy. -- **Two worker threads** (`worker_threads()` in `src/runtime/workers.cpp`), fixed: no option, - environment variable or API argument for it. -- Every new operation gets a test against NumPy and a test that it runs asynchronously - (`python/tests/async_path.py`). -- Changes to kernels, the issue path or the bridge's per-op code are performance changes: - measure them against `main` and give the numbers in the pull request. -- Linux and Windows behave identically and run the same tests. -- The Python bridge reaches the runtime only through `include/numpy2x/numpy2x.h` (C ABI, `nx_` - prefix); C++ lives in `namespace numpy2x`; Python is included via `src/numpy2x_python.h`. -- English for code, comments, docs and commit messages; format C/C++ with `clang-format -i` - (`.clang-format`). Keep `README.md` short and its numbers measured - (`python/benchmarks/readme_examples.py`). -- Build and run the tests before every commit; changes reach `main` through pull requests, - merged when `ci` is green. - -## Releases - -The version is set in one place: `project(numpy2x VERSION x.y.z ...)` in `CMakeLists.txt`. -`pyproject.toml`, `numpy2x.__version__`, `nx_version()` and the version test take it from there. -Numbers follow semantic versioning: `x.y.z+1` for fixes, `x.y+1.0` for new features, `x+1.0.0` -for incompatible changes. - -1. Change the version in `CMakeLists.txt` in a pull request and merge it once `ci` is green. -2. Tag the merge commit with `v` and the version, and push the tag: +1. On `main`, change the version in `CMakeLists.txt` and merge it (commit message + `numpy2x x.y.z`). CI must be green. Versions follow semantic versioning: x.y.(z+1) for fixes, + x.(y+1).0 for new features, (x+1).0.0 for incompatible changes. +2. Tag that commit with the same version and push the tag: ```bash - git checkout main && git pull - git tag -a v1.0.1 -m "numpy2x 1.0.1" - git push origin v1.0.1 + git checkout main + git pull origin main + git tag -a vx.y.z -m "numpy2x x.y.z" + git push origin vx.y.z ``` -3. The tag starts the `wheels` workflow (`.github/workflows/wheels.yml`): - - it stops at once unless the tag matches the version in `CMakeLists.txt`; - - it builds the wheels (CPython 3.12 and 3.13; Windows x64, Linux x86_64 manylinux) and the - sdist, and tests every wheel with the full pytest suite; - - it creates the GitHub Release for the tag, with the wheels and the sdist attached; - - it uploads them to PyPI (trusted publishing, environment `pypi`; only while the repository - variable `PYPI_PUBLISH` is `true`). -4. Add a title and release notes to the GitHub Release. - -PyPI accepts each version only once: a broken release is fixed with a new version, never by -uploading the same one again. If the PyPI step failed, run the workflow again on the tag -(Actions, `wheels`, Run workflow, "Use workflow from": Tags, the tag); a run from a branch -does not upload. Every tagged release is a Released Version under the patent license -([PATENTS](../PATENTS)); branches and commits between releases are not. - -A new Python version needs an entry in `[tool.cibuildwheel] build` in `pyproject.toml` and in -the matrix of `ci.yml`; until then, pip builds numpy2x from the sdist on it. +3. The tag starts `wheels.yml`. It first checks that the tag matches the version in + `CMakeLists.txt` and stops otherwise; then it builds the wheels (CPython 3.12 and 3.13, Windows + x64 and Linux x86_64 manylinux) and the sdist, runs the full test suite on each wheel, creates + the GitHub Release `vx.y.z` with these files and uploads them to PyPI. +4. Add the title and release notes to the GitHub Release. +5. Check: `pip install numpy2x==x.y.z` in a fresh environment, `numpy2x.__version__`, and the + project page on https://pypi.org/project/numpy2x/. + +Notes: + +- PyPI accepts each version only once. A broken release is fixed with a new version, not by + uploading the same one again. +- The PyPI upload uses trusted publishing: the GitHub environment `pypi` and the repository + variable `PYPI_PUBLISH = true` (a repository variable, not a secret or an environment + variable) must exist; the PyPI project trusts `ILNumerics/numpy2x`, workflow `wheels.yml`, + environment `pypi`. +- If the upload was skipped or failed, rerun it for the existing tag: Actions → wheels → + Run workflow → "Use workflow from": Tags → the tag. A run on `main` builds and tests but + neither releases nor uploads. +- New Python versions need an entry in `[tool.cibuildwheel] build` in `pyproject.toml` (and + in the matrix of `ci.yml`); until then, users on them build from the sdist. +- Every tagged release is a "Released Version" under the patent license; branches and commits + between releases are not. + +## Licenses + +The code is under the Clear BSD License ([LICENSE](../LICENSE)); contributions are accepted +under the same license. numpy2x implements methods patented by ILNumerics; their use is licensed +by [PATENTS](../PATENTS). Versions that run more than two worker threads per process, perform +staged execution or are adapted to other languages, array libraries or runtimes are not covered +by that license, also as forks. Third-party notices: [NOTICE](../NOTICE). Patents: +https://ilnumerics.net/patents.html. From 984eab77518a69a3ff7aeaefb61546090078f590 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 09:59:12 +0000 Subject: [PATCH 4/4] docs/development.md: leave the contribution terms out for now Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JT2yHm85oFjgf1AcgGLUux --- docs/development.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/development.md b/docs/development.md index 20a9b6f..a79213b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -150,9 +150,8 @@ Notes: ## Licenses -The code is under the Clear BSD License ([LICENSE](../LICENSE)); contributions are accepted -under the same license. numpy2x implements methods patented by ILNumerics; their use is licensed -by [PATENTS](../PATENTS). Versions that run more than two worker threads per process, perform -staged execution or are adapted to other languages, array libraries or runtimes are not covered -by that license, also as forks. Third-party notices: [NOTICE](../NOTICE). Patents: -https://ilnumerics.net/patents.html. +The code is under the Clear BSD License ([LICENSE](../LICENSE)). numpy2x implements methods +patented by ILNumerics; their use is licensed by [PATENTS](../PATENTS). Versions that run more than +two worker threads per process, perform staged execution or are adapted to other languages, array +libraries or runtimes are not covered by that license, also as forks. Third-party notices: +[NOTICE](../NOTICE). Patents: https://ilnumerics.net/patents.html.