diff --git a/python/numpy2x/_numpy2x.c b/python/numpy2x/_numpy2x.c index a9781a9..d02d5c8 100644 --- a/python/numpy2x/_numpy2x.c +++ b/python/numpy2x/_numpy2x.c @@ -56,6 +56,7 @@ static long long g_small = 2048; /* the runtime's sync_below: ops this small run /* Registered by _ndarray.py via _numpy2x.setup(). */ static PyTypeObject* g_array_cls = NULL; /* numpy2x.ndarray */ static PyObject* g_dtypes[7]; /* np.dtype per nx_dtype code */ +static PyObject* g_scalar_types[7]; /* its scalar type (np.float64, ...) */ static int g_alignment[7]; /* np.dtype(...).alignment per nx_dtype code */ static PyObject* g_matmul = NULL; /* np.matmul */ static PyObject* g_fallback = NULL; /* fallback(ufunc, method, inputs, kwargs) */ @@ -1590,6 +1591,14 @@ static int input_kind(PyObject* x) { if (PyComplex_CheckExact(x)) { return K_PYCOMPLEX; } + for (int c = 0; c < 7; ++c) { + /* NumPy's scalar of a numpy2x dtype (a[i] of a ready array, a full + * reduction): its type says the dtype, no attribute lookup and + * comparison of dtype objects (~0.4 us per op) */ + if ((PyObject*)Py_TYPE(x) == g_scalar_types[c]) { + return c; + } + } if (PyObject_TypeCheck(x, (PyTypeObject*)g_np_generic) || PyObject_TypeCheck(x, (PyTypeObject*)g_np_ndarray)) { PyObject* dt = PyObject_GetAttrString(x, "dtype"); @@ -1949,6 +1958,11 @@ static int defers(PyObject* other) { PyComplex_CheckExact(other) || PyBool_Check(other)) { return 0; } + for (int c = 0; c < 7; ++c) { + if ((PyObject*)Py_TYPE(other) == g_scalar_types[c]) { + return 0; /* NumPy's own scalar types never opt out: no attribute lookup */ + } + } PyObject* au = PyObject_GetAttrString((PyObject*)Py_TYPE(other), "__array_ufunc__"); if (au == NULL) { PyErr_Clear(); @@ -2204,6 +2218,7 @@ static PyMethodDef Array_methods[] = { }; static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs); +static PyObject* pending_item(PyObject* a, PyObject* key); static PyObject* m_basic_view(PyObject* self, PyObject* const* args, Py_ssize_t nargs); static int basic_layout(nx_array* x, PyObject* key, int* out_nd, int64_t* nshape, int64_t* nstrides, int64_t* out_offset); @@ -2219,6 +2234,13 @@ static PyObject* Array_subscript(PyObject* self, PyObject* key) { return r; /* NumPy's scalar, or NULL */ } Py_DECREF(r); + if (!nx_array_ready(((ArrayObject*)self)->a)) { + r = pending_item(self, key); + if (r != Py_None) { + return r; /* a pending 0-d copy, or NULL */ + } + Py_DECREF(r); + } r = m_basic_view(NULL, args, 2); if (r == NULL) { return NULL; @@ -2464,6 +2486,10 @@ static PyObject* m_setup(PyObject* self, PyObject* args) { Py_XSETREF(g_array_cls, (PyTypeObject*)Py_NewRef(cls)); for (int c = 0; c < 7; ++c) { Py_XSETREF(g_dtypes[c], Py_NewRef(PyTuple_GET_ITEM(dtypes, c))); + Py_XSETREF(g_scalar_types[c], PyObject_GetAttrString(g_dtypes[c], "type")); + if (g_scalar_types[c] == NULL) { + return NULL; + } PyObject* al = PyObject_GetAttrString(g_dtypes[c], "alignment"); g_alignment[c] = al != NULL ? (int)PyLong_AsLong(al) : -1; Py_XDECREF(al); @@ -3231,38 +3257,89 @@ static PyObject* m_scalar(PyObject* self, PyObject* a) { /* item(a, key): for an integer key (an int, or a tuple of one int per * dimension) the element as NumPy's scalar if `a` is ready; None if `a` is * pending or the key is something else. IndexError as NumPy's. */ -static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs) { - (void)self; - if (array_and_arg(args, nargs) < 0) { - return NULL; - } - PyObject *a = args[0], *key = args[1]; - nx_array* x = array_of(a); +/* The byte offset of the element an integer key (an int, or a tuple of one + * int per dimension) selects in x: 1, and *offset; 0 for any other key + * (arrays, slices, masks, np.bool_: the general path); -1 with NumPy's + * IndexError. */ +static int int_key_offset(nx_array* x, PyObject* key, int64_t* offset) { const int nd = nx_array_ndim(x); const Py_ssize_t nk = PyTuple_Check(key) ? PyTuple_GET_SIZE(key) : 1; - if (nk != nd || nd == 0 || !nx_array_ready(x)) { - Py_RETURN_NONE; + if (nk != nd || nd == 0) { + return 0; } const int64_t* shape = nx_array_shape(x); const int64_t* strides = nx_array_strides(x); - int64_t offset = 0; + *offset = 0; for (int i = 0; i < nd; ++i) { PyObject* k = PyTuple_Check(key) ? PyTuple_GET_ITEM(key, i) : key; if (PyBool_Check(k) || !(PyLong_Check(k) || (g_np_integer != NULL && PyObject_TypeCheck(k, (PyTypeObject*)g_np_integer)))) { - Py_RETURN_NONE; /* arrays, slices, masks, np.bool_: the general path */ + return 0; } const Py_ssize_t v = PyNumber_AsSsize_t(k, PyExc_IndexError); if (v == -1 && PyErr_Occurred()) { - return NULL; + return -1; } if (v < -shape[i] || v >= shape[i]) { PyErr_Format(PyExc_IndexError, "index %zd is out of bounds for axis %d with size %lld", v, i, (long long)shape[i]); - return NULL; + return -1; } - offset += (v < 0 ? v + shape[i] : v) * strides[i]; + *offset += (v < 0 ? v + shape[i] : v) * strides[i]; + } + return 1; +} + +/* a[i, ...] of a pending array: NumPy's scalar, a snapshot, so a 0-d copy + * of the element (one placement task, no view object), marked `_scalar`. + * None if the key is no integer key or the copy is unsupported. */ +static PyObject* pending_item(PyObject* a, PyObject* key) { + nx_array* x = array_of(a); + int64_t offset = 0; + const int k = int_key_offset(x, key, &offset); + if (k <= 0) { + return k < 0 ? NULL : Py_NewRef(Py_None); + } + const nx_dtype dt = nx_array_dtype(x); + int64_t zero = 0; + nx_array* e = nx_array_view(g_rt, x, dt, 0, &zero, &zero, offset); + if (e == NULL) { + Py_RETURN_NONE; + } + const int part_ndim = 0; + nx_array* r = NULL; + char err[ERRLEN] = ""; + const int st = nx_place(g_rt, dt, 0, &zero, NULL, 1, &e, &part_ndim, &zero, &zero, &zero, &r, + issue_site(), err, sizeof err); + nx_array_release(e); + if (st == NX_UNSUPPORTED) { + Py_RETURN_NONE; + } + if (raise_status(st, err) < 0) { + return NULL; + } + PyObject* res = wrap_array(r); + if (res != NULL) { + ((ArrayObject*)res)->scalar = 1; + } + return res; +} + +static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs) { + (void)self; + if (array_and_arg(args, nargs) < 0) { + return NULL; + } + PyObject *a = args[0], *key = args[1]; + nx_array* x = array_of(a); + if (!nx_array_ready(x)) { + Py_RETURN_NONE; + } + int64_t offset = 0; + const int k = int_key_offset(x, key, &offset); + if (k <= 0) { + return k < 0 ? NULL : Py_NewRef(Py_None); } int64_t zero = 0; nx_array* e = nx_array_view(g_rt, x, nx_array_dtype(x), 0, &zero, &zero, offset); diff --git a/python/tests/test_indexing.py b/python/tests/test_indexing.py index 82a1f62..7e43d0f 100644 --- a/python/tests/test_indexing.py +++ b/python/tests/test_indexing.py @@ -209,6 +209,103 @@ def test_scalar_indexing_of_pending_arrays_does_not_sync(): assert isinstance(ready[0, 1], np.float64) # ready data: NumPy's scalar +def test_scalar_indexing_of_pending_arrays_all_dtypes_and_keys(): + rng = np.random.default_rng(11) + for dt in ("?", "i4", "i8", "f4", "f8", "c8", "c16"): + x = (rng.standard_normal((5, 7)) * 9).astype(dt) + e = np.asarray(np.positive(x) if dt != "?" else np.logical_not(np.logical_not(x))) + v = numpy2x.asarray(x) + y = np.positive(v) if dt != "?" else np.logical_not(np.logical_not(v)) + for key in [(1, 2), (-1, -7), (4, 0), (np.int64(2), np.int32(-3)), (0, 6)]: + s = y[key] + if not numpy2x._numpy2x.ready(y): + assert type(s) is numpy2x.ndarray and s.ndim == 0 and s._scalar + got = np.asarray(s)[()] if isinstance(s, numpy2x.ndarray) else s + assert np.asarray(got).dtype == e.dtype + assert np.asarray(got).tobytes() == np.asarray(e[key]).tobytes() + for key in [(5, 0), (0, -8)]: + with pytest.raises(IndexError) as got: + (np.positive(v) if dt != "?" else np.logical_not(v))[key] + with pytest.raises(IndexError) as expected: + x[key] + assert str(got.value) == str(expected.value) + w = numpy2x.asarray(np.arange(6.0)) * 2.0 # 1-d, an int key + assert float(w[3]) == 6.0 and float(w[-1]) == 10.0 + + +def test_scalar_indexing_of_a_pending_array_is_one_task_and_no_sync(): + from async_path import SIZE, assert_async, deterministic, pending + + # 1e6 elements behind 20 copies: tens of milliseconds of work, so y is + # still pending when y[17] is issued (at SIZE the chain could finish + # first on a fast runner, and y[17] then took the ready path: no task) + x = np.random.default_rng(12).standard_normal(200 * SIZE) + with deterministic(): + y = pending(numpy2x.asarray(x)) + s = assert_async(lambda: y[17], ops=1) + assert s._scalar + y[17] = 0.0 # s is a snapshot + assert float(s) == x[17] + + +def test_cumulative_loop_over_columns_is_bit_identical(): + # ES[i] = ES[i - 1] + reduction: pending elements read, added and stored + rng = np.random.default_rng(13) + for n, dt in [(200, "f8"), (150, "f4")]: + a = rng.standard_normal((n, n)).astype(dt) + + def run(xp, A): + ES = xp.zeros(A.shape[1], dtype=dt) + for i in range(A.shape[1]): + ES[i] = (ES[i - 1] if i > 0 else 0) + xp.abs(xp.sin(A[:, i])).sum() + return np.asarray(ES) + + got = run(np, numpy2x.asarray(a)) + expected = run(np, a) + assert got.dtype == expected.dtype + assert got.tobytes() == expected.tobytes() + + +def test_numpy_scalars_of_every_dtype_as_operands(): + rng = np.random.default_rng(14) + for dt in ("i4", "i8", "f4", "f8", "c8", "c16", "?"): + x = (rng.standard_normal(3000) * 5).astype(dt) + s = x[7] # NumPy's scalar of that dtype (strong, NEP 50) + v = numpy2x.asarray(x) + for op in (np.add, np.multiply, np.not_equal): + if dt == "?" and op is not np.not_equal: + continue + for got, expected in [(op(v, s), op(x, s)), (op(s, v), op(s, x))]: + g = np.asarray(got) + assert g.dtype == expected.dtype + assert g.tobytes() == expected.tobytes() + if dt in ("f4", "f8"): + g = np.asarray(v + np.float64(0.1)) # another dtype: promoted as NumPy + e = x + np.float64(0.1) + assert g.dtype == e.dtype and g.tobytes() == e.tobytes() + + +def test_zero_d_ops_on_pending_operands(): + # the 0-d fast path of vp_ufunc_issue: pending 0-d inputs of the loop's + # dtypes, a 0-d result (values, dtypes, FP errors as NumPy) + rng = np.random.default_rng(15) + for dt in ("i4", "i8", "f4", "f8", "c8", "c16"): + x = (rng.standard_normal(4000) * 3).astype(dt) + v = numpy2x.asarray(x) + a, b = np.positive(v)[5], np.negative(v)[9] # pending 0-d elements + ea, eb = x[5], -x[9] + for f in (np.add, np.subtract, np.multiply, np.maximum, np.equal): + if dt[0] == "c" and f is np.maximum: + continue + got, expected = f(a, b), f(ea, eb) + g = np.asarray(got) + assert g.dtype == np.asarray(expected).dtype + assert g.tobytes() == np.asarray(expected).tobytes() + z = np.positive(numpy2x.asarray(np.zeros(3000)))[0] # pending 0.0 + with np.errstate(divide="raise"), pytest.raises(FloatingPointError): + float(1.0 / z) # raised at the next check or at the sync point + + def test_where_runs_as_a_select_task(): rng = np.random.default_rng(8) f0 = fallbacks() diff --git a/src/capi/capi.cpp b/src/capi/capi.cpp index c7db360..9d0ca5c 100644 --- a/src/capi/capi.cpp +++ b/src/capi/capi.cpp @@ -37,14 +37,21 @@ struct Graveyard { std::mutex mutex; std::vector objects; std::vector> releases; // kernels' Python state + // Set under the mutex when something is parked: drain() runs at every C + // ABI call and took the mutex (a cache line the workers write) each time. + std::atomic nonempty{false}; void drain() { + if (!nonempty.load(std::memory_order_acquire)) { + return; + } std::vector dead; std::vector> todo; { std::lock_guard lock(mutex); dead.swap(objects); todo.swap(releases); + nonempty.store(false, std::memory_order_relaxed); } for (PyObject* o : dead) { Py_DECREF(o); @@ -66,6 +73,7 @@ numpy::DeferRelease defer_release(const std::shared_ptr& graveyard) { } else if (auto g = weak.lock()) { std::lock_guard lock(g->mutex); g->releases.push_back(std::move(f)); + g->nonempty.store(true, std::memory_order_release); } }; } @@ -80,6 +88,7 @@ std::shared_ptr python_owner(const std::shared_ptr& graveyard, } else if (auto g = weak.lock()) { std::lock_guard lock(g->mutex); g->objects.push_back(o); + g->nonempty.store(true, std::memory_order_release); } // else: runtime gone, interpreter shutting down; leak }); } @@ -614,8 +623,24 @@ int nx_ufunc_issue(nx_runtime* rt, const nx_loop* loop, nx_array* const* inputs, set_error(err, errlen, "too many inputs"); return NX_UNSUPPORTED; } + bool scalar = out_ndim == 0 && out == nullptr; for (int i = 0; i < nin; ++i) { ins[i] = ref(inputs[i]); + scalar = scalar && ins[i]->layout().shape.empty() && + ins[i]->layout().dtype == loop->dtypes[static_cast(i)]; + } + if (scalar) { + // 0-d inputs of the loop's dtypes, a 0-d result (`s + t`, `x[i - 1] + s` + // in loops): one loop call over one element, as NumPy's trivial loop + // makes it. The layout is a constant, the kernel the loop's shared one: + // no order-K layout, no single-call check, no cast check. + Layout l = Layout::contiguous(loop->dtypes[static_cast(nin)], {}); + ArrayPtr res; + rt->rt->issue(loop->loop->elementwise_kernel(), + std::span(ins, static_cast(nin)), {}, + std::span(&l, 1), site, std::span(&res, 1)); + *result = hand_out(std::move(res)); + return NX_OK; } if (!cast_small_inputs(rt, *loop, std::span(ins, static_cast(nin)), site, err, errlen)) { diff --git a/src/kernels/numpy_ufunc.cpp b/src/kernels/numpy_ufunc.cpp index eaaebab..4fe3512 100644 --- a/src/kernels/numpy_ufunc.cpp +++ b/src/kernels/numpy_ufunc.cpp @@ -1253,6 +1253,28 @@ ArrayPtr wrap_ndarray(Runtime& rt, PyObject* obj, std::string& error, std::share } ArrayPtr scalar_array(Runtime& rt, PyObject* value, DType dtype) { + // A NumPy scalar of exactly this dtype (a[i] of a ready array, a full + // reduction) or a Python float as float64: its bytes as they are. A new + // object per op misses the bridge's scalar cache, and NumPy's conversion + // builds and frees an array (~1 µs per op in a loop adding a[i - 1]). + const Layout l0 = Layout::contiguous(dtype, {}); + if (PyFloat_CheckExact(value) && dtype == DType::Float64) { + auto buffer = make_pooled(l0.nbytes()); + const double v = PyFloat_AS_DOUBLE(value); + std::memcpy(buffer->data(), &v, sizeof v); + return rt.wrap(Layout(l0), std::move(buffer)); + } + if (PyArray_IsScalar(value, Generic)) { + PyArray_Descr* d = PyArray_DescrFromScalar(value); + const bool same = d != nullptr && d->type_num == type_num(dtype) && + PyDataType_ISNOTSWAPPED(d); + Py_XDECREF(d); + if (same) { + auto buffer = make_pooled(l0.nbytes()); + PyArray_ScalarAsCtype(value, buffer->data()); + return rt.wrap(Layout(l0), std::move(buffer)); + } + } PyArray_Descr* descr = PyArray_DescrFromType(type_num(dtype)); PyObject* arr = PyArray_FromAny(value, descr, 0, 0, 0, nullptr); // steals descr if (arr == nullptr) { diff --git a/src/runtime/task.hpp b/src/runtime/task.hpp index 61da666..9d41ef9 100644 --- a/src/runtime/task.hpp +++ b/src/runtime/task.hpp @@ -448,7 +448,9 @@ class Task { struct Copy { std::shared_ptr from, to; }; - std::vector copies_; // copy-on-write, empty for most tasks + // Copy-on-write, empty for most tasks; inline: a write into an array a + // pending task reads (`x[i] = x[i - 1] + s`) did a heap allocation per op. + InlineVector copies_; // The consumers registered for this task's completion: a lock-free stack // of ConsumerNodes (the main thread pushes with a CAS), closed for good