diff --git a/python/numpy2x/_ndarray.py b/python/numpy2x/_ndarray.py index 9e3f872..b7f4602 100644 --- a/python/numpy2x/_ndarray.py +++ b/python/numpy2x/_ndarray.py @@ -162,9 +162,56 @@ def asarray(obj, dtype=None, order=None, *, copy=None, like=None, **kwargs): return np.asarray(obj, dtype, order, copy=copy, like=like, **kwargs) if isinstance(obj, ndarray) and (dtype is None or obj.dtype == np.dtype(dtype)) and not copy: return obj + if (type(obj) is np.ndarray and order in (None, "K", "A") and copy is not True + and (dtype is None or obj.dtype == np.dtype(dtype))): + r = _import_strided(obj) + if r is not None: + return r return array(obj, dtype, copy=True, order=order or "K") +def _c_strides(a): + st, step = [], a.itemsize + for n in reversed(a.shape): + st.append(step) + step *= n + return tuple(reversed(st)) + + +def _import_strided(obj): + """asarray of an ndarray: NumPy returns the array itself, numpy2x a copy + with its strides where they matter (NumPy's loops take other paths on + other strides, and their results can differ in the last bit; later + results take their layouts from them): an empty array keeps them all + (no memory), a dense one (any axis order, also reversed axes and + extent-1 axes of any stride) all of them. A strided one (a stepped + view) is copied densely, its gaps closed, keeping reversed axes + reversed. None: the plain copy of array() has the same strides.""" + if obj.dtype not in _CODES: + return None + st = obj.strides + if obj.size == 0: + r = np.ndarray(obj.shape, obj.dtype, np.empty(0, obj.dtype), strides=st) + return _adopt(r) + if obj.flags.c_contiguous and st == _c_strides(obj): + return None # the common case: array() copies it as it is + lo = hi = 0 + for n, s in zip(obj.shape, st): + if s < 0: + lo += (n - 1) * s + else: + hi += (n - 1) * s + if hi - lo + obj.itemsize == obj.nbytes: # dense: the same strides + buf = np.empty(obj.nbytes, np.uint8) + r = np.ndarray(obj.shape, obj.dtype, buf, offset=-lo, strides=st) + r[...] = obj + return _adopt(r) + if not any(s < 0 for s in st): + return None + flip = tuple(slice(None, None, -1) if s < 0 else slice(None) for s in st) + return _adopt(obj[flip].copy(order="K")[flip]) + + def fresh(r, args=(), kwargs=None): """Result `r` of a NumPy function as numpy2x array: fresh memory is adopted without a copy, a view of other memory is copied first. Returned as is: @@ -433,12 +480,18 @@ def handler(a, axis=None, out=None, keepdims=_NV, *, where=_NV): _INTP = np.intp +_WIDE = (np.dtype(np.float64), np.dtype(np.complex128)) def _divide_by_count(r, n): """NumPy's true_divide(r, intp(n), out=r, casting='unsafe') (and, for 0-d results, r.dtype.type(r / n)): divided in the promoted dtype of r - and intp (float64, complex128), cast back to r's dtype.""" + and intp (float64, complex128), cast back to r's dtype. In place, as + NumPy: the result keeps r's layout (a reduction's extent-1 axes).""" + if r.ndim: + if r.dtype in _WIDE: # the division's own dtype: no cast + return np.true_divide(r, _INTP(n), out=r) + return np.true_divide(r, _INTP(n), out=r, casting="unsafe") q = np.true_divide(r, _INTP(n)) return q if q.dtype == r.dtype else q.astype(r.dtype) @@ -1082,7 +1135,7 @@ def _gather_fast(a, key): return None idx = asarray(idx if idx.dtype in _INDEX_DTYPES else idx.astype(np.intp)) if (idx.dtype not in _INDEX_DTYPES or not a.ndim or not a.size or not idx.size - or not a._c_contiguous): + or not a._c_ordered): # extent-1 axes too: NumPy orders by their strides return None nb = idx.ndim res = list(range(p)) + [-1 - d for d in range(nb)] + list(range(p + 1, a.ndim)) diff --git a/python/numpy2x/_numpy2x.c b/python/numpy2x/_numpy2x.c index b9dcefe..a9781a9 100644 --- a/python/numpy2x/_numpy2x.c +++ b/python/numpy2x/_numpy2x.c @@ -353,6 +353,7 @@ static int Array_set_warn_on_write(ArrayObject* self, PyObject* v, void* c) { } static int c_contiguous(const nx_array* a); +static int c_ordered(const nx_array* a); static PyObject* Array_get_size(ArrayObject* self, void* c) { (void)c; @@ -370,6 +371,11 @@ static PyObject* Array_get_c_contiguous(ArrayObject* self, void* c) { return PyBool_FromLong(c_contiguous(self->a)); } +static PyObject* Array_get_c_ordered(ArrayObject* self, void* c) { + (void)c; + return PyBool_FromLong(c_ordered(self->a)); +} + static PyObject* Array_get_base(ArrayObject* self, void* c) { (void)c; return Py_NewRef(self->base != NULL ? self->base : Py_None); @@ -776,6 +782,8 @@ static PyGetSetDef Array_getset[] = { "NumPy's flags object of the layout (no sync): a new snapshot per access.", NULL}, {"_c_contiguous", (getter)Array_get_c_contiguous, NULL, "C-contiguous, as NumPy's flag (no sync).", NULL}, + {"_c_ordered", (getter)Array_get_c_ordered, NULL, + "C order with C strides on every axis, also of extent 1 (no sync).", NULL}, {"_scalar", (getter)Array_get_scalar, (setter)Array_set_scalar, "0-d result NumPy would return as a scalar: in-place operators rebind, as on NumPy's " "scalars", @@ -1402,17 +1410,33 @@ static int c_contiguous(const nx_array* a) { const int64_t* strides = nx_array_strides(a); static const int64_t itemsizes[] = {1, 4, 8, 4, 8, 8, 16}; /* nx_dtype order */ int64_t step = itemsizes[nx_array_dtype(a)]; + int contiguous = 1; for (int i = nd; i-- > 0;) { if (shape[i] == 0) { - return 1; + return 1; /* no elements: contiguous whatever the strides */ } if (shape[i] != 1) { - if (strides[i] != step) { - return 0; - } + contiguous &= strides[i] == step; step *= shape[i]; } } + return contiguous; +} + +/* C order with every stride NumPy's C order gives, also along extent-1 axes + * (NumPy orders a result's axes by them). */ +static int c_ordered(const nx_array* a) { + const int nd = nx_array_ndim(a); + const int64_t* shape = nx_array_shape(a); + const int64_t* strides = nx_array_strides(a); + static const int64_t itemsizes[] = {1, 4, 8, 4, 8, 8, 16}; /* nx_dtype order */ + int64_t step = itemsizes[nx_array_dtype(a)]; + for (int i = nd; i-- > 0;) { + if (strides[i] != step) { + return 0; + } + step *= shape[i]; + } return 1; } @@ -1456,15 +1480,30 @@ static PyObject* elision_candidate(PyObject* const* inputs, nx_array* const* ins /* --- scalar shortcut ------------------------------------------------------ */ -/* An op on scalars only (ready 0-d numpy2x arrays, Python and NumPy scalars) - * has nothing to run asynchronously: NumPy computes it right here, on NumPy +/* An op on scalars only (ready 0-d numpy2x arrays standing for NumPy scalars, + * `_scalar`; Python and NumPy scalars) has nothing to run asynchronously: NumPy computes it right here, on NumPy * scalars, as the NumPy program would (its 0-d values are NumPy scalars: * reductions and scalar indexing return them, and ufuncs on 0-d arrays do * too). Its result is NumPy's: a NumPy scalar. Any n-d operand (even of one - * element) keeps the array path, so NumPy's output shapes stay as they + * element) and a plain 0-d array (NumPy's 0-d array) keep the array path, so NumPy's output shapes stay as they * are. Fills `out` (capacity `out_len` entries, at least `n`) with new * references and returns 1, or 0 if an operand is not a ready scalar. Call it * through SCALARS_OF, which passes the length of the `out` array. */ +static int end_borrow(nx_array* a, PyObject* view); + +/* Releases what scalars_of() filled in: borrowed views end their borrow. */ +static void release_scalars(PyObject* const* inputs, PyObject** out, Py_ssize_t n) { + for (Py_ssize_t i = 0; i < n; ++i) { + if (is_array(inputs[i]) && !((ArrayObject*)inputs[i])->scalar) { + if (end_borrow(((ArrayObject*)inputs[i])->a, out[i]) < 0) { + PyErr_WriteUnraisable(inputs[i]); + } + } else { + Py_DECREF(out[i]); + } + } +} + static int scalars_of(PyObject* const* inputs, Py_ssize_t n, PyObject** out, Py_ssize_t out_len) { if (n > MAXIN) { return 0; @@ -1489,11 +1528,15 @@ static int scalars_of(PyObject* const* inputs, Py_ssize_t n, PyObject** out, Py_ } for (Py_ssize_t i = 0; i < n; ++i) { PyObject* x = inputs[i]; - out[i] = is_array(x) ? nx_array_scalar(((ArrayObject*)x)->a) : Py_NewRef(x); + /* A stand-in for a NumPy scalar (`_scalar`) becomes that scalar; a + * plain 0-d array stays one (a borrowed view): NumPy computes on + * 0-d arrays with its array loops, on scalars partly with other + * code (complex64 square differs in the last bit). */ + out[i] = !is_array(x) ? Py_NewRef(x) + : ((ArrayObject*)x)->scalar ? nx_array_scalar(((ArrayObject*)x)->a) + : nx_array_borrow(((ArrayObject*)x)->a); if (out[i] == NULL) { - for (Py_ssize_t j = 0; j < i; ++j) { - Py_DECREF(out[j]); - } + release_scalars(inputs, out, i); return -1; } } @@ -1504,12 +1547,6 @@ static int scalars_of(PyObject* const* inputs, Py_ssize_t n, PyObject** out, Py_ #define SCALARS_OF(inputs, n, out) \ scalars_of((inputs), (n), (out), (Py_ssize_t)(sizeof(out) / sizeof((out)[0]))) -static void release_all(PyObject** objs, Py_ssize_t n) { - for (Py_ssize_t i = 0; i < n; ++i) { - Py_DECREF(objs[i]); - } -} - /* --- the ufunc fast path -------------------------------------------------- */ /* Input kinds in the dtype-resolution cache key. Arrays and NumPy scalars use @@ -1845,7 +1882,7 @@ static PyObject* Array_array_ufunc(PyObject* self, PyObject* const* args, Py_ssi } if (k > 0) { PyObject* r = PyObject_Vectorcall(args[0], sc, (size_t)n, NULL); - release_all(sc, n); + release_scalars(inputs, sc, n); return r; } } @@ -1932,7 +1969,7 @@ static PyObject* binop(const char* name, binaryfunc scalar_op, PyObject* a, PyOb if (k != 0) { PyObject* r = k > 0 ? scalar_op(sc[0], sc[1]) : NULL; if (k > 0) { - release_all(sc, 2); + release_scalars(in2, sc, 2); } return r; } @@ -1980,7 +2017,9 @@ static PyObject* unop(const char* name, unaryfunc scalar_op, PyObject* a) { const int k = SCALARS_OF(&a, 1, sc); if (k != 0) { PyObject* r = k > 0 ? scalar_op(sc[0]) : NULL; - Py_XDECREF(k > 0 ? sc[0] : NULL); + if (k > 0) { + release_scalars(&a, sc, 1); + } return r; } PyObject* ufunc = op_ufunc(name); @@ -4883,7 +4922,7 @@ static PyObject* UfuncProxy_vectorcall(PyObject* self, PyObject* const* args, si } if (k > 0) { PyObject* r = PyObject_Vectorcall(uf, sc, (size_t)n, NULL); - release_all(sc, n); + release_scalars(args, sc, n); return r; } PyObject* result = NULL; diff --git a/python/tests/test_matrix.py b/python/tests/test_matrix.py new file mode 100644 index 0000000..2011a70 --- /dev/null +++ b/python/tests/test_matrix.py @@ -0,0 +1,327 @@ +"""A parameter matrix against NumPy: every op runs on NumPy and on numpy2x +with the same inputs, and must give the same values (bit for bit), dtype, +shape, strides, warnings and errors. + +Dimensions: shapes (0-d scalars; empty arrays of +1, 2, 3 and 8 dimensions; vectors; matrices; 3-d and 8-d arrays, with +extent-1 axes), layouts (C, F, transposed, stepped, reversed), dtypes, +element values (NaN, +inf, -inf, -0.0, integer extremes), ready or +pending inputs (the direct path or a task: "sync / async"), and per op +group its own parameters (broadcasting pairs, axes and keepdims, casts +that overflow, indices and masks).""" + +import itertools +import warnings +import zlib + +import numpy as np +import pytest + +import numpy2x + +SHAPES = { + "0d": (), + "e1": (0,), + "e2": (2, 0), + "e3": (0, 3, 1), + "e8": (1, 2, 0, 1, 2, 1, 1, 2), + "v1": (1,), + "v7": (7,), + "v3000": (3000,), + "m34": (3, 4), + "m151": (1, 5, 1), + "t234": (2, 3, 4), + "t8": (2, 1, 2, 1, 2, 1, 2, 3), + "big": (60, 70), +} +VIEWS = { + "C": lambda a: a, + "F": None, # the base in F order + "T": lambda a: a.T, + "step": lambda a: a[..., ::2] if a.ndim else a, + "rev": lambda a: a[::-1] if a.ndim else a, +} +DTYPES = ["b1", "i4", "i8", "f4", "f8", "c8", "c16"] +MODES = ["ready", "pending"] + + +def values(rng, shape, dtype): + """Random elements of `dtype` with the special values mixed in.""" + n = int(np.prod(shape)) + dt = np.dtype(dtype) + if dt.kind == "b": + x = rng.integers(0, 2, n).astype(bool) + elif dt.kind == "i": + info = np.iinfo(dt) + x = rng.integers(-50, 50, n).astype(dt) + special = np.array([0, -1, 1, info.min, info.max, info.min + 1, info.max - 1], dt) + k = min(n // 3, len(special)) + x[rng.choice(n, k, replace=False)] = special[:k] if n else x[:0] + else: + real = (rng.standard_normal(n) * 10).astype(dt.char.lower() if dt.kind == "c" else dt) + special = np.array([np.nan, np.inf, -np.inf, -0.0, 0.0, 1e30, -1e30, 2.5, -2.5, 0.5]) + k = min(n // 3, len(special)) + if k: + real[rng.choice(n, k, replace=False)] = special[:k] + x = real + if dt.kind == "c": + imag = (rng.standard_normal(n) * 10).astype(real.dtype) + if k: + imag[rng.choice(n, k, replace=False)] = special[:k][::-1] + x = real + 1j * imag + x = x.astype(dt) + return x.reshape(shape) + + +def make(a, view, mode="ready"): + """The NumPy operand: `a` (C order), in F order or viewed. Pending + operands are copies made by np.where (to_numpy2x): their NumPy + counterpart is made the same way (np.where gives empty results zero + strides and extent-1 axes the strides of its iteration order).""" + if view == "F": + a = np.asfortranarray(a) + if mode == "pending": + a = np.where(True, a, a) + return a if view == "F" else VIEWS[view](a) + + +_COND = [] + + +def _pending_true(): + """A 0-d bool numpy2x array that is True but most likely still pending.""" + x = numpy2x.asarray(np.full(200_000, 0.5)) + return np.sin(np.sin(x)).max() < 2.0 + + +def to_numpy2x(a, base, view, mode): + """The numpy2x operand of the NumPy operand `a` = view(base): ready + (imported), or pending (an exact copy behind a slow op), then viewed.""" + if mode == "ready": + return numpy2x.asarray(a) + v = numpy2x.asarray(np.asfortranarray(base) if view == "F" else base) + v = np.where(_pending_true(), v, v) # exact, pending, order K + return v if view == "F" else VIEWS[view](v) + + +def same_bits(e, g): + e, g = np.asarray(e), np.asarray(g) + if e.dtype != g.dtype or e.shape != g.shape: + return False + if e.dtype.kind in "fc": # NaNs: any payload NumPy may produce + en, gn = np.isnan(e), np.isnan(g) + if not np.array_equal(en, gn): + return False + e, g = np.where(en, 0, e), np.where(gn, 0, g) + return np.ascontiguousarray(e).tobytes() == np.ascontiguousarray(g).tobytes() + + +def run(fn, args): + """fn(*args): (results, error, warnings), results synced.""" + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + try: + r = fn(*args) + rs = r if isinstance(r, tuple) else (r,) + rs = tuple(np.asarray(x) if isinstance(x, numpy2x.ndarray) else x for x in rs) + numpy2x.wait_all() + err = None + except Exception as ex: # noqa: BLE001 - compared below + rs, err = None, ex + try: + numpy2x.wait_all() + except Exception: # noqa: BLE001 - the same error, raised once more + pass + return rs, err, sorted({(x.category.__name__, str(x.message)) for x in w}) + + +def seed(key): + """A seed that does not change between runs (hash() of str does).""" + return zlib.crc32(repr(key).encode()) + + +def check(fn, np_args, vp_args, strides=True, values_=True, label=None): + """fn on NumPy and on numpy2x: the same results (values, dtype, shape, + strides), the same error (type and message) and the same warnings.""" + try: + _check(fn, np_args, vp_args, strides, values_) + except AssertionError as ex: + raise AssertionError(f"{label or fn.__name__}: {ex}") from None + + +def _check(fn, np_args, vp_args, strides, values_): + e, eerr, ew = run(fn, np_args) + g, gerr, gw = run(fn, vp_args) + if eerr is not None or gerr is not None: + assert eerr is not None and gerr is not None, (eerr, gerr, e, g) + assert isinstance(gerr, type(eerr)), (type(eerr), type(gerr), eerr, gerr) + assert str(gerr).startswith(str(eerr)), (str(eerr), str(gerr)) + return + assert len(e) == len(g) + for x, y in zip(e, g): + assert not isinstance(y, numpy2x.ndarray) + xa, ya = np.asarray(x), np.asarray(y) + assert xa.dtype == ya.dtype and xa.shape == ya.shape, (xa.dtype, ya.dtype, xa.shape, ya.shape) + if strides: + assert xa.strides == ya.strides, (xa.strides, ya.strides) + if values_: + assert same_bits(xa, ya), (xa, ya) + assert isinstance(x, np.generic) == isinstance(y, np.generic) or ya.ndim == 0 + assert ew == gw, (ew, gw) + + +def deviates(): + """Values are always compared (no optimization level computes others).""" + return False + + +def cases(*dims): + return list(itertools.product(*dims)) + + +# --- element-wise unary --------------------------------------------------- + +UNARY = { + "negative": np.negative, "absolute": np.absolute, "sqrt": np.sqrt, "exp": np.exp, + "sin": np.sin, "floor": np.floor, "rint": np.rint, "sign": np.sign, "isnan": np.isnan, + "isinf": np.isinf, "signbit": np.signbit, "square": np.square, "logical_not": np.logical_not, + "invert": np.invert, +} + + +@pytest.mark.parametrize("shape, view, dtype, mode", + cases(SHAPES, VIEWS, DTYPES, MODES)) +def test_unary(shape, view, dtype, mode): + rng = np.random.default_rng(seed((shape, view, dtype))) + base = values(rng, SHAPES[shape], dtype) + a = make(base, view, mode) + for name, f in UNARY.items(): + check(f, (a,), (to_numpy2x(a, base, view, mode),), values_=not deviates(), label=name) + + +# --- element-wise binary with broadcasting ------------------------------- + +PAIRS = [ + ((), ()), ((), (7,)), ((7,), ()), ((0,), ()), ((0,), (1,)), ((1,), (0,)), ((2, 0), (0,)), + ((2, 0), (2, 1)), ((3, 4), (4,)), ((3, 1), (1, 4)), ((1, 5, 1), (3, 1, 4)), + ((2, 3, 4), (3, 1)), ((2, 1, 2, 1, 2, 1, 2, 3), (2, 1, 3)), ((60, 70), (70,)), + ((60, 1), (1, 70)), ((3000,), (3000,)), ((3, 4), (3,)), ((2, 0), (3,)), + ((0, 3, 1), (2,)), +] +BINARY = { + "add": np.add, "subtract": np.subtract, "multiply": np.multiply, "divide": np.divide, + "floor_divide": np.floor_divide, "remainder": np.remainder, "power": np.power, + "maximum": np.maximum, "fmin": np.fmin, "less": np.less, "equal": np.equal, + "logical_and": np.logical_and, "arctan2": np.arctan2, "bitwise_and": np.bitwise_and, +} +DTYPE_PAIRS = [("f8", "f8"), ("i4", "f8"), ("i8", "i8"), ("i4", "i8"), ("b1", "i4"), + ("f4", "f4"), ("f4", "c8"), ("c16", "f8"), ("b1", "b1"), ("i8", "f4")] + + +@pytest.mark.parametrize("pair, dtypes, views, modes", + cases(PAIRS, DTYPE_PAIRS, [("C", "C"), ("F", "rev"), ("T", "step")], + [("ready", "ready"), ("pending", "ready"), + ("pending", "pending")]), + ids=lambda x: str(x).replace(" ", "")) +def test_binary(pair, dtypes, views, modes): + rng = np.random.default_rng(seed((pair, dtypes, views))) + bases = [values(rng, s, d) for s, d in zip(pair, dtypes)] + if views[1] in ("step", "rev"): # views that keep the shape: only on 1-d+ data + bases[1] = values(rng, pair[1][:-1] + (2 * pair[1][-1],), dtypes[1]) \ + if views[1] == "step" and pair[1] else bases[1] + nps = [make(b, v, m) for b, v, m in zip(bases, views, modes)] + if views[0] == "T": + bases[0] = values(rng, pair[0][::-1], dtypes[0]) + nps[0] = make(bases[0], "T", modes[0]) + vps = [to_numpy2x(a, b, v, m) for a, b, v, m in zip(nps, bases, views, modes)] + for name, f in BINARY.items(): + check(f, tuple(nps), tuple(vps), values_=not deviates(), label=name) + # Python scalars (NEP 50: weak), also special values + for s in (2, -1, 2.5, float("nan"), float("inf"), -0.0, 1j, True): + for f in (np.add, np.multiply, np.maximum, np.less): + check(lambda x: f(x, s), (nps[0],), (vps[0],), values_=not deviates(), + label=f"{f.__name__}(x, {s!r})") + check(lambda x: f(s, x), (nps[0],), (vps[0],), values_=not deviates(), + label=f"{f.__name__}({s!r}, x)") + + +# --- reductions: axes, keepdims ("scalar dimension removal") --------------- + +REDUCTIONS = { + "sum": np.sum, "prod": np.prod, "min": np.min, "max": np.max, "any": np.any, + "all": np.all, "mean": np.mean, "std": np.std, "argmax": np.argmax, +} + + +@pytest.mark.parametrize("shape, view, dtype, mode, keepdims", + cases(SHAPES, ["C", "F", "T"], ["b1", "i4", "i8", "f4", "f8", "c16"], + MODES, [False, True])) +def test_reductions(shape, view, dtype, mode, keepdims): + rng = np.random.default_rng(seed((shape, view, dtype))) + base = values(rng, SHAPES[shape], dtype) + a = make(base, view, mode) + axes = [None] + list(range(-a.ndim, a.ndim))[:3] + ([(0, a.ndim - 1)] if a.ndim > 1 else []) + for (name, f), axis in itertools.product(REDUCTIONS.items(), axes): + if name == "argmax" and isinstance(axis, tuple): + continue + kw = {"axis": axis, "keepdims": keepdims} + check(lambda x: f(x, **kw), (a,), (to_numpy2x(a, base, view, mode),), + values_=not deviates(), label=f"{name}({kw})") + + +# --- casts: overflow, rounding, NaN and infinities -------------------------- + +CASTS = [("f8", "i4"), ("f8", "i8"), ("f4", "i4"), ("f8", "b1"), ("i8", "i4"), ("i4", "i8"), + ("i8", "f4"), ("i8", "f8"), ("c16", "f8"), ("f8", "f4"), ("b1", "f8"), ("f8", "c8")] + + +@pytest.mark.parametrize("shape, view, cast, mode", + cases(SHAPES, ["C", "F", "rev"], CASTS, MODES)) +def test_casts(shape, view, cast, mode): + rng = np.random.default_rng(seed((shape, view, cast))) + base = values(rng, SHAPES[shape], cast[0]) + if np.dtype(cast[0]).kind == "f": # large values: overflow on the way to integers + base = base * np.asarray(1e9, base.dtype) + a = make(base, view, mode) + v = to_numpy2x(a, base, view, mode) + for order in ("K", "C", "F"): + check(lambda x: x.astype(cast[1], order=order), (a,), (v,), label=f"astype {order}") + check(lambda x: np.round(x, 1), (a,), (v,), values_=not deviates()) + check(lambda x: np.round(x, -2), (a,), (v,), values_=not deviates()) + if np.dtype(cast[0]).kind != "c": + check(lambda x: np.clip(x, -3, 3), (a,), (v,)) + check(lambda x: np.floor(x).astype(cast[1]), (a,), (v,)) + check(lambda x: np.add(x, x, dtype=cast[1], casting="unsafe"), (a,), (v,), + values_=not deviates()) + + +# --- indexing: integers remove dimensions, slices, index arrays, masks ------ + +def keys(shape): + nd = len(shape) + out = [(), (Ellipsis,), (None,)] + if nd: + n0 = shape[0] + out += [(slice(None),), (slice(1, None),), (slice(None, None, -2),), (Ellipsis, None)] + if n0: + out += [(0,), (-1,), (n0,), (-n0 - 1,), (np.array([0, -1, 0]),), + (np.array([n0]),), (np.array([], np.intp),), (n0 - 1, Ellipsis)] + if nd > 1 and shape[1]: + out += [(slice(None), 0), (slice(None), np.array([[0], [shape[1] - 1]])), + (np.array([0]), slice(None), ) if n0 else (slice(None),)] + return out + + +@pytest.mark.parametrize("shape, view, dtype, mode", + cases(SHAPES, ["C", "F", "T"], ["b1", "i8", "f8", "c16"], MODES)) +def test_indexing(shape, view, dtype, mode): + rng = np.random.default_rng(seed((shape, view, dtype))) + base = values(rng, SHAPES[shape], dtype) + a = make(base, view, mode) + v = to_numpy2x(a, base, view, mode) + for key in keys(a.shape): + check(lambda x: x[key], (a,), (v,), label=f"x[{key!r}]") + if dtype in ("f8", "c16"): # masks from NaN and the infinities + check(lambda x: x[np.isnan(x)], (a,), (v,)) + check(lambda x: x[~np.isfinite(x)], (a,), (v,)) + check(lambda x: np.where(np.isnan(x), 0, x), (a,), (v,)) diff --git a/src/capi/capi.cpp b/src/capi/capi.cpp index 5eff75a..c7db360 100644 --- a/src/capi/capi.cpp +++ b/src/capi/capi.cpp @@ -1569,21 +1569,15 @@ int nx_reduce_out(nx_runtime* rt, PyObject* ufunc, nx_array* in, const unsigned } const std::size_t nd = l.shape.size(); SmallVector flags(nd, 0); // no std::vector: data() - Shape out_shape; std::uint64_t axis_bits = 0; for (std::size_t i = 0; i < nd; ++i) { flags[i] = axes[i] != 0; - if (!flags[i]) { - out_shape.push_back(l.shape[i]); - } else { + if (flags[i]) { axis_bits |= std::uint64_t{1} << (i % 64); - if (keepdims) { - out_shape.push_back(1); - } } } - Layout out_layout = Layout::contiguous(l.dtype, std::move(out_shape)); const std::span reduced(reinterpret_cast(flags.data()), nd); + Layout out_layout = reduction_result(l, reduced, keepdims != 0, l.dtype); if (dtype == nullptr) { dtype = Py_None; } diff --git a/src/kernels/numpy_reduce.cpp b/src/kernels/numpy_reduce.cpp index c06134d..32dfb65 100644 --- a/src/kernels/numpy_reduce.cpp +++ b/src/kernels/numpy_reduce.cpp @@ -33,17 +33,14 @@ bool ReducePlan::supported() const { ReducePlan plan_reduction(const Layout& in, std::span reduced, const Layout& out) { const std::size_t nd = in.shape.size(); - const auto isz = static_cast(itemsize(out.dtype)); - // Result stride of each kept input axis: the result is C-contiguous over - // the kept axes (extent-1 dimensions for reduced axes, if kept, add - // nothing). + // Result stride of each kept input axis, from `out` (with or without + // the reduced axes kept). std::vector out_stride(nd, 0); - std::int64_t step = isz; - for (std::size_t i = nd; i-- > 0;) { + const bool kept_all = out.shape.size() == nd; + for (std::size_t i = 0, j = 0; i < nd; ++i) { if (!reduced[i]) { - out_stride[i] = step; - step *= in.shape[i]; + out_stride[i] = out.strides[kept_all ? i : j++]; } } diff --git a/src/runtime/layout.cpp b/src/runtime/layout.cpp index fb3a4d2..3948371 100644 --- a/src/runtime/layout.cpp +++ b/src/runtime/layout.cpp @@ -17,6 +17,9 @@ Layout Layout::contiguous(DType dtype, Shape shape) { l.strides[i] = stride; stride *= shape[i]; } + if (stride == 0) { // no elements: NumPy (>= 2.5) gives every axis stride 0 + std::fill(l.strides.begin(), l.strides.end(), 0); + } l.shape = std::move(shape); return l; } @@ -90,10 +93,29 @@ Layout Layout::keep_order(DType dtype, Shape shape, std::span inpu l.strides[ax] = step; step *= shape[ax]; } + if (step == 0) { // no elements: all strides 0, as Layout::contiguous + std::fill(l.strides.begin(), l.strides.end(), 0); + } l.shape = std::move(shape); return l; } +bool Layout::empty() const { return std::find(shape.begin(), shape.end(), 0) != shape.end(); } + +bool Layout::zero_strides_if_empty() { + if (!empty()) { + return false; + } + std::fill(strides.begin(), strides.end(), 0); + return true; +} + +Layout Layout::fresh() const { + Layout l = dense() ? *this : contiguous(dtype, shape); + l.zero_strides_if_empty(); + return l; +} + bool Layout::dense() const { if (offset != 0) { return false; @@ -122,10 +144,15 @@ bool Layout::dense() const { } bool Layout::c_contiguous() const { + // _UpdateContiguousFlags: an array without elements is C- and + // F-contiguous whatever its strides. std::int64_t stride = static_cast(itemsize(dtype)); for (std::size_t i = shape.size(); i-- > 0;) { + if (shape[i] == 0) { + return true; + } if (shape[i] != 1 && strides[i] != stride) { - return false; + return empty(); // the axes left may still hold an extent 0 } stride *= shape[i]; } @@ -139,7 +166,7 @@ bool Layout::f_contiguous() const { return true; } if (shape[i] != 1 && strides[i] != stride) { - return false; + return empty(); } stride *= shape[i]; } @@ -245,6 +272,66 @@ bool broadcast_shapes(const Shape& a, const Shape& b, Shape& out) { return true; } +Layout reduction_result(const Layout& in, std::span reduced, bool keepdims, + DType dtype) { + const std::size_t nd = in.shape.size(); + // Strides of the result with the reduced axes kept (extent 1). + Strides st(nd, 0); + std::int64_t step = static_cast(itemsize(dtype)); + if (in.c_contiguous()) { // C order (the common case, without sorting) + for (std::size_t i = nd; i-- > 0;) { + st[i] = step; + step *= reduced[i] ? 1 : in.shape[i]; + } + } else { + // Layout::iteration_order() for one operand, without heap memory: + // insertion sort from C order, innermost first; an extent-1 axis + // abstains. + SmallVector perm(nd); + for (std::size_t i = 0; i < nd; ++i) { + perm[i] = nd - 1 - i; + } + auto key = [&](std::size_t ax) { + return in.shape[ax] == 1 ? 0 : std::llabs(in.strides[ax]); + }; + for (std::size_t i0 = 1; i0 < nd; ++i0) { + const std::size_t j0 = perm[i0]; + const std::int64_t s0 = key(j0); + std::size_t ipos = i0; + for (std::size_t i1 = i0; i1-- > 0;) { + const std::int64_t s1 = key(perm[i1]); + if (s0 == 0 || s1 == 0) { + continue; // ambiguous + } + if (s1 <= s0) { + break; + } + ipos = i1; + } + for (std::size_t k = i0; k > ipos; --k) { + perm[k] = perm[k - 1]; + } + perm[ipos] = j0; + } + for (std::size_t ax : perm) { + st[ax] = step; + step *= reduced[ax] ? 1 : in.shape[ax]; + } + } + Layout out; + out.dtype = dtype; + for (std::size_t i = 0; i < nd; ++i) { + if (!reduced[i]) { + out.shape.push_back(in.shape[i]); + out.strides.push_back(step == 0 ? 0 : st[i]); // no elements: all 0 + } else if (keepdims) { + out.shape.push_back(1); + out.strides.push_back(step == 0 ? 0 : st[i]); + } + } + return out; +} + Layout broadcast_to(const Layout& in, const Shape& target) { assert(target.size() >= in.shape.size()); Layout l; diff --git a/src/runtime/layout.hpp b/src/runtime/layout.hpp index f73f6c1..b3ed923 100644 --- a/src/runtime/layout.hpp +++ b/src/runtime/layout.hpp @@ -56,8 +56,16 @@ struct Layout { std::span operands); std::int64_t size() const; + // True if an extent is 0 (no elements). + bool empty() const; + // The layout of new memory for this one: itself if dense, else C order; + // without elements every stride 0 (NumPy >= 2.5 allocates so). + Layout fresh() const; + // Every stride 0 if an extent is 0 (as fresh(), for layouts computed + // otherwise); true if it was. + bool zero_strides_if_empty(); // True if the elements lie C-contiguously (dimensions of extent 1 may - // have any stride). + // have any stride; without elements always, as NumPy's flags). bool c_contiguous() const; // True if the elements lie F-contiguously (as c_contiguous()). bool f_contiguous() const; @@ -80,6 +88,14 @@ bool broadcast_shapes(const Shape& a, const Shape& b, Shape& out); // Broadcast dimensions get stride 0. Layout broadcast_to(const Layout& in, const Shape& target); +// The result NumPy allocates for a reduction of `in` over the `reduced` +// axes: its iterator orders the axes by the input's strides +// (Layout::iteration_order) and lays the result out in that order, every +// axis of extent 1 (also a kept reduced one) taking the stride of its +// position; without keepdims the reduced axes are dropped. +Layout reduction_result(const Layout& in, std::span reduced, bool keepdims, + DType dtype); + // Calls `row(ptrs, count, strides)` once per innermost row of `shape` for a // set of operands, all already broadcast to `shape` (`layouts[k]`, first // element at `base[k]`). A 0-d shape is one row of one element. Returns diff --git a/src/runtime/runtime.cpp b/src/runtime/runtime.cpp index 44c97b5..333aaee 100644 --- a/src/runtime/runtime.cpp +++ b/src/runtime/runtime.cpp @@ -500,7 +500,7 @@ void Runtime::issue(const std::shared_ptr& kernel, std::span(contiguous.nbytes(), CountableArray::deferred); MetadataPtr st = make_metadata(std::move(contiguous)); @@ -568,7 +568,7 @@ bool Runtime::issue_direct(const std::shared_ptr& kernel, std::span(contiguous.nbytes()); MetadataPtr st = make_metadata(std::move(contiguous)); ops.push_back({buffer, st, false});