Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 91 additions & 14 deletions python/numpy2x/_numpy2x.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ static long long g_small = 2048; /* the runtime's sync_below: ops this small run
/* Registered by _ndarray.py via _numpy2x.setup(). */
static PyTypeObject* g_array_cls = NULL; /* numpy2x.ndarray */
static PyObject* g_dtypes[7]; /* np.dtype per nx_dtype code */
static PyObject* g_scalar_types[7]; /* its scalar type (np.float64, ...) */
static int g_alignment[7]; /* np.dtype(...).alignment per nx_dtype code */
static PyObject* g_matmul = NULL; /* np.matmul */
static PyObject* g_fallback = NULL; /* fallback(ufunc, method, inputs, kwargs) */
Expand Down Expand Up @@ -1590,6 +1591,14 @@ static int input_kind(PyObject* x) {
if (PyComplex_CheckExact(x)) {
return K_PYCOMPLEX;
}
for (int c = 0; c < 7; ++c) {
/* NumPy's scalar of a numpy2x dtype (a[i] of a ready array, a full
* reduction): its type says the dtype, no attribute lookup and
* comparison of dtype objects (~0.4 us per op) */
if ((PyObject*)Py_TYPE(x) == g_scalar_types[c]) {
return c;
}
}
if (PyObject_TypeCheck(x, (PyTypeObject*)g_np_generic) ||
PyObject_TypeCheck(x, (PyTypeObject*)g_np_ndarray)) {
PyObject* dt = PyObject_GetAttrString(x, "dtype");
Expand Down Expand Up @@ -1949,6 +1958,11 @@ static int defers(PyObject* other) {
PyComplex_CheckExact(other) || PyBool_Check(other)) {
return 0;
}
for (int c = 0; c < 7; ++c) {
if ((PyObject*)Py_TYPE(other) == g_scalar_types[c]) {
return 0; /* NumPy's own scalar types never opt out: no attribute lookup */
}
}
PyObject* au = PyObject_GetAttrString((PyObject*)Py_TYPE(other), "__array_ufunc__");
if (au == NULL) {
PyErr_Clear();
Expand Down Expand Up @@ -2204,6 +2218,7 @@ static PyMethodDef Array_methods[] = {
};

static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs);
static PyObject* pending_item(PyObject* a, PyObject* key);
static PyObject* m_basic_view(PyObject* self, PyObject* const* args, Py_ssize_t nargs);
static int basic_layout(nx_array* x, PyObject* key, int* out_nd, int64_t* nshape, int64_t* nstrides,
int64_t* out_offset);
Expand All @@ -2219,6 +2234,13 @@ static PyObject* Array_subscript(PyObject* self, PyObject* key) {
return r; /* NumPy's scalar, or NULL */
}
Py_DECREF(r);
if (!nx_array_ready(((ArrayObject*)self)->a)) {
r = pending_item(self, key);
if (r != Py_None) {
return r; /* a pending 0-d copy, or NULL */
}
Py_DECREF(r);
}
r = m_basic_view(NULL, args, 2);
if (r == NULL) {
return NULL;
Expand Down Expand Up @@ -2464,6 +2486,10 @@ static PyObject* m_setup(PyObject* self, PyObject* args) {
Py_XSETREF(g_array_cls, (PyTypeObject*)Py_NewRef(cls));
for (int c = 0; c < 7; ++c) {
Py_XSETREF(g_dtypes[c], Py_NewRef(PyTuple_GET_ITEM(dtypes, c)));
Py_XSETREF(g_scalar_types[c], PyObject_GetAttrString(g_dtypes[c], "type"));
if (g_scalar_types[c] == NULL) {
return NULL;
}
PyObject* al = PyObject_GetAttrString(g_dtypes[c], "alignment");
g_alignment[c] = al != NULL ? (int)PyLong_AsLong(al) : -1;
Py_XDECREF(al);
Expand Down Expand Up @@ -3231,38 +3257,89 @@ static PyObject* m_scalar(PyObject* self, PyObject* a) {
/* item(a, key): for an integer key (an int, or a tuple of one int per
* dimension) the element as NumPy's scalar if `a` is ready; None if `a` is
* pending or the key is something else. IndexError as NumPy's. */
static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs) {
(void)self;
if (array_and_arg(args, nargs) < 0) {
return NULL;
}
PyObject *a = args[0], *key = args[1];
nx_array* x = array_of(a);
/* The byte offset of the element an integer key (an int, or a tuple of one
* int per dimension) selects in x: 1, and *offset; 0 for any other key
* (arrays, slices, masks, np.bool_: the general path); -1 with NumPy's
* IndexError. */
static int int_key_offset(nx_array* x, PyObject* key, int64_t* offset) {
const int nd = nx_array_ndim(x);
const Py_ssize_t nk = PyTuple_Check(key) ? PyTuple_GET_SIZE(key) : 1;
if (nk != nd || nd == 0 || !nx_array_ready(x)) {
Py_RETURN_NONE;
if (nk != nd || nd == 0) {
return 0;
}
const int64_t* shape = nx_array_shape(x);
const int64_t* strides = nx_array_strides(x);
int64_t offset = 0;
*offset = 0;
for (int i = 0; i < nd; ++i) {
PyObject* k = PyTuple_Check(key) ? PyTuple_GET_ITEM(key, i) : key;
if (PyBool_Check(k) ||
!(PyLong_Check(k) ||
(g_np_integer != NULL && PyObject_TypeCheck(k, (PyTypeObject*)g_np_integer)))) {
Py_RETURN_NONE; /* arrays, slices, masks, np.bool_: the general path */
return 0;
}
const Py_ssize_t v = PyNumber_AsSsize_t(k, PyExc_IndexError);
if (v == -1 && PyErr_Occurred()) {
return NULL;
return -1;
}
if (v < -shape[i] || v >= shape[i]) {
PyErr_Format(PyExc_IndexError, "index %zd is out of bounds for axis %d with size %lld",
v, i, (long long)shape[i]);
return NULL;
return -1;
}
offset += (v < 0 ? v + shape[i] : v) * strides[i];
*offset += (v < 0 ? v + shape[i] : v) * strides[i];
}
return 1;
}

/* a[i, ...] of a pending array: NumPy's scalar, a snapshot, so a 0-d copy
* of the element (one placement task, no view object), marked `_scalar`.
* None if the key is no integer key or the copy is unsupported. */
static PyObject* pending_item(PyObject* a, PyObject* key) {
nx_array* x = array_of(a);
int64_t offset = 0;
const int k = int_key_offset(x, key, &offset);
if (k <= 0) {
return k < 0 ? NULL : Py_NewRef(Py_None);
}
const nx_dtype dt = nx_array_dtype(x);
int64_t zero = 0;
nx_array* e = nx_array_view(g_rt, x, dt, 0, &zero, &zero, offset);
if (e == NULL) {
Py_RETURN_NONE;
}
const int part_ndim = 0;
nx_array* r = NULL;
char err[ERRLEN] = "";
const int st = nx_place(g_rt, dt, 0, &zero, NULL, 1, &e, &part_ndim, &zero, &zero, &zero, &r,
issue_site(), err, sizeof err);
nx_array_release(e);
if (st == NX_UNSUPPORTED) {
Py_RETURN_NONE;
}
if (raise_status(st, err) < 0) {
return NULL;
}
PyObject* res = wrap_array(r);
if (res != NULL) {
((ArrayObject*)res)->scalar = 1;
}
return res;
}

static PyObject* m_item(PyObject* self, PyObject* const* args, Py_ssize_t nargs) {
(void)self;
if (array_and_arg(args, nargs) < 0) {
return NULL;
}
PyObject *a = args[0], *key = args[1];
nx_array* x = array_of(a);
if (!nx_array_ready(x)) {
Py_RETURN_NONE;
}
int64_t offset = 0;
const int k = int_key_offset(x, key, &offset);
if (k <= 0) {
return k < 0 ? NULL : Py_NewRef(Py_None);
}
int64_t zero = 0;
nx_array* e = nx_array_view(g_rt, x, nx_array_dtype(x), 0, &zero, &zero, offset);
Expand Down
97 changes: 97 additions & 0 deletions python/tests/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,103 @@ def test_scalar_indexing_of_pending_arrays_does_not_sync():
assert isinstance(ready[0, 1], np.float64) # ready data: NumPy's scalar


def test_scalar_indexing_of_pending_arrays_all_dtypes_and_keys():
rng = np.random.default_rng(11)
for dt in ("?", "i4", "i8", "f4", "f8", "c8", "c16"):
x = (rng.standard_normal((5, 7)) * 9).astype(dt)
e = np.asarray(np.positive(x) if dt != "?" else np.logical_not(np.logical_not(x)))
v = numpy2x.asarray(x)
y = np.positive(v) if dt != "?" else np.logical_not(np.logical_not(v))
for key in [(1, 2), (-1, -7), (4, 0), (np.int64(2), np.int32(-3)), (0, 6)]:
s = y[key]
if not numpy2x._numpy2x.ready(y):
assert type(s) is numpy2x.ndarray and s.ndim == 0 and s._scalar
got = np.asarray(s)[()] if isinstance(s, numpy2x.ndarray) else s
assert np.asarray(got).dtype == e.dtype
assert np.asarray(got).tobytes() == np.asarray(e[key]).tobytes()
for key in [(5, 0), (0, -8)]:
with pytest.raises(IndexError) as got:
(np.positive(v) if dt != "?" else np.logical_not(v))[key]
with pytest.raises(IndexError) as expected:
x[key]
assert str(got.value) == str(expected.value)
w = numpy2x.asarray(np.arange(6.0)) * 2.0 # 1-d, an int key
assert float(w[3]) == 6.0 and float(w[-1]) == 10.0


def test_scalar_indexing_of_a_pending_array_is_one_task_and_no_sync():
from async_path import SIZE, assert_async, deterministic, pending

# 1e6 elements behind 20 copies: tens of milliseconds of work, so y is
# still pending when y[17] is issued (at SIZE the chain could finish
# first on a fast runner, and y[17] then took the ready path: no task)
x = np.random.default_rng(12).standard_normal(200 * SIZE)
with deterministic():
y = pending(numpy2x.asarray(x))
s = assert_async(lambda: y[17], ops=1)
assert s._scalar
y[17] = 0.0 # s is a snapshot
assert float(s) == x[17]


def test_cumulative_loop_over_columns_is_bit_identical():
# ES[i] = ES[i - 1] + reduction: pending elements read, added and stored
rng = np.random.default_rng(13)
for n, dt in [(200, "f8"), (150, "f4")]:
a = rng.standard_normal((n, n)).astype(dt)

def run(xp, A):
ES = xp.zeros(A.shape[1], dtype=dt)
for i in range(A.shape[1]):
ES[i] = (ES[i - 1] if i > 0 else 0) + xp.abs(xp.sin(A[:, i])).sum()
return np.asarray(ES)

got = run(np, numpy2x.asarray(a))
expected = run(np, a)
assert got.dtype == expected.dtype
assert got.tobytes() == expected.tobytes()


def test_numpy_scalars_of_every_dtype_as_operands():
rng = np.random.default_rng(14)
for dt in ("i4", "i8", "f4", "f8", "c8", "c16", "?"):
x = (rng.standard_normal(3000) * 5).astype(dt)
s = x[7] # NumPy's scalar of that dtype (strong, NEP 50)
v = numpy2x.asarray(x)
for op in (np.add, np.multiply, np.not_equal):
if dt == "?" and op is not np.not_equal:
continue
for got, expected in [(op(v, s), op(x, s)), (op(s, v), op(s, x))]:
g = np.asarray(got)
assert g.dtype == expected.dtype
assert g.tobytes() == expected.tobytes()
if dt in ("f4", "f8"):
g = np.asarray(v + np.float64(0.1)) # another dtype: promoted as NumPy
e = x + np.float64(0.1)
assert g.dtype == e.dtype and g.tobytes() == e.tobytes()


def test_zero_d_ops_on_pending_operands():
# the 0-d fast path of vp_ufunc_issue: pending 0-d inputs of the loop's
# dtypes, a 0-d result (values, dtypes, FP errors as NumPy)
rng = np.random.default_rng(15)
for dt in ("i4", "i8", "f4", "f8", "c8", "c16"):
x = (rng.standard_normal(4000) * 3).astype(dt)
v = numpy2x.asarray(x)
a, b = np.positive(v)[5], np.negative(v)[9] # pending 0-d elements
ea, eb = x[5], -x[9]
for f in (np.add, np.subtract, np.multiply, np.maximum, np.equal):
if dt[0] == "c" and f is np.maximum:
continue
got, expected = f(a, b), f(ea, eb)
g = np.asarray(got)
assert g.dtype == np.asarray(expected).dtype
assert g.tobytes() == np.asarray(expected).tobytes()
z = np.positive(numpy2x.asarray(np.zeros(3000)))[0] # pending 0.0
with np.errstate(divide="raise"), pytest.raises(FloatingPointError):
float(1.0 / z) # raised at the next check or at the sync point


def test_where_runs_as_a_select_task():
rng = np.random.default_rng(8)
f0 = fallbacks()
Expand Down
25 changes: 25 additions & 0 deletions src/capi/capi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,21 @@ struct Graveyard {
std::mutex mutex;
std::vector<PyObject*> objects;
std::vector<std::function<void()>> releases; // kernels' Python state
// Set under the mutex when something is parked: drain() runs at every C
// ABI call and took the mutex (a cache line the workers write) each time.
std::atomic<bool> nonempty{false};

void drain() {
if (!nonempty.load(std::memory_order_acquire)) {
return;
}
std::vector<PyObject*> dead;
std::vector<std::function<void()>> todo;
{
std::lock_guard lock(mutex);
dead.swap(objects);
todo.swap(releases);
nonempty.store(false, std::memory_order_relaxed);
}
for (PyObject* o : dead) {
Py_DECREF(o);
Expand All @@ -66,6 +73,7 @@ numpy::DeferRelease defer_release(const std::shared_ptr<Graveyard>& graveyard) {
} else if (auto g = weak.lock()) {
std::lock_guard lock(g->mutex);
g->releases.push_back(std::move(f));
g->nonempty.store(true, std::memory_order_release);
}
};
}
Expand All @@ -80,6 +88,7 @@ std::shared_ptr<void> python_owner(const std::shared_ptr<Graveyard>& graveyard,
} else if (auto g = weak.lock()) {
std::lock_guard lock(g->mutex);
g->objects.push_back(o);
g->nonempty.store(true, std::memory_order_release);
} // else: runtime gone, interpreter shutting down; leak
});
}
Expand Down Expand Up @@ -614,8 +623,24 @@ int nx_ufunc_issue(nx_runtime* rt, const nx_loop* loop, nx_array* const* inputs,
set_error(err, errlen, "too many inputs");
return NX_UNSUPPORTED;
}
bool scalar = out_ndim == 0 && out == nullptr;
for (int i = 0; i < nin; ++i) {
ins[i] = ref(inputs[i]);
scalar = scalar && ins[i]->layout().shape.empty() &&
ins[i]->layout().dtype == loop->dtypes[static_cast<std::size_t>(i)];
}
if (scalar) {
// 0-d inputs of the loop's dtypes, a 0-d result (`s + t`, `x[i - 1] + s`
// in loops): one loop call over one element, as NumPy's trivial loop
// makes it. The layout is a constant, the kernel the loop's shared one:
// no order-K layout, no single-call check, no cast check.
Layout l = Layout::contiguous(loop->dtypes[static_cast<std::size_t>(nin)], {});
ArrayPtr res;
rt->rt->issue(loop->loop->elementwise_kernel(),
std::span<const ArrayPtr>(ins, static_cast<std::size_t>(nin)), {},
std::span(&l, 1), site, std::span(&res, 1));
*result = hand_out(std::move(res));
return NX_OK;
}
if (!cast_small_inputs(rt, *loop, std::span(ins, static_cast<std::size_t>(nin)), site, err,
errlen)) {
Expand Down
22 changes: 22 additions & 0 deletions src/kernels/numpy_ufunc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1253,6 +1253,28 @@ ArrayPtr wrap_ndarray(Runtime& rt, PyObject* obj, std::string& error, std::share
}

ArrayPtr scalar_array(Runtime& rt, PyObject* value, DType dtype) {
// A NumPy scalar of exactly this dtype (a[i] of a ready array, a full
// reduction) or a Python float as float64: its bytes as they are. A new
// object per op misses the bridge's scalar cache, and NumPy's conversion
// builds and frees an array (~1 µs per op in a loop adding a[i - 1]).
const Layout l0 = Layout::contiguous(dtype, {});
if (PyFloat_CheckExact(value) && dtype == DType::Float64) {
auto buffer = make_pooled<CountableArray>(l0.nbytes());
const double v = PyFloat_AS_DOUBLE(value);
std::memcpy(buffer->data(), &v, sizeof v);
return rt.wrap(Layout(l0), std::move(buffer));
}
if (PyArray_IsScalar(value, Generic)) {
PyArray_Descr* d = PyArray_DescrFromScalar(value);
const bool same = d != nullptr && d->type_num == type_num(dtype) &&
PyDataType_ISNOTSWAPPED(d);
Py_XDECREF(d);
if (same) {
auto buffer = make_pooled<CountableArray>(l0.nbytes());
PyArray_ScalarAsCtype(value, buffer->data());
return rt.wrap(Layout(l0), std::move(buffer));
}
}
PyArray_Descr* descr = PyArray_DescrFromType(type_num(dtype));
PyObject* arr = PyArray_FromAny(value, descr, 0, 0, 0, nullptr); // steals descr
if (arr == nullptr) {
Expand Down
4 changes: 3 additions & 1 deletion src/runtime/task.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,9 @@ class Task {
struct Copy {
std::shared_ptr<CountableArray> from, to;
};
std::vector<Copy> copies_; // copy-on-write, empty for most tasks
// Copy-on-write, empty for most tasks; inline: a write into an array a
// pending task reads (`x[i] = x[i - 1] + s`) did a heap allocation per op.
InlineVector<Copy, 1> copies_;

// The consumers registered for this task's completion: a lock-free stack
// of ConsumerNodes (the main thread pushes with a CAS), closed for good
Expand Down
Loading