From bdbb824faa28e2a932c3058c69625b26f8a95343 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:56:40 +0000 Subject: [PATCH 01/11] simd: odd_lut_u8, a 256-entry odd-function table over a sign-magnitude lane Looks up each u8 magnitude in a 256-entry table as four 64-entry U8x64::permute_bytes lookups chosen by the magnitude's top two bits. The sign plane passes through, cleared where the result is 0 so a zero result is stored as +0. table[0] must be 0, so a missing row (+0) stays +0 and the presence plane is not touched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- src/simd.rs | 1 + src/simd_masking_ops.rs | 201 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/src/simd.rs b/src/simd.rs index e028b232..a5885d68 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -968,6 +968,7 @@ pub use crate::simd_masking_ops::{ ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, + odd_lut_u8, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index 66f191f9..347f7d71 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3787,6 +3787,117 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { clear_mask_tail(out_words, values.len()); } +// ──────────────────────────────────────────────────────────────────────── +// Odd-function table over a sign-magnitude lane. +// +// A sign-magnitude 9-bit lane stores each value as a `u8` magnitude plus one +// bit in a `u64` sign plane per 64 rows; whether the row holds a value at all +// is a separate presence plane, and the writer stores `+0` (magnitude 0, sign +// clear) for zero and for missing rows. For an odd function +// (`f(-x) = -f(x)`, e.g. tanh, sin, x³, sign-preserving clamps) one table +// over the 256 magnitudes covers both signs: the magnitude is looked up and +// the sign plane passes through. The presence plane is not an input — a +// missing row is magnitude 0, `f(0) = 0`, so it stays `+0`. +// ──────────────────────────────────────────────────────────────────────── + +/// Applies an odd function, given as a 256-entry magnitude table, to a +/// sign-magnitude lane. +/// +/// For every row `i < magnitudes.len()`: +/// +/// * `out_magnitudes[i] = table[magnitudes[i]]`; +/// * the output sign bit is the input sign bit, cleared where the output +/// magnitude is `0`. A result that rounds to zero is therefore stored as +/// the canonical `+0`, never `-0`, and an input `-0` comes out as `+0`. +/// +/// Rows past `magnitudes.len()` are not written in `out_magnitudes`. In +/// `out_signs` the bits past `magnitudes.len()` and any surplus words are +/// written as `0`. The presence plane is not read or written: because +/// `table[0]` must be `0`, a missing row (stored as `+0`) stays `+0`. +/// +/// Each 64-row chunk is looked up as four 64-entry +/// [`crate::simd::U8x64::permute_bytes`] lookups, one per quarter of the +/// table, selected by the magnitude's top two bits with +/// [`crate::simd::U8x64::mask_blend`]. The final partial chunk is padded with +/// magnitude 0 and goes through the same path, so the tail cannot disagree +/// with the body. +/// +/// # Panics +/// +/// * if `table[0] != 0`, since an odd function maps 0 to 0; +/// * if `out_magnitudes.len() < magnitudes.len()`; +/// * if `signs.len()` or `out_signs.len()` is less than +/// `magnitudes.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::odd_lut_u8; +/// +/// // f(m) = m / 2, an odd function on the signed value. +/// let mut table = [0u8; 256]; +/// for (m, t) in table.iter_mut().enumerate() { +/// *t = (m / 2) as u8; +/// } +/// // Values: +10, -10, -1, +0 +/// let magnitudes = [10u8, 10, 1, 0]; +/// let signs = [0b0110u64]; +/// let mut out = [0u8; 4]; +/// let mut out_signs = [0u64; 1]; +/// odd_lut_u8(&magnitudes, &signs, &table, &mut out, &mut out_signs); +/// assert_eq!(out, [5, 5, 0, 0]); +/// // -1 / 2 rounds to zero and is stored as +0: only row 1 keeps its sign. +/// assert_eq!(out_signs[0], 0b0010); +/// ``` +pub fn odd_lut_u8( + magnitudes: &[u8], signs: &[u64], table: &[u8; 256], out_magnitudes: &mut [u8], out_signs: &mut [u64], +) { + use crate::simd::U8x64; + let n = magnitudes.len(); + let words = n.div_ceil(64); + assert_eq!(table[0], 0, "odd_lut_u8: an odd function maps 0 to 0, so table[0] must be 0"); + assert!( + out_magnitudes.len() >= n, + "odd_lut_u8: out_magnitudes holds {} rows, need {n}", + out_magnitudes.len() + ); + assert!(signs.len() >= words, "odd_lut_u8: signs holds {} words, need {words}", signs.len()); + assert!(out_signs.len() >= words, "odd_lut_u8: out_signs holds {} words, need {words}", out_signs.len()); + + let quarter = |k: usize| U8x64::from_slice(&table[k * 64..(k + 1) * 64]); + let (t0, t1, t2, t3) = (quarter(0), quarter(1), quarter(2), quarter(3)); + let zero = U8x64::splat(0); + let lookup = |idx: U8x64| -> U8x64 { + // `permute_bytes` reads the low 6 bits of each index. + let bit6 = idx.shl_epi16(1).movemask(); + let bit7 = idx.movemask(); + let low = U8x64::mask_blend(bit6, t0.permute_bytes(idx), t1.permute_bytes(idx)); + let high = U8x64::mask_blend(bit6, t2.permute_bytes(idx), t3.permute_bytes(idx)); + U8x64::mask_blend(bit7, low, high) + }; + + let mut chunks = magnitudes.chunks_exact(64); + let mut w = 0; + for chunk in &mut chunks { + let r = lookup(U8x64::from_slice(chunk)); + r.copy_to_slice(&mut out_magnitudes[w * 64..(w + 1) * 64]); + out_signs[w] = signs[w] & !r.cmpeq_mask(zero); + w += 1; + } + let tail = chunks.remainder(); + if !tail.is_empty() { + let mut padded = [0u8; 64]; + padded[..tail.len()].copy_from_slice(tail); + let r = lookup(U8x64::from_array(padded)).to_array(); + out_magnitudes[w * 64..n].copy_from_slice(&r[..tail.len()]); + out_signs[w] = signs[w] & !U8x64::from_array(r).cmpeq_mask(zero) & live64(tail.len()); + w += 1; + } + for s in &mut out_signs[w..] { + *s = 0; + } +} + // ──────────────────────────────────────────────────────────────────────── // Ordered 64-bit predicates — T1 gap G2 of the DuckDB→V3 translation matrix. // @@ -5529,6 +5640,96 @@ pub fn mask_shift_morton(src: &[u64], dir: MortonDir, dst: &mut [u64]) { mod tests { use super::*; + // ── odd_lut_u8 ────────────────────────────────────────────────────────── + + fn odd_lut_reference(mags: &[u8], signs: &[u64], table: &[u8; 256]) -> (Vec, Vec) { + let out: Vec = mags.iter().map(|&m| table[m as usize]).collect(); + let mut out_signs = vec![0u64; mags.len().div_ceil(64)]; + for (i, &o) in out.iter().enumerate() { + let neg = (signs[i / 64] >> (i % 64)) & 1 == 1; + if neg && o != 0 { + out_signs[i / 64] |= 1 << (i % 64); + } + } + (out, out_signs) + } + + fn odd_lut_xorshift(s: &mut u64) -> u64 { + *s ^= *s << 13; + *s ^= *s >> 7; + *s ^= *s << 17; + *s + } + + /// FAILS IF: a quarter of the table is picked by the wrong magnitude bits, + /// a sign bit survives on a zero result, the final partial chunk writes + /// rows past the end, or bits/words past the last row are left set. + #[test] + fn odd_lut_u8_matches_the_table_at_every_lane_and_length() { + let mut s = 0x9E37_79B9_7F4A_7C15u64; + // Random table with table[0] = 0 and some zero entries elsewhere, so + // the sign-clearing rule is exercised away from magnitude 0 as well. + let mut table = [0u8; 256]; + for t in table.iter_mut().skip(1) { + let r = odd_lut_xorshift(&mut s) as u8; + *t = if r % 7 == 0 { 0 } else { r }; + } + for &n in &[0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300, 1024 + 17] { + for shift in [0usize, 1, 37, 101] { + // Every magnitude appears at every lane position across shifts. + let mags: Vec = (0..n).map(|i| ((i * 37 + shift) % 256) as u8).collect(); + let words = n.div_ceil(64); + let mut signs: Vec = (0..words).map(|_| odd_lut_xorshift(&mut s)).collect(); + // Junk sign bits past the last row must not leak into the output. + if let Some(last) = signs.last_mut() { + *last |= !live64(n - (words - 1) * 64); + } + let (want, want_signs) = odd_lut_reference(&mags, &signs, &table); + let mut out = vec![0xAAu8; n + 5]; + let mut out_signs = vec![u64::MAX; words + 2]; + odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); + assert_eq!(&out[..n], &want[..], "n={n} shift={shift}"); + assert!(out[n..].iter().all(|&b| b == 0xAA), "rows past the end written, n={n}"); + assert_eq!(&out_signs[..words], &want_signs[..], "n={n} shift={shift}"); + assert!(out_signs[words..].iter().all(|&w| w == 0), "surplus words, n={n}"); + } + } + } + + /// FAILS IF: a negative value whose result rounds to zero keeps its sign + /// (a stored `-0`), or an input `-0` stays negative. Silence half: a + /// nonzero negative result keeps its sign. + #[test] + fn odd_lut_u8_stores_zero_results_as_plus_zero() { + let mut table = [0u8; 256]; + for (m, t) in table.iter_mut().enumerate() { + *t = (m / 4) as u8; + } + // -0, -3 (rounds to 0), -4 (-> -1), +3, +200 + let mags = [0u8, 3, 4, 3, 200]; + let signs = [0b00111u64]; + let mut out = [0u8; 5]; + let mut out_signs = [u64::MAX; 1]; + odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); + assert_eq!(out, [0, 0, 1, 0, 50]); + assert_eq!(out_signs[0], 0b00100); + } + + #[test] + #[should_panic(expected = "table[0] must be 0")] + fn odd_lut_u8_refuses_a_table_that_moves_zero() { + let mut table = [0u8; 256]; + table[0] = 1; + odd_lut_u8(&[0], &[0], &table, &mut [0], &mut [0]); + } + + #[test] + #[should_panic(expected = "signs holds 1 words, need 2")] + fn odd_lut_u8_refuses_a_short_sign_plane() { + let table = [0u8; 256]; + odd_lut_u8(&[0; 65], &[0], &table, &mut [0; 65], &mut [0; 2]); + } + // ── Packed-bitmask predicates + mask algebra ──────────────────────────── // // Every test compares the shipped path against an INDEPENDENT scalar From 6497f1a7182381efb9309eb77fc32894bcb5d230 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 16:03:34 +0000 Subject: [PATCH 02/11] simd: odd_lut_u8 drops a tail mask the zero test already implies Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- src/simd_masking_ops.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index 347f7d71..d9ae62ce 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3890,7 +3890,10 @@ pub fn odd_lut_u8( padded[..tail.len()].copy_from_slice(tail); let r = lookup(U8x64::from_array(padded)).to_array(); out_magnitudes[w * 64..n].copy_from_slice(&r[..tail.len()]); - out_signs[w] = signs[w] & !U8x64::from_array(r).cmpeq_mask(zero) & live64(tail.len()); + // Padding lanes are magnitude 0 and `table[0] == 0`, so their result + // is 0 and the zero test already clears their sign bits; no + // separate tail mask is needed. + out_signs[w] = signs[w] & !U8x64::from_array(r).cmpeq_mask(zero); w += 1; } for s in &mut out_signs[w..] { From 8412c8a61f75e6dcf85198ce290ce60d7172704c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 16:06:14 +0000 Subject: [PATCH 03/11] parity: odd_lut_u8 group (0x14xx) against a plain table lookup Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- crates/simd-masking-parity/src/lib.rs | 56 +++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index b1d51969..ebf236ed 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -50,14 +50,14 @@ use ndarray::simd::{ mask_shift_morton, mask_ternlog, mask_ternlog_any, mask_ternlog_assign, mask_ternlog_popcount, mask_xor, mask_xor_assign, masked_group_sum_i32, masked_group_sum_i32_via, masked_key_run_count_u32, masked_max_i32, masked_min_i32, masked_strided_group_sum, masked_sum_i32, masked_sum_wrapping_add_i32, ne_i32_to_mask, - ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, + ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, odd_lut_u8, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, ternlog, F32x16, F64x8, I16x32, I32x16, I8x16, KeyRunCarry, MortonDir, U32x16, U64x8, U8x16, }; /// Number of check groups [`run`] executes (for the log line only). -pub const CHECKS: usize = 19; +pub const CHECKS: usize = 20; /// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it. #[no_mangle] @@ -72,7 +72,7 @@ pub fn run() -> u32 { check_mask_algebra, check_care_match, check_masked_reductions, check_blend, check_morton_shift, check_predicates_under, check_set_range, check_unsigned_compare_to_mask, check_gather_scatter_group, check_f64x8_compare, check_i8x16_u8x16_lanes, check_care_match16, check_narrow_ladder, - check_mul_add_single_rounding, check_fixedpoint_requant, + check_mul_add_single_rounding, check_fixedpoint_requant, check_odd_lut_u8, ]; for g in groups { if let Err(code) = g() { @@ -2637,6 +2637,56 @@ fn oracle_fma_f64(a: f64, b: f64, c: f64) -> f64 { } } +// ── 0x14xx: odd_lut_u8 — a 256-entry magnitude table over a sign-magnitude +// lane. The reference indexes `table` with a plain array lookup and builds +// the output sign plane bit by bit; it never calls the primitive. + +fn check_odd_lut_u8() -> Result<(), u32> { + let mut rng = SplitMix64(0x1400_0000_0001); + // A random table with some zero entries away from index 0, so a sign bit + // must be cleared at nonzero magnitudes too, and every quarter differs. + let mut table = [0u8; 256]; + for t in table.iter_mut().skip(1) { + let r = rng.next() as u8; + *t = if r % 7 == 0 { 0 } else { r }; + } + const ODD_LUT_LENS: [usize; 11] = [0, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300]; + for (case, &n) in ODD_LUT_LENS.iter().enumerate() { + let case = case as u32; + let words = words_for(n); + for pass in 0..2u32 { + // Pass 0: every magnitude at every lane position modulo a stride; + // pass 1: random magnitudes. + let stride = |i: usize| ((i * 37 + 11) % 256) as u8; + let mags: Vec = (0..n) + .map(|i| if pass == 0 { stride(i) } else { rng.next() as u8 }) + .collect(); + let mut signs: Vec = (0..words).map(|_| rng.next()).collect(); + if words > 0 && n % 64 != 0 { + // Junk sign bits past the last row must not reach the output. + signs[words - 1] |= u64::MAX << (n % 64); + } + let mut out = vec![0xAAu8; n + 3]; + let mut out_signs = vec![u64::MAX; words + 1]; + odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); + for i in 0..n { + if out[i] != table[mags[i] as usize] { + return Err(0x1400 | (pass << 7) | case); + } + } + if out[n..].iter().any(|&b| b != 0xAA) { + return Err(0x1410 | (pass << 7) | case); + } + let want = + reference_mask(n, words + 1, |i| (signs[i / 64] >> (i % 64)) & 1 == 1 && table[mags[i] as usize] != 0); + if out_signs != want { + return Err(0x1420 | (pass << 7) | case); + } + } + } + Ok(()) +} + #[cfg(test)] mod oracle_tests { use super::*; From e5aa2c5e1157ac80193b4962e2d61b5ce7823f83 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 16:17:19 +0000 Subject: [PATCH 04/11] simd: odd_lut_u8 on as_chunks; throughput probe Clippy 1.99 rejects chunks_exact with a constant size. The probe examples/odd_lut_bench.rs times odd_lut_u8 against a scalar i8 table and the scalar sign-magnitude form, after checking the outputs agree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- Cargo.toml | 4 + crates/simd-masking-parity/src/lib.rs | 4 +- examples/odd_lut_bench.rs | 104 ++++++++++++++++++++++++++ src/simd_masking_ops.rs | 12 ++- 4 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 examples/odd_lut_bench.rs diff --git a/Cargo.toml b/Cargo.toml index ae75923a..1fad814c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,10 @@ required-features = ["std"] name = "campq_cascade_probe" required-features = ["std"] +[[example]] +name = "odd_lut_bench" +required-features = ["std"] + [dependencies] num-integer = { workspace = true } num-traits = { workspace = true } diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index ebf236ed..a156a3fb 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -2648,7 +2648,7 @@ fn check_odd_lut_u8() -> Result<(), u32> { let mut table = [0u8; 256]; for t in table.iter_mut().skip(1) { let r = rng.next() as u8; - *t = if r % 7 == 0 { 0 } else { r }; + *t = if r.is_multiple_of(7) { 0 } else { r }; } const ODD_LUT_LENS: [usize; 11] = [0, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300]; for (case, &n) in ODD_LUT_LENS.iter().enumerate() { @@ -2662,7 +2662,7 @@ fn check_odd_lut_u8() -> Result<(), u32> { .map(|i| if pass == 0 { stride(i) } else { rng.next() as u8 }) .collect(); let mut signs: Vec = (0..words).map(|_| rng.next()).collect(); - if words > 0 && n % 64 != 0 { + if words > 0 && !n.is_multiple_of(64) { // Junk sign bits past the last row must not reach the output. signs[words - 1] |= u64::MAX << (n % 64); } diff --git a/examples/odd_lut_bench.rs b/examples/odd_lut_bench.rs new file mode 100644 index 00000000..527f51b7 --- /dev/null +++ b/examples/odd_lut_bench.rs @@ -0,0 +1,104 @@ +//! Throughput of `odd_lut_u8` against plain table loops. +//! +//! Three paths over the same 1 Mi random magnitudes and sign plane: +//! +//! * `i8 table`: the i8 `Sym` form, one 255-entry table indexed by the +//! signed code (`table[c + 127]`), no sign plane; +//! * `u8 table + sign`: the scalar form of what `odd_lut_u8` computes, a +//! 256-entry lookup plus the sign plane cleared where the result is 0; +//! * `odd_lut_u8`: the SIMD primitive. +//! +//! The scalar and SIMD sign-magnitude outputs are compared before timing. +//! Reports the best of several runs in ns per element and the CPU features +//! the binary was built for. + +use std::hint::black_box; +use std::time::Instant; + +use ndarray::simd::odd_lut_u8; + +const N: usize = 1 << 20; +const RUNS: usize = 30; + +fn best(mut f: F) -> f64 { + let mut best = f64::INFINITY; + for _ in 0..RUNS { + let t = Instant::now(); + f(); + best = best.min(t.elapsed().as_secs_f64()); + } + best * 1e9 / N as f64 +} + +fn main() { + let mut s = 0x9E37_79B9_7F4A_7C15u64; + let mut next = || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s + }; + let mags: Vec = (0..N).map(|_| next() as u8).collect(); + let signs: Vec = (0..N / 64).map(|_| next()).collect(); + let codes: Vec = (0..N) + .map(|_| ((next() % 255) as i16 - 127) as i8) + .collect(); + + let r = 4.0f64; + let mut table = [0u8; 256]; + for (m, t) in table.iter_mut().enumerate() { + *t = ((m as f64 * r / 255.0).tanh() * 255.0).round() as u8; + } + let mut table_i8 = [0i8; 255]; + for (k, t) in table_i8.iter_mut().enumerate() { + *t = (((k as f64 - 127.0) * r / 127.0).tanh() * 127.0).round() as i8; + } + + // Correctness first: the SIMD output must equal the scalar form. + let mut out_simd = vec![0u8; N]; + let mut signs_simd = vec![0u64; N / 64]; + odd_lut_u8(&mags, &signs, &table, &mut out_simd, &mut signs_simd); + let mut out_scalar = vec![0u8; N]; + let mut signs_scalar = vec![0u64; N / 64]; + let scalar = |out: &mut [u8], out_signs: &mut [u64]| { + let rows = mags.as_chunks::<64>().0.iter(); + for (w, (chunk, o)) in rows.zip(out.as_chunks_mut::<64>().0).enumerate() { + let mut nonzero = 0u64; + for (k, (&m, d)) in chunk.iter().zip(o.iter_mut()).enumerate() { + *d = table[m as usize]; + nonzero |= u64::from(*d != 0) << k; + } + out_signs[w] = signs[w] & nonzero; + } + }; + scalar(&mut out_scalar, &mut signs_scalar); + assert_eq!(out_simd, out_scalar, "magnitudes differ"); + assert_eq!(signs_simd, signs_scalar, "sign planes differ"); + + let mut out_i8 = vec![0i8; N]; + let t_i8 = best(|| { + for (d, &c) in out_i8.iter_mut().zip(black_box(&codes)) { + *d = table_i8[(i32::from(c) + 127) as usize]; + } + black_box(&out_i8); + }); + let t_scalar = best(|| { + scalar(&mut out_scalar, &mut signs_scalar); + black_box((&out_scalar, &signs_scalar)); + }); + let t_simd = best(|| { + odd_lut_u8(black_box(&mags), black_box(&signs), &table, &mut out_simd, &mut signs_simd); + black_box((&out_simd, &signs_simd)); + }); + + println!( + "odd_lut_bench: avx512f={} avx512vbmi={} avx2={} neon={} n={N}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx512vbmi"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "neon"), + ); + println!(" i8 table (no sign plane) {t_i8:6.3} ns/elem"); + println!(" u8 table + sign, scalar {t_scalar:6.3} ns/elem"); + println!(" odd_lut_u8 {t_simd:6.3} ns/elem ({:.1}x scalar)", t_scalar / t_simd); +} diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index d9ae62ce..f1db627c 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3876,15 +3876,13 @@ pub fn odd_lut_u8( U8x64::mask_blend(bit7, low, high) }; - let mut chunks = magnitudes.chunks_exact(64); - let mut w = 0; - for chunk in &mut chunks { - let r = lookup(U8x64::from_slice(chunk)); + let (chunks, tail) = magnitudes.as_chunks::<64>(); + for (w, chunk) in chunks.iter().enumerate() { + let r = lookup(U8x64::from_array(*chunk)); r.copy_to_slice(&mut out_magnitudes[w * 64..(w + 1) * 64]); out_signs[w] = signs[w] & !r.cmpeq_mask(zero); - w += 1; } - let tail = chunks.remainder(); + let mut w = chunks.len(); if !tail.is_empty() { let mut padded = [0u8; 64]; padded[..tail.len()].copy_from_slice(tail); @@ -5675,7 +5673,7 @@ mod tests { let mut table = [0u8; 256]; for t in table.iter_mut().skip(1) { let r = odd_lut_xorshift(&mut s) as u8; - *t = if r % 7 == 0 { 0 } else { r }; + *t = if r.is_multiple_of(7) { 0 } else { r }; } for &n in &[0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300, 1024 + 17] { for shift in [0usize, 1, 37, 101] { From 0befc3775b4bdc7f56db9f01cc8c6c417116f1bd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 16:35:34 +0000 Subject: [PATCH 05/11] simd: odd_lut_u8 carries an input -0 through A negative value whose result rounds to zero is still stored as +0, but an input -0 now comes out as -0. A writer that only stores +0 never produces it, so the plain reading is unchanged; a class reading that gives -0 its own meaning (observed NaN beside a clear presence bit for not observed) keeps it, the way IEEE carries NaN. The tail needs its live mask again: padding lanes are magnitude 0 and would keep their junk sign bits. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- crates/simd-masking-parity/src/lib.rs | 5 ++-- src/simd_masking_ops.rs | 43 ++++++++++++++++----------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index a156a3fb..ca51bb82 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -2677,8 +2677,9 @@ fn check_odd_lut_u8() -> Result<(), u32> { if out[n..].iter().any(|&b| b != 0xAA) { return Err(0x1410 | (pass << 7) | case); } - let want = - reference_mask(n, words + 1, |i| (signs[i / 64] >> (i % 64)) & 1 == 1 && table[mags[i] as usize] != 0); + let want = reference_mask(n, words + 1, |i| { + (signs[i / 64] >> (i % 64)) & 1 == 1 && (table[mags[i] as usize] != 0 || mags[i] == 0) + }); if out_signs != want { return Err(0x1420 | (pass << 7) | case); } diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index f1db627c..d3edcc46 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3806,9 +3806,16 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { /// For every row `i < magnitudes.len()`: /// /// * `out_magnitudes[i] = table[magnitudes[i]]`; -/// * the output sign bit is the input sign bit, cleared where the output -/// magnitude is `0`. A result that rounds to zero is therefore stored as -/// the canonical `+0`, never `-0`, and an input `-0` comes out as `+0`. +/// * the output sign bit is the input sign bit, cleared where a nonzero +/// input gives a zero result. A negative value whose result rounds to +/// zero is stored as `+0`, never `-0`. +/// * an input `-0` (magnitude 0, sign set) comes out as `-0`. A writer +/// that only stores `+0` never produces it, so under the plain numeric +/// reading this changes nothing. A class whose reading gives `-0` a +/// meaning of its own, such as "observed, value NaN" beside a clear +/// presence bit for "not observed", keeps that meaning through the +/// function, the way IEEE arithmetic carries NaN through. Clearing it +/// would turn that reading's NaN into the value 0. /// /// Rows past `magnitudes.len()` are not written in `out_magnitudes`. In /// `out_signs` the bits past `magnitudes.len()` and any surplus words are @@ -3880,7 +3887,7 @@ pub fn odd_lut_u8( for (w, chunk) in chunks.iter().enumerate() { let r = lookup(U8x64::from_array(*chunk)); r.copy_to_slice(&mut out_magnitudes[w * 64..(w + 1) * 64]); - out_signs[w] = signs[w] & !r.cmpeq_mask(zero); + out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | U8x64::from_array(*chunk).cmpeq_mask(zero)); } let mut w = chunks.len(); if !tail.is_empty() { @@ -3888,10 +3895,10 @@ pub fn odd_lut_u8( padded[..tail.len()].copy_from_slice(tail); let r = lookup(U8x64::from_array(padded)).to_array(); out_magnitudes[w * 64..n].copy_from_slice(&r[..tail.len()]); - // Padding lanes are magnitude 0 and `table[0] == 0`, so their result - // is 0 and the zero test already clears their sign bits; no - // separate tail mask is needed. - out_signs[w] = signs[w] & !U8x64::from_array(r).cmpeq_mask(zero); + // Padding lanes are input `-0` candidates (magnitude 0), so the + // keep-`-0` rule would pass their junk sign bits; mask them off. + let kept = !U8x64::from_array(r).cmpeq_mask(zero) | U8x64::from_array(padded).cmpeq_mask(zero); + out_signs[w] = signs[w] & kept & live64(tail.len()); w += 1; } for s in &mut out_signs[w..] { @@ -5648,7 +5655,7 @@ mod tests { let mut out_signs = vec![0u64; mags.len().div_ceil(64)]; for (i, &o) in out.iter().enumerate() { let neg = (signs[i / 64] >> (i % 64)) & 1 == 1; - if neg && o != 0 { + if neg && (o != 0 || mags[i] == 0) { out_signs[i / 64] |= 1 << (i % 64); } } @@ -5698,22 +5705,22 @@ mod tests { } /// FAILS IF: a negative value whose result rounds to zero keeps its sign - /// (a stored `-0`), or an input `-0` stays negative. Silence half: a - /// nonzero negative result keeps its sign. + /// (a stored `-0`), or an input `-0` loses its sign. Silence half: a + /// nonzero negative result keeps its sign, and `+0` stays `+0`. #[test] - fn odd_lut_u8_stores_zero_results_as_plus_zero() { + fn odd_lut_u8_rounds_to_plus_zero_and_keeps_an_input_minus_zero() { let mut table = [0u8; 256]; for (m, t) in table.iter_mut().enumerate() { *t = (m / 4) as u8; } - // -0, -3 (rounds to 0), -4 (-> -1), +3, +200 - let mags = [0u8, 3, 4, 3, 200]; - let signs = [0b00111u64]; - let mut out = [0u8; 5]; + // -0, -3 (rounds to 0), -4 (-> -1), +3, +200, +0 + let mags = [0u8, 3, 4, 3, 200, 0]; + let signs = [0b000111u64]; + let mut out = [0u8; 6]; let mut out_signs = [u64::MAX; 1]; odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); - assert_eq!(out, [0, 0, 1, 0, 50]); - assert_eq!(out_signs[0], 0b00100); + assert_eq!(out, [0, 0, 1, 0, 50, 0]); + assert_eq!(out_signs[0], 0b000101); } #[test] From bfc5d23f7f3634696c19e2cca0cc27fb4d5a0367 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:06:09 +0000 Subject: [PATCH 06/11] simd: U8x64::lut256 as 16 in-lane rows; odd_lut_u8 uses it The four-permute lookup ran 5.6x slower than a scalar loop on AVX-512 without VBMI, because permute_bytes is a scalar loop on every backend but AVX-512+VBMI and nightly. lut256 reads the 256-entry table as 16 rows of 16 bytes: the low nibble picks the byte (in-lane vpshufb), the high nibble picks the row. Native on AVX-512 (BW), AVX2 (U8x64 as two U8x32 halves) and AVX-without-AVX2 (two 128-bit halves); the plain loop only in the scalar backend; swizzle_dyn on nightly. examples/odd_lut_bench.rs on AVX-512 without VBMI, 1 Mi elements: scalar u8 table + sign 0.606 ns/elem, odd_lut_u8 0.176 ns/elem. The probe's scalar reference now keeps an input -0, as odd_lut_u8 does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- examples/odd_lut_bench.rs | 6 ++-- src/simd_avx.rs | 29 ++++++++++++++++++ src/simd_avx2.rs | 43 +++++++++++++++++++++++++++ src/simd_avx512.rs | 26 ++++++++++++++++ src/simd_masking_ops.rs | 57 +++++++++++++++++------------------- src/simd_nightly/u8_types.rs | 17 +++++++++++ src/simd_scalar.rs | 9 ++++++ 7 files changed, 155 insertions(+), 32 deletions(-) diff --git a/examples/odd_lut_bench.rs b/examples/odd_lut_bench.rs index 527f51b7..8e95a9ff 100644 --- a/examples/odd_lut_bench.rs +++ b/examples/odd_lut_bench.rs @@ -5,7 +5,8 @@ //! * `i8 table`: the i8 `Sym` form, one 255-entry table indexed by the //! signed code (`table[c + 127]`), no sign plane; //! * `u8 table + sign`: the scalar form of what `odd_lut_u8` computes, a -//! 256-entry lookup plus the sign plane cleared where the result is 0; +//! 256-entry lookup plus the sign plane cleared where a nonzero input +//! gives 0; //! * `odd_lut_u8`: the SIMD primitive. //! //! The scalar and SIMD sign-magnitude outputs are compared before timing. @@ -66,7 +67,8 @@ fn main() { let mut nonzero = 0u64; for (k, (&m, d)) in chunk.iter().zip(o.iter_mut()).enumerate() { *d = table[m as usize]; - nonzero |= u64::from(*d != 0) << k; + // Keep the sign where the result is nonzero or the input is -0. + nonzero |= u64::from(*d != 0 || m == 0) << k; } out_signs[w] = signs[w] & nonzero; } diff --git a/src/simd_avx.rs b/src/simd_avx.rs index 084aef02..455040ab 100644 --- a/src/simd_avx.rs +++ b/src/simd_avx.rs @@ -613,6 +613,35 @@ impl U8x32 { Self(halves2!(_mm_shuffle_epi8, self.0, idx.0)) } + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + /// + /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 + /// bytes, the low nibble of each index selects the byte within a row + /// and the high nibble selects the row. No byte crosses a 128-bit lane, + /// so every x86 tier from SSSE3 up runs it in vector registers. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + let half = |v: __m128i| -> __m128i { + // SAFETY: SSSE3 is compiled in on this arm (module docs). Each + // 16-byte load reads `table[16 * k..16 * k + 16]` with `k < 16`. + unsafe { + let nibble = _mm_set1_epi8(0x0F); + let lo = _mm_and_si128(v, nibble); + let hi = _mm_and_si128(_mm_srli_epi16::<4>(v), nibble); + let mut acc = _mm_setzero_si128(); + for k in 0..16 { + let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); + let hit = _mm_shuffle_epi8(row, lo); + let here = _mm_cmpeq_epi8(hi, _mm_set1_epi8(k as i8)); + acc = _mm_or_si128(acc, _mm_and_si128(hit, here)); + } + acc + } + }; + let (lo, hi) = split(self.0); + Self(join(half(lo), half(hi))) + } + /// Interleave the low 8 bytes of each 128-bit half. #[inline(always)] pub fn unpack_lo_epi8(self, other: Self) -> Self { diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 20ed7997..b1db3fed 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1675,6 +1675,21 @@ impl U8x64 { Self(out) } + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + /// + /// Two [`U8x32::lut256`] halves, each 16-entry in-lane shuffles. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + let mut out = [0u8; 64]; + U8x32::from_slice(&self.0[..32]) + .lut256(table) + .copy_to_slice(&mut out[..32]); + U8x32::from_slice(&self.0[32..]) + .lut256(table) + .copy_to_slice(&mut out[32..]); + Self(out) + } + /// Sum all 64 bytes into a single `u64` without wrapping. #[inline(always)] pub fn sum_bytes_u64(self) -> u64 { @@ -3312,6 +3327,34 @@ impl U8x32 { Self(unsafe { _mm256_shuffle_epi8(self.0, idx.0) }) } + #[cfg(not(all(target_feature = "avx", not(target_feature = "avx2"))))] + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + /// + /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 + /// bytes, the low nibble of each index selects the byte within a row + /// and the high nibble selects the row. No byte crosses a 128-bit lane, + /// so every x86 tier from SSSE3 up runs it in vector registers. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + // SAFETY: AVX2 must be present at run time (caller obligation, not a + // compile-time guarantee; see `U64x8::avx2_halves`). Each 16-byte + // load reads `table[16 * k..16 * k + 16]` with `k < 16`, inside the + // 256-byte array. + unsafe { + let nibble = _mm256_set1_epi8(0x0F); + let lo = _mm256_and_si256(self.0, nibble); + let hi = _mm256_and_si256(_mm256_srli_epi16::<4>(self.0), nibble); + let mut acc = _mm256_setzero_si256(); + for k in 0..16 { + let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); + let hit = _mm256_shuffle_epi8(_mm256_broadcastsi128_si256(row), lo); + let here = _mm256_cmpeq_epi8(hi, _mm256_set1_epi8(k as i8)); + acc = _mm256_or_si256(acc, _mm256_and_si256(hit, here)); + } + Self(acc) + } + } + /// Cross-lane byte permute (full 32-byte). AVX2 has no native /// cross-lane byte permute, so this falls back to scalar — same /// shape as `simd_avx512::U8x64::permute_bytes` does on diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index cc77d6e2..5d6413a6 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -762,6 +762,32 @@ impl U8x64 { Self(unsafe { _mm512_shuffle_epi8(self.0, idx.0) }) } + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + /// + /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 + /// bytes, the low nibble of each index selects the byte within a row + /// and the high nibble selects the row. No byte crosses a 128-bit lane, + /// so every x86 tier from SSSE3 up runs it in vector registers. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + // SAFETY: AVX-512F+BW are compiled in for this type's realization + // (as for `shuffle_bytes` above). Each 16-byte load reads + // `table[16 * k..16 * k + 16]` with `k < 16`. + unsafe { + let nibble = _mm512_set1_epi8(0x0F); + let lo = _mm512_and_si512(self.0, nibble); + let hi = _mm512_and_si512(_mm512_srli_epi16::<4>(self.0), nibble); + let mut acc = _mm512_setzero_si512(); + for k in 0..16 { + let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); + let hit = _mm512_shuffle_epi8(_mm512_broadcast_i32x4(row), lo); + let here = _mm512_cmpeq_epi8_mask(hi, _mm512_set1_epi8(k as i8)); + acc = _mm512_mask_mov_epi8(acc, here, hit); + } + Self(acc) + } + } + /// Sum all 64 bytes into a single `u64` without wrapping. /// /// Uses `_mm512_sad_epu8` (groups of 8 bytes → u64 lanes) then horizontal add. diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index d3edcc46..7022dd30 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3822,12 +3822,13 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { /// written as `0`. The presence plane is not read or written: because /// `table[0]` must be `0`, a missing row (stored as `+0`) stays `+0`. /// -/// Each 64-row chunk is looked up as four 64-entry -/// [`crate::simd::U8x64::permute_bytes`] lookups, one per quarter of the -/// table, selected by the magnitude's top two bits with -/// [`crate::simd::U8x64::mask_blend`]. The final partial chunk is padded with -/// magnitude 0 and goes through the same path, so the tail cannot disagree -/// with the body. +/// Each 64-row chunk is looked up with [`crate::simd::U8x64::lut256`]: the +/// table is 16 rows of 16 bytes, the low nibble picks the byte and the high +/// nibble the row, so the lookup stays inside 128-bit lanes and runs as +/// in-lane byte shuffles on every x86 tier. The sign plane is 64 rows per +/// [`crate::simd::U8x64::cmpeq_mask`]. The final partial chunk is padded with +/// magnitude 0 and goes through the same path, with the padding lanes masked +/// off the sign plane. /// /// # Panics /// @@ -3871,34 +3872,30 @@ pub fn odd_lut_u8( assert!(signs.len() >= words, "odd_lut_u8: signs holds {} words, need {words}", signs.len()); assert!(out_signs.len() >= words, "odd_lut_u8: out_signs holds {} words, need {words}", out_signs.len()); - let quarter = |k: usize| U8x64::from_slice(&table[k * 64..(k + 1) * 64]); - let (t0, t1, t2, t3) = (quarter(0), quarter(1), quarter(2), quarter(3)); + // `U8x64::lut256` splits the 256-entry table into 16 rows of 16 bytes + // and looks each row up with an in-lane byte shuffle, so no byte crosses + // a 128-bit lane: native on every x86 tier from SSSE3 up, a plain loop + // only on the scalar backend. let zero = U8x64::splat(0); - let lookup = |idx: U8x64| -> U8x64 { - // `permute_bytes` reads the low 6 bits of each index. - let bit6 = idx.shl_epi16(1).movemask(); - let bit7 = idx.movemask(); - let low = U8x64::mask_blend(bit6, t0.permute_bytes(idx), t1.permute_bytes(idx)); - let high = U8x64::mask_blend(bit6, t2.permute_bytes(idx), t3.permute_bytes(idx)); - U8x64::mask_blend(bit7, low, high) - }; - - let (chunks, tail) = magnitudes.as_chunks::<64>(); - for (w, chunk) in chunks.iter().enumerate() { - let r = lookup(U8x64::from_array(*chunk)); - r.copy_to_slice(&mut out_magnitudes[w * 64..(w + 1) * 64]); - out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | U8x64::from_array(*chunk).cmpeq_mask(zero)); - } - let mut w = chunks.len(); + let (in_chunks, tail) = magnitudes.as_chunks::<64>(); + let (out_chunks, out_tail) = out_magnitudes[..n].as_chunks_mut::<64>(); + for (w, (chunk, out)) in in_chunks.iter().zip(out_chunks).enumerate() { + let m = U8x64::from_array(*chunk); + let r = m.lut256(table); + r.copy_to_slice(out); + // Keep the sign where the result is nonzero or the input is `-0`. + out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)); + } + let mut w = in_chunks.len(); if !tail.is_empty() { let mut padded = [0u8; 64]; padded[..tail.len()].copy_from_slice(tail); - let r = lookup(U8x64::from_array(padded)).to_array(); - out_magnitudes[w * 64..n].copy_from_slice(&r[..tail.len()]); - // Padding lanes are input `-0` candidates (magnitude 0), so the - // keep-`-0` rule would pass their junk sign bits; mask them off. - let kept = !U8x64::from_array(r).cmpeq_mask(zero) | U8x64::from_array(padded).cmpeq_mask(zero); - out_signs[w] = signs[w] & kept & live64(tail.len()); + let m = U8x64::from_array(padded); + let r = m.lut256(table); + out_tail.copy_from_slice(&r.to_array()[..tail.len()]); + // Padding lanes look like input `-0` (magnitude 0), so the keep-`-0` + // rule would pass their junk sign bits; mask them off. + out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)) & live64(tail.len()); w += 1; } for s in &mut out_signs[w..] { diff --git a/src/simd_nightly/u8_types.rs b/src/simd_nightly/u8_types.rs index 01d6c5cf..3005508c 100644 --- a/src/simd_nightly/u8_types.rs +++ b/src/simd_nightly/u8_types.rs @@ -375,6 +375,23 @@ impl U8x64 { self.0.simd_gt(core_u8x64::splat(0x7F)).to_bitmask() } + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + /// + /// Four 64-entry `swizzle_dyn` lookups, one per quarter of the table, + /// selected by the index's top two bits. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + let low = self.0 & core_u8x64::splat(0x3F); + let quarter = self.0 >> core_u8x64::splat(6); + let mut acc = core_u8x64::splat(0); + for q in 0..4u8 { + let rows = core_u8x64::from_slice(&table[64 * q as usize..64 * (q as usize + 1)]); + let here = quarter.simd_eq(core_u8x64::splat(q)); + acc = here.select(rows.swizzle_dyn(low), acc); + } + Self(acc) + } + // ── Nibble popcount LUT ──────────────────────────────────────── /// Build the nibble-popcount lookup table (replicated across all 64 bytes). diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 6a049d3c..7b1023ad 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -1767,6 +1767,15 @@ impl U8x64 { } Self(out) } + /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. + #[inline(always)] + pub fn lut256(self, table: &[u8; 256]) -> Self { + let mut out = [0u8; 64]; + for (o, &m) in out.iter_mut().zip(&self.0) { + *o = table[m as usize]; + } + Self(out) + } /// Sum all 64 bytes into a single `u64` without wrapping. #[inline(always)] pub fn sum_bytes_u64(self) -> u64 { From 46dc21af48ac54f841e0cdf65656a6a424a2c9f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:16:01 +0000 Subject: [PATCH 07/11] FACET STRIDE: P0 rule in CLAUDE.md, blackboard, guard hook '{0}..{15}' -f $128.split(): a 128-bit value is sixteen u8 positions, one mov reads them all. No byte shuffles to compute; scalar loops only in simd_scalar.rs; u8 has no sign, never mixed with i8. .claude/hooks/facet-stride-guard.sh runs after every Edit/Write and blocks added lines that use a shuffle to compute (R1), a scalar lane loop in a non-scalar backend (R2) or a u8/i8 mix (R3). Waiver: '// facet-stride: ok '. --diff and --all for ranges and audits. Baseline audit of the existing tree: 248 hits outside tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- .claude/blackboard.md | 13 + .claude/hooks/facet-stride-guard.sh | 159 ++++++++++ .../facet-stride-audit-2026-10-27.txt | 291 ++++++++++++++++++ .claude/settings.json | 16 +- CLAUDE.md | 55 ++++ 5 files changed, 533 insertions(+), 1 deletion(-) create mode 100755 .claude/hooks/facet-stride-guard.sh create mode 100644 .claude/knowledge/facet-stride-audit-2026-10-27.txt diff --git a/.claude/blackboard.md b/.claude/blackboard.md index deff2874..ffe75855 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,16 @@ +## 2026-10-27 — ███ FACET STRIDE ███ ruled P0; guard wired + +- **Rule (operator, verbatim shape):** `'{0}..{15}' -f $128.split()`. A 128-bit value is sixteen u8 positions; one mov reads them all; nothing moves to read one. Full text: `CLAUDE.md` § FACET STRIDE (top of file). + 1. Never compute with byte shuffles (`shuffle_bytes`, `permute_bytes`, `*_shuffle_epi8`, `permutexvar_epi8`, `swizzle_dyn`, `vqtbl*`): no lookups or folds assembled from shuffles + masks. A shuffle is only a genuine byte move that one mov-cast cannot do. + 2. Scalar loops only in `simd_scalar.rs`. Forbidden in AVX2/AVX-512/AVX/NEON/wasm/AMX/nightly backends when a split exists (AVX2: 2 x U8x32). + 3. u8 has no sign: never mix u8 and i8. +- **Precedent:** `ternary_match_strided16_to_mask` (`src/simd_masking_ops.rs`) — 16-byte registers loaded with plain moves, every byte position read in place. +- **How it was found:** PR #382 `odd_lut_u8` went through four-permute lookup (5.6x slower than a scalar loop on AVX-512 without VBMI), then a 16-row in-lane `vpshufb` + mask fold (`U8x64::lut256`, faster on this host but the forbidden shape, and its nightly arm did not compile: `Mask::select` needs the `Select` trait). Both are replaced by the facet-stride form. +- **Guard:** `.claude/hooks/facet-stride-guard.sh`, PostToolUse on Edit|Write|MultiEdit (`.claude/settings.json`). Checks added lines: R1 shuffle-as-compute, R2 scalar loop in a non-scalar backend, R3 u8/i8 mix. Exit 2 blocks. Waiver: `// facet-stride: ok ` (a reason is required). Also `--diff ` and `--all`. +- **Guard verified both ways:** fires on the `lut256` diff (R1 x4, R2 x3, R3 x3) and on probe lines for each rule (hook exit 2); silent on clean added lines (exit 0); honours a waiver with a reason, rejects one without. +- **Baseline audit (not clean, burn-down list):** 248 hits outside tests — R1 25, R2 195, R3 28 — in `.claude/knowledge/facet-stride-audit-2026-10-27.txt`. Heaviest: `simd_avx2.rs`, `simd_avx512.rs`, `simd_neon.rs`. Existing violations are never precedent for new ones. +- **Open:** the polyfill `shuffle_bytes` disagrees across backends when an index has bit 7 set (AVX-512/nightly zero the byte, scalar and AVX2-array read the low nibble), and AVX2 `U8x64::shuffle_bytes` / `permute_bytes` are scalar loops. Haiku sweeps of scalar loops, u8/i8 mixing and shuffle call sites are in flight; findings land here after the orchestrator reads them. + ## 2026-10-27 — wasm `mul_add` certification audit (#378): relaxed-SIMD, proof obligations, X11 - **Relaxed SIMD, tested:** no source path selects on `relaxed-simd` (only the `simd_wasm.rs` module note mentions it). The parity wasm built with `+simd128` and with `+simd128,+relaxed-simd` has a byte-identical code section (621,383 bytes); only the `name` and `target_features` custom sections differ. Byte scan of the code section for relaxed-SIMD opcodes (`0xFD`, sub-opcode `0x100..0x113`): 0 hits; a control calling `f64x2_relaxed_madd` scans to 1 hit. The relaxed build passes under node 22. New arm `scripts/masking-parity.sh wasm-relaxed` and a CI step in `realization/wasm + scalar × wasm32`, as a guard against a relaxed path appearing later. What it establishes: the shipped arithmetic is the same under both flags for this compiler; it does not test an engine executing relaxed instructions, since none are emitted. diff --git a/.claude/hooks/facet-stride-guard.sh b/.claude/hooks/facet-stride-guard.sh new file mode 100755 index 00000000..bc04af4c --- /dev/null +++ b/.claude/hooks/facet-stride-guard.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# facet-stride-guard.sh — catches violations of the FACET STRIDE shape. +# +# '{0}..{15}' -f $128.split() +# +# A 128-bit value IS sixteen u8 positions. Reading position k costs the one +# load (mov) that brought the 128 bits in; nothing is moved to read it. A byte +# shuffle only MOVES bytes between positions. Using shuffles to compute a +# result (a lookup assembled from 16 shuffles and masks, a fold across rows, +# a reduction) where the positions can simply be read is the violation. +# +# Rules, applied to ADDED lines only (existing code is audited with --all): +# R1 SHUFFLE-AS-COMPUTE byte shuffle / permute / swizzle / table-lookup +# instruction in new code. +# R2 SCALAR-IN-SIMD a per-lane scalar loop in a non-scalar SIMD +# backend (simd_avx2/avx512/avx/neon/wasm/amx, +# simd_nightly/). Scalar belongs in simd_scalar.rs +# only; on AVX2 the split (2 x U8x32, 128-bit +# halves) is the realization. +# R3 U8-I8-MIX u8 has no sign: `as i8` in a SIMD file, or a line +# that converts between u8 and i8. +# +# Escape hatch, on the line itself or the line above, with a reason: +# // facet-stride: ok +# A marker without a reason does not count. +# +# Modes: +# --hook PostToolUse hook: reads the tool JSON on stdin, checks the +# edited file's added lines against HEAD. Exit 2 = blocked. +# --diff check lines added in ..HEAD plus the working tree. +# --all audit every line of the tree (report; exit 1 if any hit). +# --file ... check the whole of the given files. +set -uo pipefail + +ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$ROOT" || exit 0 + +in_scope() { # rust sources only; test files may use scalar oracles by design + case "$1" in + */tests.rs|*/tests/*|tests/*) return 1 ;; + src/*.rs|src/*/*.rs|src/*/*/*.rs|crates/*.rs|crates/*/*.rs|crates/*/*/*.rs|crates/*/*/*/*.rs|examples/*.rs) return 0 ;; + *) return 1 ;; + esac +} +simd_backend() { # non-scalar SIMD realizations + case "$1" in + src/simd_avx2.rs|src/simd_avx512.rs|src/simd_avx.rs|src/simd_neon.rs|src/simd_wasm.rs|src/simd_amx.rs|src/simd_nightly/*.rs) return 0 ;; + *) return 1 ;; + esac +} +simd_file() { + case "$1" in src/simd*.rs|src/simd*/*.rs) return 0 ;; *) return 1 ;; esac +} + +R1='\.shuffle_bytes\(|\.permute_bytes\(|_shuffle_epi8|permutexvar_epi8|permutex2var_epi8|swizzle_dyn|vqtbl[1-4]q?|vtbl[1-4]|i8x16_swizzle|u8x16_swizzle' +R2='for +[a-z_]+ +in +0 *\.\. *(8|16|32|64)\b|for +lane +in|for +\(?[a-z_, &]*\)? +in +[a-z_.]*\.iter(_mut)?\(\)\.zip\(' +R3='as +i8\b|i8::from\(|u8::from\([^)]*i8|as +u8\b.*\bi8\b|\bi8\b.*as +u8\b' + +HITS_FILE="$(mktemp)"; trap 'rm -f "$HITS_FILE"' EXIT + +# check_lines FILE TEXT" for each line to check> +# One python pass per file (a grep per line made --all take minutes). +check_lines() { + R1="$R1" R2="$R2" R3="$R3" BACKEND="$(simd_backend "$1" && echo 1)" SIMD="$(simd_file "$1" && echo 1)" \ + python3 -c ' +import os, re, sys +f = sys.argv[1] +r1, r2, r3 = (re.compile(os.environ[k]) for k in ("R1", "R2", "R3")) +backend, simd = os.environ.get("BACKEND") == "1", os.environ.get("SIMD") == "1" +ok = re.compile(r"facet-stride: ok \S") +comment = re.compile(r"^\s*//") +prev, hits = "", 0 +for raw in sys.stdin: + n, _, t = raw.rstrip("\n").partition("\t") + waived = ok.search(prev) or ok.search(t) + prev = t + if waived or comment.match(t): + continue + for rule, rx, on in (("R1 SHUFFLE-AS-COMPUTE", r1, True), ("R2 SCALAR-IN-SIMD", r2, backend), ("R3 U8-I8-MIX", r3, simd)): + if on and rx.search(t): + hits += 1 + print(f"{f}:{n} {rule} {t.strip()[:140]}", file=sys.stderr) +print(hits) +' "$1" >> "$HITS_FILE" +} + +# added lines of FILE relative to REV (working tree included); untracked = whole file +added_lines() { + local f="$1" rev="$2" + if ! git ls-files --error-unmatch -- "$f" >/dev/null 2>&1; then + awk '{printf "%d\t%s\n", NR, $0}' "$f"; return + fi + git diff -U0 "$rev" -- "$f" | awk ' + /^@@/ { split($3, a, ","); ln = substr(a[1], 2) + 0; next } + /^\+\+\+/ { next } + /^\+/ { printf "%d\t%s\n", ln, substr($0, 2); ln++ ; next }' +} + +whole_file() { awk '{printf "%d\t%s\n", NR, $0}' "$1"; } + +# Skip `#[cfg(test)] mod tests { ... }` regions in whole-file mode: tests may +# use scalar oracles by design. +strip_tests() { + awk -F'\t' ' + $2 ~ /^#\[cfg\(test\)\]/ { skip = 1 } + skip && $2 ~ /^}/ { skip = 0; next } + !skip { print }' +} + +banner() { + cat >&2 <<'MSG' + + ███ FACET STRIDE VIOLATION ███ '{0}..{15}' -f $128.split() + A 128-bit value is 16 u8 positions; one mov reads them all. Shuffles only + MOVE bytes; never assemble a result from them. Scalar loops live only in + simd_scalar.rs (on AVX2 use the 2 x U8x32 / 128-bit-half split). u8 has no + sign: do not mix u8 and i8. + Genuine exception: `// facet-stride: ok ` on or above the line. + Rule text: CLAUDE.md § FACET STRIDE. +MSG +} + +mode="${1:---all}"; shift || true +case "$mode" in + --hook) + payload="$(cat)" + f="$(printf '%s' "$payload" | python3 -c 'import json,sys +d=json.load(sys.stdin); ti=d.get("tool_input",{}) +print(ti.get("file_path") or ti.get("notebook_path") or "")' 2>/dev/null)" + [ -z "$f" ] && exit 0 + f="${f#"$ROOT"/}" + in_scope "$f" || exit 0 + [ -f "$f" ] || exit 0 + added_lines "$f" HEAD | check_lines "$f" + ;; + --diff) + rev="${1:?--diff needs a revision}" + while IFS= read -r f; do + in_scope "$f" && [ -f "$f" ] && added_lines "$f" "$rev" | check_lines "$f" + done < <( { git diff --name-only "$rev"; git ls-files --others --exclude-standard; } | sort -u) + ;; + --file) + for f in "$@"; do f="${f#"$ROOT"/}"; [ -f "$f" ] && whole_file "$f" | strip_tests | check_lines "$f"; done + ;; + --all) + while IFS= read -r f; do whole_file "$f" | strip_tests | check_lines "$f"; done \ + < <(git ls-files 'src/*.rs' 'crates/*.rs' 'examples/*.rs' | while IFS= read -r f; do in_scope "$f" && echo "$f"; done) + ;; + *) echo "usage: $0 --hook | --diff | --all | --file ..." >&2; exit 64 ;; +esac + +hits=$(awk '{s += $1} END {print s + 0}' "$HITS_FILE") +if [ "$hits" -gt 0 ]; then + banner + echo " $hits violation(s)." >&2 + [ "$mode" = "--hook" ] && exit 2 + exit 1 +fi +exit 0 diff --git a/.claude/knowledge/facet-stride-audit-2026-10-27.txt b/.claude/knowledge/facet-stride-audit-2026-10-27.txt new file mode 100644 index 00000000..e183b668 --- /dev/null +++ b/.claude/knowledge/facet-stride-audit-2026-10-27.txt @@ -0,0 +1,291 @@ +awk: cannot open "crates/burn/src/backend.rs" (No such file or directory) +awk: cannot open "crates/burn/src/element.rs" (No such file or directory) +awk: cannot open "crates/burn/src/lib.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/adaptive_avgpool.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/avgpool.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/base.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/bool_tensor.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/conv.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/deform_conv.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/grid_sample.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/int_tensor.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/interpolate.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/macros.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/maxpool.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/mod.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/module.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/padding.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/qtensor.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/quantization.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/avgpool.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/base.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/binary.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/binary_elemwise.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/cmp.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/conv.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/maxpool.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/mod.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/simd/unary.rs" (No such file or directory) +awk: cannot open "crates/burn/src/ops/transaction.rs" (No such file or directory) +awk: cannot open "crates/burn/src/parallel.rs" (No such file or directory) +awk: cannot open "crates/burn/src/rand.rs" (No such file or directory) +awk: cannot open "crates/burn/src/sharing.rs" (No such file or directory) +awk: cannot open "crates/burn/src/storage.rs" (No such file or directory) +awk: cannot open "crates/burn/src/tensor.rs" (No such file or directory) +examples/ternlog_popcnt_gap_probe.rs:87 R1 SHUFFLE-AS-COMPUTE let l = _mm512_shuffle_epi8(lut, _mm512_and_si512(t, lo)); +examples/ternlog_popcnt_gap_probe.rs:88 R1 SHUFFLE-AS-COMPUTE let h = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16::<4>(t), lo)); +src/bitwise.rs:120 R1 SHUFFLE-AS-COMPUTE let popcnt_lo = lookup.shuffle_bytes(lo); +src/bitwise.rs:121 R1 SHUFFLE-AS-COMPUTE let popcnt_hi = lookup.shuffle_bytes(hi); +src/bitwise.rs:165 R1 SHUFFLE-AS-COMPUTE let popcnt_lo = lookup.shuffle_bytes(lo); +src/bitwise.rs:166 R1 SHUFFLE-AS-COMPUTE let popcnt_hi = lookup.shuffle_bytes(hi); +src/hpc/perm.rs:192 R1 SHUFFLE-AS-COMPUTE let composed = U8x64::from_array(self.idx).permute_bytes(U8x64::from_array(next.idx)); +src/hpc/perm.rs:268 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x64::from_array(self.idx)) +src/hpc/perm.rs:285 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x64::from_array(self.idx)) +src/hpc/perm.rs:307 R1 SHUFFLE-AS-COMPUTE U8x64::from_slice(s).permute_bytes(idx).copy_to_slice(o); +src/simd_amx.rs:508 R3 U8-I8-MIX output[i] = (energy[i] * scale).round().clamp(0.0, 127.0) as i8; +src/simd_avx.rs:107 R1 SHUFFLE-AS-COMPUTE let cnt = _mm_add_epi8(_mm_shuffle_epi8(lookup, lo), _mm_shuffle_epi8(lookup, hi)); +src/simd_avx.rs:159 R3 U8-I8-MIX .map(|(&x, &y)| (x as i8 as i64) * (y as i8 as i64)) +src/simd_avx.rs:613 R1 SHUFFLE-AS-COMPUTE Self(halves2!(_mm_shuffle_epi8, self.0, idx.0)) +src/simd_avx.rs:632 R2 SCALAR-IN-SIMD for k in 0..16 { +src/simd_avx.rs:634 R1 SHUFFLE-AS-COMPUTE let hit = _mm_shuffle_epi8(row, lo); +src/simd_avx.rs:635 R3 U8-I8-MIX let here = _mm_cmpeq_epi8(hi, _mm_set1_epi8(k as i8)); +src/simd_avx2.rs:375 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:380 R1 SHUFFLE-AS-COMPUTE let cnt = _mm256_add_epi8(_mm256_shuffle_epi8(lookup, lo), _mm256_shuffle_epi8(lookup, hi)); +src/simd_avx2.rs:392 R1 SHUFFLE-AS-COMPUTE let cnt = _mm256_add_epi8(_mm256_shuffle_epi8(lookup, lo), _mm256_shuffle_epi8(lookup, hi)); +src/simd_avx2.rs:455 R3 U8-I8-MIX .map(|(&x, &y)| (x as i8 as i64) * (y as i8 as i64)) +src/simd_avx2.rs:576 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:585 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:596 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:612 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:622 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:636 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:648 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:668 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:680 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:703 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:714 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:722 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:731 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:796 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:847 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:905 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:914 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:923 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:934 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:950 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:960 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:974 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:986 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:999 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1012 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1025 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1038 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1049 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1057 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1122 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1152 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1340 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1348 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1356 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1364 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1372 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1450 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:1458 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:1466 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:1474 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:1482 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:1526 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1537 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1558 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1577 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1586 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1594 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1602 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1615 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_avx2.rs:1617 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1629 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_avx2.rs:1631 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:1651 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1659 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:1669 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_avx2.rs:1671 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2085 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_avx2.rs:2103 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2124 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2425 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2544 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2563 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2650 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2658 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2666 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2669 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2677 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2685 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2693 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2911 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2919 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2927 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2935 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2946 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2956 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3019 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3036 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3055 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:3063 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:3111 R3 U8-I8-MIX Self(unsafe { _mm256_set1_epi8(v as i8) }) +src/simd_avx2.rs:3327 R1 SHUFFLE-AS-COMPUTE Self(unsafe { _mm256_shuffle_epi8(self.0, idx.0) }) +src/simd_avx2.rs:3348 R2 SCALAR-IN-SIMD for k in 0..16 { +src/simd_avx2.rs:3350 R1 SHUFFLE-AS-COMPUTE let hit = _mm256_shuffle_epi8(_mm256_broadcastsi128_si256(row), lo); +src/simd_avx2.rs:3351 R3 U8-I8-MIX let here = _mm256_cmpeq_epi8(hi, _mm256_set1_epi8(k as i8)); +src/simd_avx2.rs:3369 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3574 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3587 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3601 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3648 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x32::from_array(idx)) +src/simd_avx2.rs:3650 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3663 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3674 R1 SHUFFLE-AS-COMPUTE let counts = lut.shuffle_bytes(U8x32::from_array(idx)).to_array(); +src/simd_avx2.rs:3675 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:3745 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:3782 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:3847 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:3862 R2 SCALAR-IN-SIMD for lane in [0usize, 31, 32, 63] { +src/simd_avx2.rs:4340 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:529 R3 U8-I8-MIX Self(unsafe { _mm512_set1_epi8(v as i8) }) +src/simd_avx512.rs:729 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:762 R1 SHUFFLE-AS-COMPUTE Self(unsafe { _mm512_shuffle_epi8(self.0, idx.0) }) +src/simd_avx512.rs:781 R2 SCALAR-IN-SIMD for k in 0..16 { +src/simd_avx512.rs:783 R1 SHUFFLE-AS-COMPUTE let hit = _mm512_shuffle_epi8(_mm512_broadcast_i32x4(row), lo); +src/simd_avx512.rs:784 R3 U8-I8-MIX let here = _mm512_cmpeq_epi8_mask(hi, _mm512_set1_epi8(k as i8)); +src/simd_avx512.rs:828 R1 SHUFFLE-AS-COMPUTE _mm512_permutexvar_epi8(idx, v) +src/simd_avx512.rs:1177 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:1316 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:1338 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:1759 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_avx512.rs:3114 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:3115 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; +src/simd_avx512.rs:3174 R3 U8-I8-MIX let clamped = _mm_min_epu8(_mm_abs_epi8(v), _mm_set1_epi8(0x7f_u8 as i8)); +src/simd_avx512.rs:3474 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_avx512.rs:3522 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_avx512.rs:3568 R3 U8-I8-MIX core::arch::x86_64::_mm256_min_epu8(raw_abs, core::arch::x86_64::_mm256_set1_epi8(0x7f_u8 as i8)); +src/simd_avx512.rs:3576 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:3630 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:3640 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:3677 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:5108 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5114 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5117 R1 SHUFFLE-AS-COMPUTE let perm = v.permute_bytes(U8x64::from_slice(&idx)); +src/simd_avx512.rs:5126 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5132 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5135 R1 SHUFFLE-AS-COMPUTE let perm = v.permute_bytes(U8x64::from_slice(&idx)); +src/simd_avx512.rs:5138 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5192 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5198 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5212 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5284 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5285 R3 U8-I8-MIX a[i] = (i + 1) as i8; +src/simd_avx512.rs:5286 R3 U8-I8-MIX b[i] = (64 - i) as i8; +src/simd_avx512.rs:5293 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5314 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5315 R3 U8-I8-MIX a[i] = (i as i32 - 32) as i8; +src/simd_avx512.rs:5321 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx512.rs:5333 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5334 R3 U8-I8-MIX a[i] = (i + 1) as i8; +src/simd_avx512.rs:5335 R3 U8-I8-MIX b[i] = (32 - i) as i8; +src/simd_avx512.rs:5341 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5349 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5350 R3 U8-I8-MIX a[i] = (i as i32 - 16) as i8; +src/simd_avx512.rs:5356 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5383 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5390 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:5414 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:5421 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:5452 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_fixedpoint.rs:275 R3 U8-I8-MIX out[base + k] = r[k] as i8; +src/simd_fixedpoint.rs:282 R3 U8-I8-MIX out[i] = v.max(act_min).min(act_max) as i8; +src/simd_neon.rs:639 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:651 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:671 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:683 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:695 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:704 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:713 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:852 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:1005 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1017 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1030 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1043 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1056 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1069 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1081 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1090 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1208 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1235 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:1267 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1375 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:1477 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_neon.rs:1857 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_neon.rs:1946 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:1985 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:2511 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_neon.rs:2512 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; +src/simd_neon.rs:2567 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_neon.rs:2603 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_neon.rs:2670 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_nightly/bf16_types.rs:80 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/bf16_types.rs:90 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/bf16_types.rs:143 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/bf16_types.rs:153 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/exotic_methods.rs:39 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_nightly/exotic_methods.rs:70 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_nightly/exotic_methods.rs:72 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/exotic_methods.rs:107 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_nightly/exotic_methods.rs:138 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_nightly/exotic_methods.rs:140 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/exotic_methods.rs:172 R2 SCALAR-IN-SIMD for lane in 0..4 { +src/simd_nightly/exotic_methods.rs:174 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/exotic_methods.rs:212 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_nightly/exotic_methods.rs:242 R2 SCALAR-IN-SIMD for lane in 0..2 { +src/simd_nightly/exotic_methods.rs:244 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/exotic_methods.rs:279 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_nightly/exotic_methods.rs:310 R2 SCALAR-IN-SIMD for lane in 0..2 { +src/simd_nightly/exotic_methods.rs:312 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/exotic_methods.rs:344 R2 SCALAR-IN-SIMD for lane in 0..2 { +src/simd_nightly/exotic_methods.rs:346 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/f16_types.rs:175 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/f16_types.rs:185 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_nightly/f32_types.rs:103 R2 SCALAR-IN-SIMD for (o, &i) in out.iter_mut().zip(idx.iter()) { +src/simd_nightly/u8_types.rs:390 R1 SHUFFLE-AS-COMPUTE acc = here.select(rows.swizzle_dyn(low), acc); +src/simd_nightly/u_word_types.rs:42 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/u_word_types.rs:59 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_nightly/u_word_types.rs:684 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_nightly/w1a_types.rs:531 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_nightly/w1a_types.rs:663 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_scalar.rs:810 R3 U8-I8-MIX v.clamp(-128, 127) as i8 +src/simd_scalar.rs:1885 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; +src/simd_scalar.rs:1937 R3 U8-I8-MIX let mut o = [0 as i8; 16]; +src/simd_scalar.rs:1954 R3 U8-I8-MIX let mut o = [0 as i8; 16]; +src/simd_scalar.rs:1971 R3 U8-I8-MIX let mut o = [0 as i8; 16]; +src/simd_scalar.rs:1988 R3 U8-I8-MIX let mut o = [0 as i8; 16]; +src/simd_wasm.rs:302 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:311 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:320 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:452 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:678 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_wasm.rs:687 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_wasm.rs:798 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_wasm.rs:922 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:923 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; +src/simd_wasm.rs:1423 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_wasm.rs:1506 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:1832 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_wasm.rs:1922 R3 U8-I8-MIX (self.next_u64() & 0xFF) as u8 as i8 + + ███ FACET STRIDE VIOLATION ███ '{0}..{15}' -f $128.split() + A 128-bit value is 16 u8 positions; one mov reads them all. Shuffles only + MOVE bytes; never assemble a result from them. Scalar loops live only in + simd_scalar.rs (on AVX2 use the 2 x U8x32 / 128-bit-half split). u8 has no + sign: do not mix u8 and i8. + Genuine exception: `// facet-stride: ok ` on or above the line. + Rule text: CLAUDE.md § FACET STRIDE. + 248 violation(s). diff --git a/.claude/settings.json b/.claude/settings.json index 88f1c3d6..2b6f94aa 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -253,5 +253,19 @@ "Write(./.git/**)", "Edit(./.git/**)" ] + }, + "hooks": { + "PostToolUse": [ + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/.claude/hooks/facet-stride-guard.sh --hook", + "timeout": 60 + } + ] + } + ] } -} \ No newline at end of file +} diff --git a/CLAUDE.md b/CLAUDE.md index d2c1529a..acbe6aa6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,61 @@ > HPC Rust Transformation — porting `adaworldapi/rustynum` features into this ndarray fork. +## ███ FACET STRIDE ███ — P0, read before writing any SIMD or byte code + +``` + ╔══════════════════════════════════════════════════════════════════╗ + ║ ║ + ║ '{0}..{15}' -f $128.split() ║ + ║ ║ + ║ A 128-bit value IS sixteen u8 positions. ║ + ║ ONE mov reads all sixteen. Nothing moves to read one. ║ + ║ ║ + ╚══════════════════════════════════════════════════════════════════╝ +``` + +Operator-ruled 2026-10-27, after a session built a byte table lookup out of +sixteen in-lane shuffles and masks three times over. **This is the best +practice. Do not re-derive it, do not argue it, apply it.** The precedent is +`ternary_match_strided16_to_mask` in `src/simd_masking_ops.rs`: it loads each +16-byte register with plain moves and reads every byte position in place, +with no shuffle anywhere. + +1. **Read positions, don't move them.** A 128-bit (or 256/512-bit) value is + read as `{0}..{15}` positions by the load that brought it in. Never use a + byte shuffle, permute, swizzle or table-lookup instruction + (`shuffle_bytes`, `permute_bytes`, `*_shuffle_epi8`, `permutexvar_epi8`, + `swizzle_dyn`, `vqtbl*`) to compute a result: no lookups assembled from + shuffles plus masks, no folds or reductions across rows built from + shuffles. A shuffle is only for a genuine byte MOVE from one position to + another, and only when a cast with one mov cannot do it. +2. **Scalar only in the scalar backend.** A per-lane scalar loop belongs in + `src/simd_scalar.rs` and nowhere else. In `simd_avx2.rs`, `simd_avx512.rs`, + `simd_avx.rs`, `simd_neon.rs`, `simd_wasm.rs`, `simd_amx.rs` and + `simd_nightly/`, a scalar loop is forbidden when a split realization exists + (on AVX2: two `U8x32` / two 128-bit halves). +3. **u8 has no sign. Never mix u8 with i8.** No `as i8` on unsigned data in + SIMD code, no signed compare on unsigned bytes, no u8 <-> i8 conversion + lines. Use the unsigned type end to end. + +**Guard:** `.claude/hooks/facet-stride-guard.sh` runs after every Edit/Write +(wired in `.claude/settings.json`) and checks the lines the change adds: +R1 shuffle-as-compute, R2 scalar loop in a non-scalar backend, R3 u8/i8 mix. +A hit blocks the edit (exit 2) with the rule text. A genuine exception carries +`// facet-stride: ok ` on the line or the line above; a marker without +a reason is rejected. + +```sh +bash .claude/hooks/facet-stride-guard.sh --diff origin/master # lines a branch adds +bash .claude/hooks/facet-stride-guard.sh --all # whole-tree audit +``` + +The existing tree is NOT clean: the 2026-10-27 audit found 248 hits outside +tests (25 R1, 195 R2, 28 R3), listed in +`.claude/knowledge/facet-stride-audit-2026-10-27.txt`. They are a burn-down +list, not a precedent: existing code that violates the shape is never a +reason to write more of it. + ## Quick Context - **What**: High-performance linear algebra with pluggable BLAS backends (Native SIMD, MKL, OpenBLAS) - **Source**: `adaworldapi/rustynum` — reference GEMM, SIMD, and FFI implementations From 2961449a4cbb8747b145a051345471c8f6e4d260 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:19:45 +0000 Subject: [PATCH 08/11] simd: odd_lut_u8 in facet-stride shape; remove U8x64::lut256 Sixteen magnitudes are one 128-bit value read in place as positions {0}..{15}; each position addresses the table directly. No shuffles, no result assembled from masks. lut256 (16 in-lane vpshufb + mask fold) is removed from all six backends; it was the forbidden shape and its nightly arm did not compile. examples/odd_lut_bench.rs, AVX-512 without VBMI, 1 Mi elements: scalar u8 table + sign 0.606 ns/elem, odd_lut_u8 0.496 ns/elem. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- src/simd_avx.rs | 29 ------------------------ src/simd_avx2.rs | 43 ------------------------------------ src/simd_avx512.rs | 26 ---------------------- src/simd_masking_ops.rs | 37 +++++++++++++++++++------------ src/simd_nightly/u8_types.rs | 17 -------------- src/simd_scalar.rs | 9 -------- 6 files changed, 23 insertions(+), 138 deletions(-) diff --git a/src/simd_avx.rs b/src/simd_avx.rs index 455040ab..084aef02 100644 --- a/src/simd_avx.rs +++ b/src/simd_avx.rs @@ -613,35 +613,6 @@ impl U8x32 { Self(halves2!(_mm_shuffle_epi8, self.0, idx.0)) } - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - /// - /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 - /// bytes, the low nibble of each index selects the byte within a row - /// and the high nibble selects the row. No byte crosses a 128-bit lane, - /// so every x86 tier from SSSE3 up runs it in vector registers. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - let half = |v: __m128i| -> __m128i { - // SAFETY: SSSE3 is compiled in on this arm (module docs). Each - // 16-byte load reads `table[16 * k..16 * k + 16]` with `k < 16`. - unsafe { - let nibble = _mm_set1_epi8(0x0F); - let lo = _mm_and_si128(v, nibble); - let hi = _mm_and_si128(_mm_srli_epi16::<4>(v), nibble); - let mut acc = _mm_setzero_si128(); - for k in 0..16 { - let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); - let hit = _mm_shuffle_epi8(row, lo); - let here = _mm_cmpeq_epi8(hi, _mm_set1_epi8(k as i8)); - acc = _mm_or_si128(acc, _mm_and_si128(hit, here)); - } - acc - } - }; - let (lo, hi) = split(self.0); - Self(join(half(lo), half(hi))) - } - /// Interleave the low 8 bytes of each 128-bit half. #[inline(always)] pub fn unpack_lo_epi8(self, other: Self) -> Self { diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index b1db3fed..20ed7997 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1675,21 +1675,6 @@ impl U8x64 { Self(out) } - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - /// - /// Two [`U8x32::lut256`] halves, each 16-entry in-lane shuffles. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - let mut out = [0u8; 64]; - U8x32::from_slice(&self.0[..32]) - .lut256(table) - .copy_to_slice(&mut out[..32]); - U8x32::from_slice(&self.0[32..]) - .lut256(table) - .copy_to_slice(&mut out[32..]); - Self(out) - } - /// Sum all 64 bytes into a single `u64` without wrapping. #[inline(always)] pub fn sum_bytes_u64(self) -> u64 { @@ -3327,34 +3312,6 @@ impl U8x32 { Self(unsafe { _mm256_shuffle_epi8(self.0, idx.0) }) } - #[cfg(not(all(target_feature = "avx", not(target_feature = "avx2"))))] - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - /// - /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 - /// bytes, the low nibble of each index selects the byte within a row - /// and the high nibble selects the row. No byte crosses a 128-bit lane, - /// so every x86 tier from SSSE3 up runs it in vector registers. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - // SAFETY: AVX2 must be present at run time (caller obligation, not a - // compile-time guarantee; see `U64x8::avx2_halves`). Each 16-byte - // load reads `table[16 * k..16 * k + 16]` with `k < 16`, inside the - // 256-byte array. - unsafe { - let nibble = _mm256_set1_epi8(0x0F); - let lo = _mm256_and_si256(self.0, nibble); - let hi = _mm256_and_si256(_mm256_srli_epi16::<4>(self.0), nibble); - let mut acc = _mm256_setzero_si256(); - for k in 0..16 { - let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); - let hit = _mm256_shuffle_epi8(_mm256_broadcastsi128_si256(row), lo); - let here = _mm256_cmpeq_epi8(hi, _mm256_set1_epi8(k as i8)); - acc = _mm256_or_si256(acc, _mm256_and_si256(hit, here)); - } - Self(acc) - } - } - /// Cross-lane byte permute (full 32-byte). AVX2 has no native /// cross-lane byte permute, so this falls back to scalar — same /// shape as `simd_avx512::U8x64::permute_bytes` does on diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 5d6413a6..cc77d6e2 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -762,32 +762,6 @@ impl U8x64 { Self(unsafe { _mm512_shuffle_epi8(self.0, idx.0) }) } - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - /// - /// Built from 16-entry in-lane shuffles: the table is 16 rows of 16 - /// bytes, the low nibble of each index selects the byte within a row - /// and the high nibble selects the row. No byte crosses a 128-bit lane, - /// so every x86 tier from SSSE3 up runs it in vector registers. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - // SAFETY: AVX-512F+BW are compiled in for this type's realization - // (as for `shuffle_bytes` above). Each 16-byte load reads - // `table[16 * k..16 * k + 16]` with `k < 16`. - unsafe { - let nibble = _mm512_set1_epi8(0x0F); - let lo = _mm512_and_si512(self.0, nibble); - let hi = _mm512_and_si512(_mm512_srli_epi16::<4>(self.0), nibble); - let mut acc = _mm512_setzero_si512(); - for k in 0..16 { - let row = _mm_loadu_si128(table.as_ptr().add(16 * k).cast::<__m128i>()); - let hit = _mm512_shuffle_epi8(_mm512_broadcast_i32x4(row), lo); - let here = _mm512_cmpeq_epi8_mask(hi, _mm512_set1_epi8(k as i8)); - acc = _mm512_mask_mov_epi8(acc, here, hit); - } - Self(acc) - } - } - /// Sum all 64 bytes into a single `u64` without wrapping. /// /// Uses `_mm512_sad_epu8` (groups of 8 bytes → u64 lanes) then horizontal add. diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index 7022dd30..197fc0ba 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3822,10 +3822,9 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { /// written as `0`. The presence plane is not read or written: because /// `table[0]` must be `0`, a missing row (stored as `+0`) stays `+0`. /// -/// Each 64-row chunk is looked up with [`crate::simd::U8x64::lut256`]: the -/// table is 16 rows of 16 bytes, the low nibble picks the byte and the high -/// nibble the row, so the lookup stays inside 128-bit lanes and runs as -/// in-lane byte shuffles on every x86 tier. The sign plane is 64 rows per +/// Facet stride: every 16 magnitudes are one 128-bit value whose positions +/// `{0}..{15}` are read where they sit, and each position addresses the table +/// directly. No byte is shuffled. The sign plane is 64 rows per /// [`crate::simd::U8x64::cmpeq_mask`]. The final partial chunk is padded with /// magnitude 0 and goes through the same path, with the padding lanes masked /// off the sign plane. @@ -3872,17 +3871,26 @@ pub fn odd_lut_u8( assert!(signs.len() >= words, "odd_lut_u8: signs holds {} words, need {words}", signs.len()); assert!(out_signs.len() >= words, "odd_lut_u8: out_signs holds {} words, need {words}", out_signs.len()); - // `U8x64::lut256` splits the 256-entry table into 16 rows of 16 bytes - // and looks each row up with an in-lane byte shuffle, so no byte crosses - // a 128-bit lane: native on every x86 tier from SSSE3 up, a plain loop - // only on the scalar backend. + // FACET STRIDE, '{0}..{15}' -f $128.split(): sixteen magnitudes are one + // 128-bit value, read where they sit; each position addresses the table + // directly. No byte is shuffled and no result is assembled from masks. + let lookup = |positions: &[u8; 16]| -> [u8; 16] { positions.map(|m| table[usize::from(m)]) }; + let fill = |chunk: &[u8; 64], out: &mut [u8; 64]| { + for (positions, o) in chunk + .as_chunks::<16>() + .0 + .iter() + .zip(out.as_chunks_mut::<16>().0) + { + *o = lookup(positions); + } + }; let zero = U8x64::splat(0); let (in_chunks, tail) = magnitudes.as_chunks::<64>(); let (out_chunks, out_tail) = out_magnitudes[..n].as_chunks_mut::<64>(); for (w, (chunk, out)) in in_chunks.iter().zip(out_chunks).enumerate() { - let m = U8x64::from_array(*chunk); - let r = m.lut256(table); - r.copy_to_slice(out); + fill(chunk, out); + let (m, r) = (U8x64::from_array(*chunk), U8x64::from_array(*out)); // Keep the sign where the result is nonzero or the input is `-0`. out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)); } @@ -3890,9 +3898,10 @@ pub fn odd_lut_u8( if !tail.is_empty() { let mut padded = [0u8; 64]; padded[..tail.len()].copy_from_slice(tail); - let m = U8x64::from_array(padded); - let r = m.lut256(table); - out_tail.copy_from_slice(&r.to_array()[..tail.len()]); + let mut looked_up = [0u8; 64]; + fill(&padded, &mut looked_up); + out_tail.copy_from_slice(&looked_up[..tail.len()]); + let (m, r) = (U8x64::from_array(padded), U8x64::from_array(looked_up)); // Padding lanes look like input `-0` (magnitude 0), so the keep-`-0` // rule would pass their junk sign bits; mask them off. out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)) & live64(tail.len()); diff --git a/src/simd_nightly/u8_types.rs b/src/simd_nightly/u8_types.rs index 3005508c..01d6c5cf 100644 --- a/src/simd_nightly/u8_types.rs +++ b/src/simd_nightly/u8_types.rs @@ -375,23 +375,6 @@ impl U8x64 { self.0.simd_gt(core_u8x64::splat(0x7F)).to_bitmask() } - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - /// - /// Four 64-entry `swizzle_dyn` lookups, one per quarter of the table, - /// selected by the index's top two bits. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - let low = self.0 & core_u8x64::splat(0x3F); - let quarter = self.0 >> core_u8x64::splat(6); - let mut acc = core_u8x64::splat(0); - for q in 0..4u8 { - let rows = core_u8x64::from_slice(&table[64 * q as usize..64 * (q as usize + 1)]); - let here = quarter.simd_eq(core_u8x64::splat(q)); - acc = here.select(rows.swizzle_dyn(low), acc); - } - Self(acc) - } - // ── Nibble popcount LUT ──────────────────────────────────────── /// Build the nibble-popcount lookup table (replicated across all 64 bytes). diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 7b1023ad..6a049d3c 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -1767,15 +1767,6 @@ impl U8x64 { } Self(out) } - /// 256-entry byte table lookup: lane `i` becomes `table[self[i]]`. - #[inline(always)] - pub fn lut256(self, table: &[u8; 256]) -> Self { - let mut out = [0u8; 64]; - for (o, &m) in out.iter_mut().zip(&self.0) { - *o = table[m as usize]; - } - Self(out) - } /// Sum all 64 bytes into a single `u64` without wrapping. #[inline(always)] pub fn sum_bytes_u64(self) -> u64 { From d5d8a0cff7e90d9bba7a30fcecf3ce0c61918611 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:39:23 +0000 Subject: [PATCH 09/11] facet-stride guard: context-aware checks, exclusions, --root The checker reads the whole file and reports only the lines in scope, so hook mode judges added lines with context: test modules (including multi-line cfg(all(test, ..)) and mod *tests) are skipped, a shuffle wrapper's own definition is not compute, and R3 is typed (cast to i8 in a U8 type, to u8 in an I8 type, or a u8<->i8 line). Exclusions with a mandatory reason for cipher rounds, permutation maps and a benchmark of the anti-pattern. --root audits another checkout. Audit: ndarray 166 hits (R1 7, R2 157, R3 2); lance-graph 10 (all R1 nibble-table popcount). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- .claude/blackboard.md | 4 +- .claude/hooks/facet-stride-exclude.txt | 6 + .claude/hooks/facet-stride-guard.sh | 122 +++++-- .../facet-stride-audit-2026-10-27.txt | 326 +++++++++++------- ...et-stride-audit-lance-graph-2026-10-27.txt | 19 + CLAUDE.md | 19 +- 6 files changed, 347 insertions(+), 149 deletions(-) create mode 100644 .claude/hooks/facet-stride-exclude.txt create mode 100644 .claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt diff --git a/.claude/blackboard.md b/.claude/blackboard.md index ffe75855..f099c1a6 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -8,7 +8,9 @@ - **How it was found:** PR #382 `odd_lut_u8` went through four-permute lookup (5.6x slower than a scalar loop on AVX-512 without VBMI), then a 16-row in-lane `vpshufb` + mask fold (`U8x64::lut256`, faster on this host but the forbidden shape, and its nightly arm did not compile: `Mask::select` needs the `Select` trait). Both are replaced by the facet-stride form. - **Guard:** `.claude/hooks/facet-stride-guard.sh`, PostToolUse on Edit|Write|MultiEdit (`.claude/settings.json`). Checks added lines: R1 shuffle-as-compute, R2 scalar loop in a non-scalar backend, R3 u8/i8 mix. Exit 2 blocks. Waiver: `// facet-stride: ok ` (a reason is required). Also `--diff ` and `--all`. - **Guard verified both ways:** fires on the `lut256` diff (R1 x4, R2 x3, R3 x3) and on probe lines for each rule (hook exit 2); silent on clean added lines (exit 0); honours a waiver with a reason, rejects one without. -- **Baseline audit (not clean, burn-down list):** 248 hits outside tests — R1 25, R2 195, R3 28 — in `.claude/knowledge/facet-stride-audit-2026-10-27.txt`. Heaviest: `simd_avx2.rs`, `simd_avx512.rs`, `simd_neon.rs`. Existing violations are never precedent for new ones. +- **Baseline audit (not clean, burn-down list):** first pass 248 hits; after the guard learned test modules (incl. multi-line `cfg(all(test, ..))` and `mod *tests`), shuffle-wrapper definitions, typed R3 (cast to i8 inside a U8 type, to u8 inside an I8 type, or a u8<->i8 line) and the exclusions: **166** — R1 7 (nibble-table popcount: `bitwise.rs` x4, `simd_avx.rs`, `simd_avx2.rs` x2), R2 157 (`simd_avx2.rs` 81, mostly the array-backed AVX2 types; NEON 26; nightly `exotic_methods` 16; AVX-512 12; wasm 10; other nightly 12), R3 2 (u8 `splat` -> `set1_epi8(v as i8)`, AVX2 and AVX-512). Mapped by 10 Haiku `substrate-mapper` chunks (A-J, <= 30 hits each); 4 reported stale line numbers (the baseline predated the `lut256` removal) and located hits by text. Spot-checked J against source: correct. +- **Exclusions** (`.claude/hooks/facet-stride-exclude.txt`): `src/hpc/perm.rs` (Perm64 permutation map), `vendor/chacha20/*` (cipher rotate-by-8/16 via fixed byte rotation), `examples/ternlog_popcnt_gap_probe.rs` (measures the anti-pattern on purpose). +- **lance-graph** (`--root`): 10 hits, all R1 nibble-table popcount (`holograph/src/hamming.rs` x2, `blasgraph/ndarray_bridge.rs` x8), also raw intrinsics outside `ndarray::simd`. R3 candidates (29 by the broad pattern) were all signed-to-signed and drop out under the typed rule. No crypto shuffles found there. List: `.claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt`. - **Open:** the polyfill `shuffle_bytes` disagrees across backends when an index has bit 7 set (AVX-512/nightly zero the byte, scalar and AVX2-array read the low nibble), and AVX2 `U8x64::shuffle_bytes` / `permute_bytes` are scalar loops. Haiku sweeps of scalar loops, u8/i8 mixing and shuffle call sites are in flight; findings land here after the orchestrator reads them. ## 2026-10-27 — wasm `mul_add` certification audit (#378): relaxed-SIMD, proof obligations, X11 diff --git a/.claude/hooks/facet-stride-exclude.txt b/.claude/hooks/facet-stride-exclude.txt new file mode 100644 index 00000000..0c28dfd4 --- /dev/null +++ b/.claude/hooks/facet-stride-exclude.txt @@ -0,0 +1,6 @@ +# facet-stride exclusions: | | (a reason is required) +# Only for code whose byte shuffle IS the defined operation, never for lookups +# or folds. Each entry was read before it was added. +src/hpc/perm.rs | R1 | Perm64 is a 64-lane permutation map; composing and materializing it is a genuine byte move, the operation itself (module doc: "composable maps"). +examples/ternlog_popcnt_gap_probe.rs | R1 | Benchmark that measures the nibble-LUT popcount as its baseline against ternlog + popcnt; the shuffle is the thing being measured. +vendor/chacha20/* | R1 | ChaCha20 rotate-left by 8 and 16 bits is a fixed byte rotation (pshufb / vqtbl1q with a constant mask): the cipher's defined operation, in vendored upstream code. diff --git a/.claude/hooks/facet-stride-guard.sh b/.claude/hooks/facet-stride-guard.sh index bc04af4c..098a75e1 100755 --- a/.claude/hooks/facet-stride-guard.sh +++ b/.claude/hooks/facet-stride-guard.sh @@ -32,24 +32,39 @@ # --file ... check the whole of the given files. set -uo pipefail -ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)" +GUARD_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(git -C "$GUARD_DIR" rev-parse --show-toplevel 2>/dev/null || pwd)" +if [ "${1:-}" = "--root" ]; then ROOT="$(cd "${2:?--root needs a directory}" && pwd)"; shift 2; fi cd "$ROOT" || exit 0 +NDARRAY=0; [ -f src/simd_scalar.rs ] && [ -f src/simd_masking_ops.rs ] && NDARRAY=1 +# Exclusions: /.claude/hooks/facet-stride-exclude.txt, lines +# | | +# for code whose byte shuffle IS the defined operation (cipher rounds, +# permutation maps) or a deliberate measurement of the anti-pattern. +# A line without a reason is ignored. +EXCLUDE_FILE="$ROOT/.claude/hooks/facet-stride-exclude.txt" in_scope() { # rust sources only; test files may use scalar oracles by design case "$1" in */tests.rs|*/tests/*|tests/*) return 1 ;; - src/*.rs|src/*/*.rs|src/*/*/*.rs|crates/*.rs|crates/*/*.rs|crates/*/*/*.rs|crates/*/*/*/*.rs|examples/*.rs) return 0 ;; + */benches/*|benches/*) return 1 ;; + *.rs) return 0 ;; *) return 1 ;; esac } -simd_backend() { # non-scalar SIMD realizations +simd_backend() { # non-scalar SIMD realizations (ndarray only) + [ "$NDARRAY" = 1 ] || return 1 case "$1" in src/simd_avx2.rs|src/simd_avx512.rs|src/simd_avx.rs|src/simd_neon.rs|src/simd_wasm.rs|src/simd_amx.rs|src/simd_nightly/*.rs) return 0 ;; *) return 1 ;; esac } simd_file() { - case "$1" in src/simd*.rs|src/simd*/*.rs) return 0 ;; *) return 1 ;; esac + if [ "$NDARRAY" = 1 ]; then + case "$1" in src/simd*.rs|src/simd*/*.rs) return 0 ;; *) return 1 ;; esac + fi + # Elsewhere: any file that touches SIMD at all. + grep -qE 'core::arch|std::arch|ndarray::simd' "$1" 2>/dev/null } R1='\.shuffle_bytes\(|\.permute_bytes\(|_shuffle_epi8|permutexvar_epi8|permutex2var_epi8|swizzle_dyn|vqtbl[1-4]q?|vtbl[1-4]|i8x16_swizzle|u8x16_swizzle' @@ -58,28 +73,87 @@ R3='as +i8\b|i8::from\(|u8::from\([^)]*i8|as +u8\b.*\bi8\b|\bi8\b.*as +u8\b' HITS_FILE="$(mktemp)"; trap 'rm -f "$HITS_FILE"' EXIT -# check_lines FILE TEXT" for each line to check> -# One python pass per file (a grep per line made --all take minutes). +# check_lines FILE TEXT" for the lines in scope> +# The checker reads the WHOLE file for context (test-module brace depth, the +# enclosing impl type and fn) and reports only the line numbers it was given. check_lines() { - R1="$R1" R2="$R2" R3="$R3" BACKEND="$(simd_backend "$1" && echo 1)" SIMD="$(simd_file "$1" && echo 1)" \ - python3 -c ' -import os, re, sys + R1="$R1" R2="$R2" BACKEND="$(simd_backend "$1" && echo 1)" SIMD="$(simd_file "$1" && echo 1)" \ + EXCLUDE_FILE="$EXCLUDE_FILE" python3 -c ' +import fnmatch, os, re, sys f = sys.argv[1] -r1, r2, r3 = (re.compile(os.environ[k]) for k in ("R1", "R2", "R3")) +want = set() +for raw in sys.stdin: + n, _, _ = raw.partition("\t") + if n.strip().isdigit(): + want.add(int(n)) +r1, r2 = re.compile(os.environ["R1"]), re.compile(os.environ["R2"]) backend, simd = os.environ.get("BACKEND") == "1", os.environ.get("SIMD") == "1" +excluded = set() +try: + for line in open(os.environ["EXCLUDE_FILE"]): + parts = [x.strip() for x in line.split("|")] + if len(parts) == 3 and parts[2] and not parts[0].startswith("#") and fnmatch.fnmatch(f, parts[0]): + excluded |= {r.strip() for r in parts[1].split(",")} +except OSError: + pass +# A byte-shuffle wrapper may contain the instruction: it IS the move. +shuffle_def = re.compile(r"^(shuffle_bytes|permute_bytes|permute_bytes_vbmi|swizzle\w*|shuffle\w*|permute\w*)$") ok = re.compile(r"facet-stride: ok \S") comment = re.compile(r"^\s*//") -prev, hits = "", 0 -for raw in sys.stdin: - n, _, t = raw.rstrip("\n").partition("\t") - waived = ok.search(prev) or ok.search(t) - prev = t - if waived or comment.match(t): - continue - for rule, rx, on in (("R1 SHUFFLE-AS-COMPUTE", r1, True), ("R2 SCALAR-IN-SIMD", r2, backend), ("R3 U8-I8-MIX", r3, simd)): - if on and rx.search(t): - hits += 1 - print(f"{f}:{n} {rule} {t.strip()[:140]}", file=sys.stderr) +impl_re = re.compile(r"^\s*impl\b[^{;]*?\b(?:for\s+)?([A-Z][A-Za-z0-9_]*)\s*(?:<[^{]*>)?\s*\{") +fn_re = re.compile(r"\bfn\s+([A-Za-z0-9_]+)") +lines = open(f, encoding="utf-8", errors="replace").read().split("\n") +depth, test_until, pending_test, attr = 0, None, False, None +impl_name, impl_depth, fn_name = "", None, "" +hits = 0 +for i, t in enumerate(lines, 1): + code = t.split("//")[0] + # cfg attributes may span lines: collect until the closing bracket. + if attr is None and re.match(r"^\s*#\[cfg\(", t): + attr = "" + if attr is not None: + attr += code + if "]" in code: + if re.search(r"\btest\b", attr): + pending_test = True + attr = None + if re.match(r"^\s*(pub(\(\w+\))?\s+)?mod\s+\w*tests?\b", t): + pending_test = True + m = impl_re.match(t) + if m: + impl_name, impl_depth = m.group(1), depth + fm = fn_re.search(code) + if fm: + fn_name = fm.group(1) + in_test = test_until is not None + if pending_test and "{" in code: + if test_until is None: + test_until = depth + pending_test = False + in_test = True + if i in want and not in_test and not comment.match(t): + prev = lines[i - 2] if i >= 2 else "" + if not (ok.search(prev) or ok.search(t)): + found = [] + if "R1" not in excluded and r1.search(code) and not shuffle_def.match(fn_name): + found.append("R1 SHUFFLE-AS-COMPUTE") + if "R2" not in excluded and backend and r2.search(code): + found.append("R2 SCALAR-IN-SIMD") + if "R3" not in excluded and simd: + u8_ty, i8_ty = impl_name.startswith("U8"), impl_name.startswith("I8") + to_i8, to_u8 = re.search(r"as\s+i8\b", code), re.search(r"as\s+u8\b", code) + mix = (u8_ty and to_i8) or (i8_ty and to_u8) \ + or (to_i8 and re.search(r"\bu8\b", code)) or (to_u8 and re.search(r"\bi8\b", code)) + if mix: + found.append("R3 U8-I8-MIX") + for rule in found: + hits += 1 + print(f"{f}:{i} {rule} {t.strip()[:140]}", file=sys.stderr) + depth += code.count("{") - code.count("}") + if test_until is not None and depth <= test_until: + test_until = None + if impl_depth is not None and depth <= impl_depth: + impl_name, impl_depth = "", None print(hits) ' "$1" >> "$HITS_FILE" } @@ -140,11 +214,11 @@ print(ti.get("file_path") or ti.get("notebook_path") or "")' 2>/dev/null)" done < <( { git diff --name-only "$rev"; git ls-files --others --exclude-standard; } | sort -u) ;; --file) - for f in "$@"; do f="${f#"$ROOT"/}"; [ -f "$f" ] && whole_file "$f" | strip_tests | check_lines "$f"; done + for f in "$@"; do f="${f#"$ROOT"/}"; [ -f "$f" ] && whole_file "$f" | check_lines "$f"; done ;; --all) - while IFS= read -r f; do whole_file "$f" | strip_tests | check_lines "$f"; done \ - < <(git ls-files 'src/*.rs' 'crates/*.rs' 'examples/*.rs' | while IFS= read -r f; do in_scope "$f" && echo "$f"; done) + while IFS= read -r f; do whole_file "$f" | check_lines "$f"; done \ + < <(git ls-files '*.rs' | while IFS= read -r f; do in_scope "$f" && echo "$f"; done) ;; *) echo "usage: $0 --hook | --diff | --all | --file ..." >&2; exit 64 ;; esac diff --git a/.claude/knowledge/facet-stride-audit-2026-10-27.txt b/.claude/knowledge/facet-stride-audit-2026-10-27.txt index e183b668..c83087b5 100644 --- a/.claude/knowledge/facet-stride-audit-2026-10-27.txt +++ b/.claude/knowledge/facet-stride-audit-2026-10-27.txt @@ -1,58 +1,215 @@ awk: cannot open "crates/burn/src/backend.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/backend.rs' awk: cannot open "crates/burn/src/element.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/element.rs' awk: cannot open "crates/burn/src/lib.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/lib.rs' awk: cannot open "crates/burn/src/ops/adaptive_avgpool.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/adaptive_avgpool.rs' awk: cannot open "crates/burn/src/ops/avgpool.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/avgpool.rs' awk: cannot open "crates/burn/src/ops/base.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/base.rs' awk: cannot open "crates/burn/src/ops/bool_tensor.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/bool_tensor.rs' awk: cannot open "crates/burn/src/ops/conv.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/conv.rs' awk: cannot open "crates/burn/src/ops/deform_conv.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/deform_conv.rs' awk: cannot open "crates/burn/src/ops/grid_sample.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/grid_sample.rs' awk: cannot open "crates/burn/src/ops/int_tensor.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/int_tensor.rs' awk: cannot open "crates/burn/src/ops/interpolate.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/interpolate.rs' awk: cannot open "crates/burn/src/ops/macros.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/macros.rs' awk: cannot open "crates/burn/src/ops/maxpool.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/maxpool.rs' awk: cannot open "crates/burn/src/ops/mod.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/mod.rs' awk: cannot open "crates/burn/src/ops/module.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/module.rs' awk: cannot open "crates/burn/src/ops/padding.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/padding.rs' awk: cannot open "crates/burn/src/ops/qtensor.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/qtensor.rs' awk: cannot open "crates/burn/src/ops/quantization.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/quantization.rs' awk: cannot open "crates/burn/src/ops/simd/avgpool.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/avgpool.rs' awk: cannot open "crates/burn/src/ops/simd/base.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/base.rs' awk: cannot open "crates/burn/src/ops/simd/binary.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/binary.rs' awk: cannot open "crates/burn/src/ops/simd/binary_elemwise.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/binary_elemwise.rs' awk: cannot open "crates/burn/src/ops/simd/cmp.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/cmp.rs' awk: cannot open "crates/burn/src/ops/simd/conv.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/conv.rs' awk: cannot open "crates/burn/src/ops/simd/maxpool.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/maxpool.rs' awk: cannot open "crates/burn/src/ops/simd/mod.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/mod.rs' awk: cannot open "crates/burn/src/ops/simd/unary.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/simd/unary.rs' awk: cannot open "crates/burn/src/ops/transaction.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/ops/transaction.rs' awk: cannot open "crates/burn/src/parallel.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/parallel.rs' awk: cannot open "crates/burn/src/rand.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/rand.rs' awk: cannot open "crates/burn/src/sharing.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/sharing.rs' awk: cannot open "crates/burn/src/storage.rs" (No such file or directory) +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/storage.rs' awk: cannot open "crates/burn/src/tensor.rs" (No such file or directory) -examples/ternlog_popcnt_gap_probe.rs:87 R1 SHUFFLE-AS-COMPUTE let l = _mm512_shuffle_epi8(lut, _mm512_and_si512(t, lo)); -examples/ternlog_popcnt_gap_probe.rs:88 R1 SHUFFLE-AS-COMPUTE let h = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16::<4>(t), lo)); +Traceback (most recent call last): + File "", line 25, in + lines = open(f, encoding="utf-8", errors="replace").read().split("\n") + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'crates/burn/src/tensor.rs' src/bitwise.rs:120 R1 SHUFFLE-AS-COMPUTE let popcnt_lo = lookup.shuffle_bytes(lo); src/bitwise.rs:121 R1 SHUFFLE-AS-COMPUTE let popcnt_hi = lookup.shuffle_bytes(hi); src/bitwise.rs:165 R1 SHUFFLE-AS-COMPUTE let popcnt_lo = lookup.shuffle_bytes(lo); src/bitwise.rs:166 R1 SHUFFLE-AS-COMPUTE let popcnt_hi = lookup.shuffle_bytes(hi); -src/hpc/perm.rs:192 R1 SHUFFLE-AS-COMPUTE let composed = U8x64::from_array(self.idx).permute_bytes(U8x64::from_array(next.idx)); -src/hpc/perm.rs:268 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x64::from_array(self.idx)) -src/hpc/perm.rs:285 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x64::from_array(self.idx)) -src/hpc/perm.rs:307 R1 SHUFFLE-AS-COMPUTE U8x64::from_slice(s).permute_bytes(idx).copy_to_slice(o); -src/simd_amx.rs:508 R3 U8-I8-MIX output[i] = (energy[i] * scale).round().clamp(0.0, 127.0) as i8; src/simd_avx.rs:107 R1 SHUFFLE-AS-COMPUTE let cnt = _mm_add_epi8(_mm_shuffle_epi8(lookup, lo), _mm_shuffle_epi8(lookup, hi)); -src/simd_avx.rs:159 R3 U8-I8-MIX .map(|(&x, &y)| (x as i8 as i64) * (y as i8 as i64)) -src/simd_avx.rs:613 R1 SHUFFLE-AS-COMPUTE Self(halves2!(_mm_shuffle_epi8, self.0, idx.0)) -src/simd_avx.rs:632 R2 SCALAR-IN-SIMD for k in 0..16 { -src/simd_avx.rs:634 R1 SHUFFLE-AS-COMPUTE let hit = _mm_shuffle_epi8(row, lo); -src/simd_avx.rs:635 R3 U8-I8-MIX let here = _mm_cmpeq_epi8(hi, _mm_set1_epi8(k as i8)); src/simd_avx2.rs:375 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_avx2.rs:380 R1 SHUFFLE-AS-COMPUTE let cnt = _mm256_add_epi8(_mm256_shuffle_epi8(lookup, lo), _mm256_shuffle_epi8(lookup, hi)); src/simd_avx2.rs:392 R1 SHUFFLE-AS-COMPUTE let cnt = _mm256_add_epi8(_mm256_shuffle_epi8(lookup, lo), _mm256_shuffle_epi8(lookup, hi)); -src/simd_avx2.rs:455 R3 U8-I8-MIX .map(|(&x, &y)| (x as i8 as i64) * (y as i8 as i64)) src/simd_avx2.rs:576 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_avx2.rs:585 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_avx2.rs:596 R2 SCALAR-IN-SIMD for i in 0..16 { @@ -109,100 +266,44 @@ src/simd_avx2.rs:1651 R2 SCALAR-IN-SIMD for i in 0..64 { src/simd_avx2.rs:1659 R2 SCALAR-IN-SIMD for i in 0..64 { src/simd_avx2.rs:1669 R2 SCALAR-IN-SIMD for lane in 0..4 { src/simd_avx2.rs:1671 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2085 R2 SCALAR-IN-SIMD for c in 0..16 { -src/simd_avx2.rs:2103 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2124 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2425 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:2544 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:2563 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:2650 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2658 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2666 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2669 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2677 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2685 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2693 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:2911 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2919 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2927 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2935 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2946 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:2956 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:3019 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:3036 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx2.rs:3055 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:3063 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:3111 R3 U8-I8-MIX Self(unsafe { _mm256_set1_epi8(v as i8) }) -src/simd_avx2.rs:3327 R1 SHUFFLE-AS-COMPUTE Self(unsafe { _mm256_shuffle_epi8(self.0, idx.0) }) -src/simd_avx2.rs:3348 R2 SCALAR-IN-SIMD for k in 0..16 { -src/simd_avx2.rs:3350 R1 SHUFFLE-AS-COMPUTE let hit = _mm256_shuffle_epi8(_mm256_broadcastsi128_si256(row), lo); -src/simd_avx2.rs:3351 R3 U8-I8-MIX let here = _mm256_cmpeq_epi8(hi, _mm256_set1_epi8(k as i8)); -src/simd_avx2.rs:3369 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3574 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3587 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3601 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3648 R1 SHUFFLE-AS-COMPUTE .permute_bytes(U8x32::from_array(idx)) -src/simd_avx2.rs:3650 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3663 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3674 R1 SHUFFLE-AS-COMPUTE let counts = lut.shuffle_bytes(U8x32::from_array(idx)).to_array(); -src/simd_avx2.rs:3675 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx2.rs:3745 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx2.rs:3782 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx2.rs:3847 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx2.rs:3862 R2 SCALAR-IN-SIMD for lane in [0usize, 31, 32, 63] { -src/simd_avx2.rs:4340 R2 SCALAR-IN-SIMD for i in 0..64 { +src/simd_avx2.rs:2070 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_avx2.rs:2088 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2109 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2410 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2529 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2548 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:2635 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2643 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2651 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2654 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2662 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2670 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2678 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx2.rs:2896 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2904 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2912 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2920 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2931 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:2941 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3004 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3021 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx2.rs:3040 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:3048 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx2.rs:3096 R3 U8-I8-MIX Self(unsafe { _mm256_set1_epi8(v as i8) }) +src/simd_avx2.rs:3326 R2 SCALAR-IN-SIMD for i in 0..32 { src/simd_avx512.rs:529 R3 U8-I8-MIX Self(unsafe { _mm512_set1_epi8(v as i8) }) src/simd_avx512.rs:729 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:762 R1 SHUFFLE-AS-COMPUTE Self(unsafe { _mm512_shuffle_epi8(self.0, idx.0) }) -src/simd_avx512.rs:781 R2 SCALAR-IN-SIMD for k in 0..16 { -src/simd_avx512.rs:783 R1 SHUFFLE-AS-COMPUTE let hit = _mm512_shuffle_epi8(_mm512_broadcast_i32x4(row), lo); -src/simd_avx512.rs:784 R3 U8-I8-MIX let here = _mm512_cmpeq_epi8_mask(hi, _mm512_set1_epi8(k as i8)); -src/simd_avx512.rs:828 R1 SHUFFLE-AS-COMPUTE _mm512_permutexvar_epi8(idx, v) -src/simd_avx512.rs:1177 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx512.rs:1316 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx512.rs:1338 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx512.rs:1759 R2 SCALAR-IN-SIMD for c in 0..16 { -src/simd_avx512.rs:3114 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx512.rs:3115 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; -src/simd_avx512.rs:3174 R3 U8-I8-MIX let clamped = _mm_min_epu8(_mm_abs_epi8(v), _mm_set1_epi8(0x7f_u8 as i8)); -src/simd_avx512.rs:3474 R2 SCALAR-IN-SIMD for k in 0..8 { -src/simd_avx512.rs:3522 R2 SCALAR-IN-SIMD for k in 0..8 { -src/simd_avx512.rs:3568 R3 U8-I8-MIX core::arch::x86_64::_mm256_min_epu8(raw_abs, core::arch::x86_64::_mm256_set1_epi8(0x7f_u8 as i8)); -src/simd_avx512.rs:3576 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:3630 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx512.rs:3640 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx512.rs:3677 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_avx512.rs:5108 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5114 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5117 R1 SHUFFLE-AS-COMPUTE let perm = v.permute_bytes(U8x64::from_slice(&idx)); -src/simd_avx512.rs:5126 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5132 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5135 R1 SHUFFLE-AS-COMPUTE let perm = v.permute_bytes(U8x64::from_slice(&idx)); -src/simd_avx512.rs:5138 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5192 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5198 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5212 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5284 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5285 R3 U8-I8-MIX a[i] = (i + 1) as i8; -src/simd_avx512.rs:5286 R3 U8-I8-MIX b[i] = (64 - i) as i8; -src/simd_avx512.rs:5293 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5314 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5315 R3 U8-I8-MIX a[i] = (i as i32 - 32) as i8; -src/simd_avx512.rs:5321 R2 SCALAR-IN-SIMD for i in 0..64 { -src/simd_avx512.rs:5333 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5334 R3 U8-I8-MIX a[i] = (i + 1) as i8; -src/simd_avx512.rs:5335 R3 U8-I8-MIX b[i] = (32 - i) as i8; -src/simd_avx512.rs:5341 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5349 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5350 R3 U8-I8-MIX a[i] = (i as i32 - 16) as i8; -src/simd_avx512.rs:5356 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5383 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5390 R2 SCALAR-IN-SIMD for i in 0..32 { -src/simd_avx512.rs:5414 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx512.rs:5421 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_avx512.rs:5452 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_fixedpoint.rs:275 R3 U8-I8-MIX out[base + k] = r[k] as i8; -src/simd_fixedpoint.rs:282 R3 U8-I8-MIX out[i] = v.max(act_min).min(act_max) as i8; +src/simd_avx512.rs:1151 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:1290 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:1312 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:1733 R2 SCALAR-IN-SIMD for c in 0..16 { +src/simd_avx512.rs:3088 R2 SCALAR-IN-SIMD for i in 0..16 { +src/simd_avx512.rs:3448 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_avx512.rs:3496 R2 SCALAR-IN-SIMD for k in 0..8 { +src/simd_avx512.rs:3550 R2 SCALAR-IN-SIMD for i in 0..32 { +src/simd_avx512.rs:3604 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:3614 R2 SCALAR-IN-SIMD for i in 0..8 { +src/simd_avx512.rs:3651 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:639 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_neon.rs:651 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_neon.rs:671 R2 SCALAR-IN-SIMD for i in 0..16 { @@ -220,15 +321,12 @@ src/simd_neon.rs:1069 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:1081 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:1090 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:1208 R2 SCALAR-IN-SIMD for i in 0..8 { -src/simd_neon.rs:1235 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_neon.rs:1267 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:1375 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_neon.rs:1477 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_neon.rs:1857 R2 SCALAR-IN-SIMD for c in 0..16 { src/simd_neon.rs:1946 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_neon.rs:1985 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_neon.rs:2511 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_neon.rs:2512 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; src/simd_neon.rs:2567 R2 SCALAR-IN-SIMD for i in 0..32 { src/simd_neon.rs:2603 R2 SCALAR-IN-SIMD for k in 0..8 { src/simd_neon.rs:2670 R2 SCALAR-IN-SIMD for k in 0..8 { @@ -255,18 +353,11 @@ src/simd_nightly/exotic_methods.rs:346 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_nightly/f16_types.rs:175 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_nightly/f16_types.rs:185 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_nightly/f32_types.rs:103 R2 SCALAR-IN-SIMD for (o, &i) in out.iter_mut().zip(idx.iter()) { -src/simd_nightly/u8_types.rs:390 R1 SHUFFLE-AS-COMPUTE acc = here.select(rows.swizzle_dyn(low), acc); src/simd_nightly/u_word_types.rs:42 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_nightly/u_word_types.rs:59 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_nightly/u_word_types.rs:684 R2 SCALAR-IN-SIMD for c in 0..16 { src/simd_nightly/w1a_types.rs:531 R2 SCALAR-IN-SIMD for k in 0..8 { src/simd_nightly/w1a_types.rs:663 R2 SCALAR-IN-SIMD for k in 0..8 { -src/simd_scalar.rs:810 R3 U8-I8-MIX v.clamp(-128, 127) as i8 -src/simd_scalar.rs:1885 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; -src/simd_scalar.rs:1937 R3 U8-I8-MIX let mut o = [0 as i8; 16]; -src/simd_scalar.rs:1954 R3 U8-I8-MIX let mut o = [0 as i8; 16]; -src/simd_scalar.rs:1971 R3 U8-I8-MIX let mut o = [0 as i8; 16]; -src/simd_scalar.rs:1988 R3 U8-I8-MIX let mut o = [0 as i8; 16]; src/simd_wasm.rs:302 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_wasm.rs:311 R2 SCALAR-IN-SIMD for i in 0..16 { src/simd_wasm.rs:320 R2 SCALAR-IN-SIMD for i in 0..16 { @@ -275,11 +366,8 @@ src/simd_wasm.rs:678 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_wasm.rs:687 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_wasm.rs:798 R2 SCALAR-IN-SIMD for i in 0..8 { src/simd_wasm.rs:922 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_wasm.rs:923 R3 U8-I8-MIX let nibble = ((packed >> (4 * i)) & 0xf) as i8; src/simd_wasm.rs:1423 R2 SCALAR-IN-SIMD for c in 0..16 { src/simd_wasm.rs:1506 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_wasm.rs:1832 R2 SCALAR-IN-SIMD for i in 0..16 { -src/simd_wasm.rs:1922 R3 U8-I8-MIX (self.next_u64() & 0xFF) as u8 as i8 ███ FACET STRIDE VIOLATION ███ '{0}..{15}' -f $128.split() A 128-bit value is 16 u8 positions; one mov reads them all. Shuffles only @@ -288,4 +376,4 @@ src/simd_wasm.rs:1922 R3 U8-I8-MIX (self.next_u64() & 0xFF) as u8 as i8 sign: do not mix u8 and i8. Genuine exception: `// facet-stride: ok ` on or above the line. Rule text: CLAUDE.md § FACET STRIDE. - 248 violation(s). + 166 violation(s). diff --git a/.claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt b/.claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt new file mode 100644 index 00000000..07c26dc4 --- /dev/null +++ b/.claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt @@ -0,0 +1,19 @@ +crates/holograph/src/hamming.rs:593 R1 SHUFFLE-AS-COMPUTE let popcnt_lo = _mm256_shuffle_epi8(lookup, lo); +crates/holograph/src/hamming.rs:594 R1 SHUFFLE-AS-COMPUTE let popcnt_hi = _mm256_shuffle_epi8(lookup, hi); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:291 R1 SHUFFLE-AS-COMPUTE let pop_lo = _mm256_shuffle_epi8(lut, lo_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:292 R1 SHUFFLE-AS-COMPUTE let pop_hi = _mm256_shuffle_epi8(lut, hi_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:336 R1 SHUFFLE-AS-COMPUTE let pop_lo = _mm256_shuffle_epi8(lut, lo_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:337 R1 SHUFFLE-AS-COMPUTE let pop_hi = _mm256_shuffle_epi8(lut, hi_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:386 R1 SHUFFLE-AS-COMPUTE let pop_lo = _mm512_shuffle_epi8(lut, lo_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:387 R1 SHUFFLE-AS-COMPUTE let pop_hi = _mm512_shuffle_epi8(lut, hi_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:428 R1 SHUFFLE-AS-COMPUTE let pop_lo = _mm512_shuffle_epi8(lut, lo_nibbles); +crates/lance-graph/src/graph/blasgraph/ndarray_bridge.rs:429 R1 SHUFFLE-AS-COMPUTE let pop_hi = _mm512_shuffle_epi8(lut, hi_nibbles); + + ███ FACET STRIDE VIOLATION ███ '{0}..{15}' -f $128.split() + A 128-bit value is 16 u8 positions; one mov reads them all. Shuffles only + MOVE bytes; never assemble a result from them. Scalar loops live only in + simd_scalar.rs (on AVX2 use the 2 x U8x32 / 128-bit-half split). u8 has no + sign: do not mix u8 and i8. + Genuine exception: `// facet-stride: ok ` on or above the line. + Rule text: CLAUDE.md § FACET STRIDE. + 10 violation(s). diff --git a/CLAUDE.md b/CLAUDE.md index acbe6aa6..ae2b160b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,11 +51,20 @@ bash .claude/hooks/facet-stride-guard.sh --diff origin/master # lines a branch bash .claude/hooks/facet-stride-guard.sh --all # whole-tree audit ``` -The existing tree is NOT clean: the 2026-10-27 audit found 248 hits outside -tests (25 R1, 195 R2, 28 R3), listed in -`.claude/knowledge/facet-stride-audit-2026-10-27.txt`. They are a burn-down -list, not a precedent: existing code that violates the shape is never a -reason to write more of it. +**Exclusions** (`.claude/hooks/facet-stride-exclude.txt`, `path | rules | +reason`, a reason is required) are only for code whose byte shuffle IS the +defined operation: cipher rounds (`vendor/chacha20`, rotate by 8/16 bits), +permutation maps (`src/hpc/perm.rs`), and a benchmark that measures the +anti-pattern on purpose. Never for a lookup or a fold. `--root ` audits +another checkout (lance-graph: 10 hits, all nibble-table popcount, in +`.claude/knowledge/facet-stride-audit-lance-graph-2026-10-27.txt`). + +The existing tree is NOT clean: the 2026-10-27 audit found 166 hits outside +tests (7 R1, all nibble-table popcount; 157 R2, 81 of them the array-backed +AVX2 types; 2 R3, the u8 `splat`s that cast to the `i8` intrinsic +parameter), listed in `.claude/knowledge/facet-stride-audit-2026-10-27.txt`. +They are a burn-down list, not a precedent: existing code that violates the +shape is never a reason to write more of it. ## Quick Context - **What**: High-performance linear algebra with pluggable BLAS backends (Native SIMD, MKL, OpenBLAS) From f3a7bf7908cca7c1f944efb7447c0aa5695031d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:45:16 +0000 Subject: [PATCH 10/11] simd_stride: lut_u8, the reusable FACET STRIDE table lookup out[i] = table[values[i]], reading each 16-byte facet's positions where they sit; odd_lut_u8 now uses it for the magnitude lookup. Re-exported through ndarray::simd with FACET_BYTES. Unit tests at every length around the facet, plus parity group 0x15 (21 groups). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- crates/simd-masking-parity/src/lib.rs | 32 ++++++++- src/lib.rs | 6 ++ src/simd.rs | 2 + src/simd_masking_ops.rs | 17 +---- src/simd_stride.rs | 93 +++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 src/simd_stride.rs diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index ca51bb82..50e7ef1f 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -50,14 +50,14 @@ use ndarray::simd::{ mask_shift_morton, mask_ternlog, mask_ternlog_any, mask_ternlog_assign, mask_ternlog_popcount, mask_xor, mask_xor_assign, masked_group_sum_i32, masked_group_sum_i32_via, masked_key_run_count_u32, masked_max_i32, masked_min_i32, masked_strided_group_sum, masked_sum_i32, masked_sum_wrapping_add_i32, ne_i32_to_mask, - ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, odd_lut_u8, + ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, odd_lut_u8, lut_u8, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, ternlog, F32x16, F64x8, I16x32, I32x16, I8x16, KeyRunCarry, MortonDir, U32x16, U64x8, U8x16, }; /// Number of check groups [`run`] executes (for the log line only). -pub const CHECKS: usize = 20; +pub const CHECKS: usize = 21; /// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it. #[no_mangle] @@ -72,7 +72,7 @@ pub fn run() -> u32 { check_mask_algebra, check_care_match, check_masked_reductions, check_blend, check_morton_shift, check_predicates_under, check_set_range, check_unsigned_compare_to_mask, check_gather_scatter_group, check_f64x8_compare, check_i8x16_u8x16_lanes, check_care_match16, check_narrow_ladder, - check_mul_add_single_rounding, check_fixedpoint_requant, check_odd_lut_u8, + check_mul_add_single_rounding, check_fixedpoint_requant, check_odd_lut_u8, check_lut_u8, ]; for g in groups { if let Err(code) = g() { @@ -2688,6 +2688,32 @@ fn check_odd_lut_u8() -> Result<(), u32> { Ok(()) } +/// Group 0x15: `lut_u8`, the FACET STRIDE table lookup, against a +/// per-byte read of the table at every length around the 16-byte facet. +fn check_lut_u8() -> Result<(), u32> { + let mut rng = SplitMix64(0x1500_0000_0001); + let mut table = [0u8; 256]; + for t in table.iter_mut() { + *t = rng.next() as u8; + } + const LUT_LENS: [usize; 12] = [0, 1, 15, 16, 17, 31, 32, 33, 64, 65, 255, 300]; + for (case, &n) in LUT_LENS.iter().enumerate() { + let case = case as u32; + let values: Vec = (0..n).map(|_| rng.next() as u8).collect(); + let mut out = vec![0xAAu8; n + 3]; + lut_u8(&values, &table, &mut out); + for i in 0..n { + if out[i] != table[values[i] as usize] { + return Err(0x1500 | case); + } + } + if out[n..].iter().any(|&b| b != 0xAA) { + return Err(0x1510 | case); + } + } + Ok(()) +} + #[cfg(test)] mod oracle_tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 4bb6ae5c..40f5e6e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -474,6 +474,12 @@ pub mod simd_int_ops; #[cfg(feature = "std")] pub mod simd_masking_ops; +/// FACET STRIDE patterns: byte work that reads the sixteen u8 positions of +/// a 128-bit value where they sit (`lut_u8`). Re-exported through +/// `ndarray::simd`. +#[cfg(feature = "std")] +pub mod simd_stride; + /// Slice-level elementwise ops (f32/f64) built on the polyfill SIMD types. /// `add_f32`, `mul_f32`, `add_f32_inplace`, `scale_f32`, etc. /// Re-exported flat through `ndarray::simd::add_f32`. diff --git a/src/simd.rs b/src/simd.rs index a5885d68..9d109074 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -985,6 +985,8 @@ pub use crate::simd_masking_ops::{ BOUNDED_TILE_ROWS, SYM_EMPTY_I64, }; +#[cfg(feature = "std")] +pub use crate::simd_stride::{lut_u8, FACET_BYTES}; // The popcount that closes the loop on the masks above: `mask_count` in ABI // terms. Already public at `ndarray::bitwise::popcount_batch_u64`; re-exported // here so a mask producer and its reducer share one import path (the sibling diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index 197fc0ba..fb6c076f 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3871,20 +3871,9 @@ pub fn odd_lut_u8( assert!(signs.len() >= words, "odd_lut_u8: signs holds {} words, need {words}", signs.len()); assert!(out_signs.len() >= words, "odd_lut_u8: out_signs holds {} words, need {words}", out_signs.len()); - // FACET STRIDE, '{0}..{15}' -f $128.split(): sixteen magnitudes are one - // 128-bit value, read where they sit; each position addresses the table - // directly. No byte is shuffled and no result is assembled from masks. - let lookup = |positions: &[u8; 16]| -> [u8; 16] { positions.map(|m| table[usize::from(m)]) }; - let fill = |chunk: &[u8; 64], out: &mut [u8; 64]| { - for (positions, o) in chunk - .as_chunks::<16>() - .0 - .iter() - .zip(out.as_chunks_mut::<16>().0) - { - *o = lookup(positions); - } - }; + // FACET STRIDE: the magnitude lookup reads positions where they sit + // (`simd_stride::lut_u8`); no byte is shuffled. + let fill = |chunk: &[u8; 64], out: &mut [u8; 64]| crate::simd_stride::lut_u8(chunk, table, out); let zero = U8x64::splat(0); let (in_chunks, tail) = magnitudes.as_chunks::<64>(); let (out_chunks, out_tail) = out_magnitudes[..n].as_chunks_mut::<64>(); diff --git a/src/simd_stride.rs b/src/simd_stride.rs new file mode 100644 index 00000000..f3914cce --- /dev/null +++ b/src/simd_stride.rs @@ -0,0 +1,93 @@ +//! FACET STRIDE patterns: byte work that reads positions where they sit. +//! +//! `'{0}..{15}' -f $128.split()`: a 128-bit value IS sixteen u8 positions, +//! and the load that brought it in already reads all sixteen. Nothing here +//! moves a byte to read it: no shuffle, no permute, no table lookup +//! assembled from shuffles and masks (CLAUDE.md, FACET STRIDE). +//! +//! This module holds the reusable form of that shape so callers do not +//! rebuild it. Like [`crate::simd_masking_ops`] it names no ISA and owns no +//! backend; it is slice ergonomics over the position view. + +/// Bytes per facet: one 128-bit value read as sixteen u8 positions. +pub const FACET_BYTES: usize = 16; + +/// Maps every byte through a 256-entry table: `out[i] = table[values[i]]`. +/// +/// Each run of [`FACET_BYTES`] values is one 128-bit facet; every position +/// addresses the table directly, so the lookup costs one table read per +/// byte and no byte is moved to compute it. This is the shape to reach for +/// instead of a byte shuffle used as a lookup. +/// +/// `out` may be longer than `values`; bytes past `values.len()` are left +/// untouched. +/// +/// # Panics +/// +/// When `out` is shorter than `values`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::lut_u8; +/// +/// let mut table = [0u8; 256]; +/// for (i, t) in table.iter_mut().enumerate() { +/// *t = (255 - i) as u8; +/// } +/// let mut out = [0u8; 3]; +/// lut_u8(&[0, 1, 255], &table, &mut out); +/// assert_eq!(out, [255, 254, 0]); +/// ``` +pub fn lut_u8(values: &[u8], table: &[u8; 256], out: &mut [u8]) { + let n = values.len(); + assert!(out.len() >= n, "lut_u8: out holds {} bytes, need {n}", out.len()); + let (facets, tail) = values.as_chunks::(); + let (out_facets, out_tail) = out[..n].as_chunks_mut::(); + for (positions, o) in facets.iter().zip(out_facets) { + *o = positions.map(|v| table[usize::from(v)]); + } + for (&v, o) in tail.iter().zip(out_tail) { + *o = table[usize::from(v)]; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn table() -> [u8; 256] { + // A non-monotone, non-affine table, so a position read at the wrong + // index cannot land on the right value by accident. + core::array::from_fn(|i| (i.wrapping_mul(73) ^ 0x5A) as u8) + } + + #[test] + fn lut_u8_matches_a_per_byte_lookup_at_every_length() { + let t = table(); + for n in [0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 255, 256, 300] { + let values: Vec = (0..n).map(|i| (i * 37 + 11) as u8).collect(); + let mut out = vec![0xC3u8; n + 5]; + lut_u8(&values, &t, &mut out); + for i in 0..n { + assert_eq!(out[i], t[usize::from(values[i])], "n={n} i={i}"); + } + assert!(out[n..].iter().all(|&b| b == 0xC3), "n={n}: wrote past values.len()"); + } + } + + #[test] + fn lut_u8_reads_every_table_entry() { + let t = table(); + let values: Vec = (0..=255).collect(); + let mut out = vec![0u8; 256]; + lut_u8(&values, &t, &mut out); + assert_eq!(out[..], t[..]); + } + + #[test] + #[should_panic(expected = "lut_u8: out holds")] + fn lut_u8_rejects_a_short_output() { + lut_u8(&[1, 2, 3], &table(), &mut [0u8; 2]); + } +} From e47d456023cd8560ef857159b794b5edb07b0f55 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:54:19 +0000 Subject: [PATCH 11/11] simd_stride: lut9, the SM9 odd-function lookup as a reusable item Moves the #382 pattern out of simd_masking_ops into simd_stride beside lut_u8 and names it lut9: a 256-entry magnitude table over a sign-magnitude 9-bit lane (u8 magnitude + sign plane), sign cleared where a nonzero input rounds to zero, input -0 kept. The magnitude lookup is lut_u8. Behaviour unchanged; odd_lut_u8 is renamed, its tests, parity group 0x14 and bench (examples/lut9_bench.rs) follow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMPh2Seh2WLccCksAdY --- Cargo.toml | 2 +- crates/simd-masking-parity/src/lib.rs | 10 +- examples/{odd_lut_bench.rs => lut9_bench.rs} | 16 +- src/simd.rs | 3 +- src/simd_masking_ops.rs | 206 +----------------- src/simd_stride.rs | 207 ++++++++++++++++++- 6 files changed, 220 insertions(+), 224 deletions(-) rename examples/{odd_lut_bench.rs => lut9_bench.rs} (86%) diff --git a/Cargo.toml b/Cargo.toml index 1fad814c..1e5f676d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -253,7 +253,7 @@ name = "campq_cascade_probe" required-features = ["std"] [[example]] -name = "odd_lut_bench" +name = "lut9_bench" required-features = ["std"] [dependencies] diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index 50e7ef1f..805285a7 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -50,7 +50,7 @@ use ndarray::simd::{ mask_shift_morton, mask_ternlog, mask_ternlog_any, mask_ternlog_assign, mask_ternlog_popcount, mask_xor, mask_xor_assign, masked_group_sum_i32, masked_group_sum_i32_via, masked_key_run_count_u32, masked_max_i32, masked_min_i32, masked_strided_group_sum, masked_sum_i32, masked_sum_wrapping_add_i32, ne_i32_to_mask, - ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, odd_lut_u8, lut_u8, + ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, lut9, lut_u8, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, ternlog, F32x16, F64x8, I16x32, I32x16, I8x16, KeyRunCarry, MortonDir, U32x16, U64x8, U8x16, @@ -72,7 +72,7 @@ pub fn run() -> u32 { check_mask_algebra, check_care_match, check_masked_reductions, check_blend, check_morton_shift, check_predicates_under, check_set_range, check_unsigned_compare_to_mask, check_gather_scatter_group, check_f64x8_compare, check_i8x16_u8x16_lanes, check_care_match16, check_narrow_ladder, - check_mul_add_single_rounding, check_fixedpoint_requant, check_odd_lut_u8, check_lut_u8, + check_mul_add_single_rounding, check_fixedpoint_requant, check_lut9, check_lut_u8, ]; for g in groups { if let Err(code) = g() { @@ -2637,11 +2637,11 @@ fn oracle_fma_f64(a: f64, b: f64, c: f64) -> f64 { } } -// ── 0x14xx: odd_lut_u8 — a 256-entry magnitude table over a sign-magnitude +// ── 0x14xx: lut9 — a 256-entry magnitude table over a sign-magnitude // lane. The reference indexes `table` with a plain array lookup and builds // the output sign plane bit by bit; it never calls the primitive. -fn check_odd_lut_u8() -> Result<(), u32> { +fn check_lut9() -> Result<(), u32> { let mut rng = SplitMix64(0x1400_0000_0001); // A random table with some zero entries away from index 0, so a sign bit // must be cleared at nonzero magnitudes too, and every quarter differs. @@ -2668,7 +2668,7 @@ fn check_odd_lut_u8() -> Result<(), u32> { } let mut out = vec![0xAAu8; n + 3]; let mut out_signs = vec![u64::MAX; words + 1]; - odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); + lut9(&mags, &signs, &table, &mut out, &mut out_signs); for i in 0..n { if out[i] != table[mags[i] as usize] { return Err(0x1400 | (pass << 7) | case); diff --git a/examples/odd_lut_bench.rs b/examples/lut9_bench.rs similarity index 86% rename from examples/odd_lut_bench.rs rename to examples/lut9_bench.rs index 8e95a9ff..3f040fc0 100644 --- a/examples/odd_lut_bench.rs +++ b/examples/lut9_bench.rs @@ -1,13 +1,13 @@ -//! Throughput of `odd_lut_u8` against plain table loops. +//! Throughput of `lut9` against plain table loops. //! //! Three paths over the same 1 Mi random magnitudes and sign plane: //! //! * `i8 table`: the i8 `Sym` form, one 255-entry table indexed by the //! signed code (`table[c + 127]`), no sign plane; -//! * `u8 table + sign`: the scalar form of what `odd_lut_u8` computes, a +//! * `u8 table + sign`: the scalar form of what `lut9` computes, a //! 256-entry lookup plus the sign plane cleared where a nonzero input //! gives 0; -//! * `odd_lut_u8`: the SIMD primitive. +//! * `lut9`: the SIMD primitive. //! //! The scalar and SIMD sign-magnitude outputs are compared before timing. //! Reports the best of several runs in ns per element and the CPU features @@ -16,7 +16,7 @@ use std::hint::black_box; use std::time::Instant; -use ndarray::simd::odd_lut_u8; +use ndarray::simd::lut9; const N: usize = 1 << 20; const RUNS: usize = 30; @@ -58,7 +58,7 @@ fn main() { // Correctness first: the SIMD output must equal the scalar form. let mut out_simd = vec![0u8; N]; let mut signs_simd = vec![0u64; N / 64]; - odd_lut_u8(&mags, &signs, &table, &mut out_simd, &mut signs_simd); + lut9(&mags, &signs, &table, &mut out_simd, &mut signs_simd); let mut out_scalar = vec![0u8; N]; let mut signs_scalar = vec![0u64; N / 64]; let scalar = |out: &mut [u8], out_signs: &mut [u64]| { @@ -89,12 +89,12 @@ fn main() { black_box((&out_scalar, &signs_scalar)); }); let t_simd = best(|| { - odd_lut_u8(black_box(&mags), black_box(&signs), &table, &mut out_simd, &mut signs_simd); + lut9(black_box(&mags), black_box(&signs), &table, &mut out_simd, &mut signs_simd); black_box((&out_simd, &signs_simd)); }); println!( - "odd_lut_bench: avx512f={} avx512vbmi={} avx2={} neon={} n={N}", + "lut9_bench: avx512f={} avx512vbmi={} avx2={} neon={} n={N}", cfg!(target_feature = "avx512f"), cfg!(target_feature = "avx512vbmi"), cfg!(target_feature = "avx2"), @@ -102,5 +102,5 @@ fn main() { ); println!(" i8 table (no sign plane) {t_i8:6.3} ns/elem"); println!(" u8 table + sign, scalar {t_scalar:6.3} ns/elem"); - println!(" odd_lut_u8 {t_simd:6.3} ns/elem ({:.1}x scalar)", t_scalar / t_simd); + println!(" lut9 {t_simd:6.3} ns/elem ({:.1}x scalar)", t_scalar / t_simd); } diff --git a/src/simd.rs b/src/simd.rs index 9d109074..50a26429 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -968,7 +968,6 @@ pub use crate::simd_masking_ops::{ ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, - odd_lut_u8, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, @@ -986,7 +985,7 @@ pub use crate::simd_masking_ops::{ SYM_EMPTY_I64, }; #[cfg(feature = "std")] -pub use crate::simd_stride::{lut_u8, FACET_BYTES}; +pub use crate::simd_stride::{lut9, lut_u8, FACET_BYTES}; // The popcount that closes the loop on the masks above: `mask_count` in ABI // terms. Already public at `ndarray::bitwise::popcount_batch_u64`; re-exported // here so a mask producer and its reducer share one import path (the sibling diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index fb6c076f..4418c039 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -3787,120 +3787,6 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { clear_mask_tail(out_words, values.len()); } -// ──────────────────────────────────────────────────────────────────────── -// Odd-function table over a sign-magnitude lane. -// -// A sign-magnitude 9-bit lane stores each value as a `u8` magnitude plus one -// bit in a `u64` sign plane per 64 rows; whether the row holds a value at all -// is a separate presence plane, and the writer stores `+0` (magnitude 0, sign -// clear) for zero and for missing rows. For an odd function -// (`f(-x) = -f(x)`, e.g. tanh, sin, x³, sign-preserving clamps) one table -// over the 256 magnitudes covers both signs: the magnitude is looked up and -// the sign plane passes through. The presence plane is not an input — a -// missing row is magnitude 0, `f(0) = 0`, so it stays `+0`. -// ──────────────────────────────────────────────────────────────────────── - -/// Applies an odd function, given as a 256-entry magnitude table, to a -/// sign-magnitude lane. -/// -/// For every row `i < magnitudes.len()`: -/// -/// * `out_magnitudes[i] = table[magnitudes[i]]`; -/// * the output sign bit is the input sign bit, cleared where a nonzero -/// input gives a zero result. A negative value whose result rounds to -/// zero is stored as `+0`, never `-0`. -/// * an input `-0` (magnitude 0, sign set) comes out as `-0`. A writer -/// that only stores `+0` never produces it, so under the plain numeric -/// reading this changes nothing. A class whose reading gives `-0` a -/// meaning of its own, such as "observed, value NaN" beside a clear -/// presence bit for "not observed", keeps that meaning through the -/// function, the way IEEE arithmetic carries NaN through. Clearing it -/// would turn that reading's NaN into the value 0. -/// -/// Rows past `magnitudes.len()` are not written in `out_magnitudes`. In -/// `out_signs` the bits past `magnitudes.len()` and any surplus words are -/// written as `0`. The presence plane is not read or written: because -/// `table[0]` must be `0`, a missing row (stored as `+0`) stays `+0`. -/// -/// Facet stride: every 16 magnitudes are one 128-bit value whose positions -/// `{0}..{15}` are read where they sit, and each position addresses the table -/// directly. No byte is shuffled. The sign plane is 64 rows per -/// [`crate::simd::U8x64::cmpeq_mask`]. The final partial chunk is padded with -/// magnitude 0 and goes through the same path, with the padding lanes masked -/// off the sign plane. -/// -/// # Panics -/// -/// * if `table[0] != 0`, since an odd function maps 0 to 0; -/// * if `out_magnitudes.len() < magnitudes.len()`; -/// * if `signs.len()` or `out_signs.len()` is less than -/// `magnitudes.len().div_ceil(64)`. -/// -/// # Examples -/// -/// ``` -/// use ndarray::simd::odd_lut_u8; -/// -/// // f(m) = m / 2, an odd function on the signed value. -/// let mut table = [0u8; 256]; -/// for (m, t) in table.iter_mut().enumerate() { -/// *t = (m / 2) as u8; -/// } -/// // Values: +10, -10, -1, +0 -/// let magnitudes = [10u8, 10, 1, 0]; -/// let signs = [0b0110u64]; -/// let mut out = [0u8; 4]; -/// let mut out_signs = [0u64; 1]; -/// odd_lut_u8(&magnitudes, &signs, &table, &mut out, &mut out_signs); -/// assert_eq!(out, [5, 5, 0, 0]); -/// // -1 / 2 rounds to zero and is stored as +0: only row 1 keeps its sign. -/// assert_eq!(out_signs[0], 0b0010); -/// ``` -pub fn odd_lut_u8( - magnitudes: &[u8], signs: &[u64], table: &[u8; 256], out_magnitudes: &mut [u8], out_signs: &mut [u64], -) { - use crate::simd::U8x64; - let n = magnitudes.len(); - let words = n.div_ceil(64); - assert_eq!(table[0], 0, "odd_lut_u8: an odd function maps 0 to 0, so table[0] must be 0"); - assert!( - out_magnitudes.len() >= n, - "odd_lut_u8: out_magnitudes holds {} rows, need {n}", - out_magnitudes.len() - ); - assert!(signs.len() >= words, "odd_lut_u8: signs holds {} words, need {words}", signs.len()); - assert!(out_signs.len() >= words, "odd_lut_u8: out_signs holds {} words, need {words}", out_signs.len()); - - // FACET STRIDE: the magnitude lookup reads positions where they sit - // (`simd_stride::lut_u8`); no byte is shuffled. - let fill = |chunk: &[u8; 64], out: &mut [u8; 64]| crate::simd_stride::lut_u8(chunk, table, out); - let zero = U8x64::splat(0); - let (in_chunks, tail) = magnitudes.as_chunks::<64>(); - let (out_chunks, out_tail) = out_magnitudes[..n].as_chunks_mut::<64>(); - for (w, (chunk, out)) in in_chunks.iter().zip(out_chunks).enumerate() { - fill(chunk, out); - let (m, r) = (U8x64::from_array(*chunk), U8x64::from_array(*out)); - // Keep the sign where the result is nonzero or the input is `-0`. - out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)); - } - let mut w = in_chunks.len(); - if !tail.is_empty() { - let mut padded = [0u8; 64]; - padded[..tail.len()].copy_from_slice(tail); - let mut looked_up = [0u8; 64]; - fill(&padded, &mut looked_up); - out_tail.copy_from_slice(&looked_up[..tail.len()]); - let (m, r) = (U8x64::from_array(padded), U8x64::from_array(looked_up)); - // Padding lanes look like input `-0` (magnitude 0), so the keep-`-0` - // rule would pass their junk sign bits; mask them off. - out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)) & live64(tail.len()); - w += 1; - } - for s in &mut out_signs[w..] { - *s = 0; - } -} - // ──────────────────────────────────────────────────────────────────────── // Ordered 64-bit predicates — T1 gap G2 of the DuckDB→V3 translation matrix. // @@ -4836,7 +4722,7 @@ fn live8(live: usize) -> u8 { /// `live`-lane validity mask for a 64-lane group (one `u8x64` register is one /// whole mask word): all ones for a full group, the low `live` bits otherwise. #[inline(always)] -fn live64(live: usize) -> u64 { +pub(crate) fn live64(live: usize) -> u64 { if live == 64 { u64::MAX } else { @@ -5643,96 +5529,6 @@ pub fn mask_shift_morton(src: &[u64], dir: MortonDir, dst: &mut [u64]) { mod tests { use super::*; - // ── odd_lut_u8 ────────────────────────────────────────────────────────── - - fn odd_lut_reference(mags: &[u8], signs: &[u64], table: &[u8; 256]) -> (Vec, Vec) { - let out: Vec = mags.iter().map(|&m| table[m as usize]).collect(); - let mut out_signs = vec![0u64; mags.len().div_ceil(64)]; - for (i, &o) in out.iter().enumerate() { - let neg = (signs[i / 64] >> (i % 64)) & 1 == 1; - if neg && (o != 0 || mags[i] == 0) { - out_signs[i / 64] |= 1 << (i % 64); - } - } - (out, out_signs) - } - - fn odd_lut_xorshift(s: &mut u64) -> u64 { - *s ^= *s << 13; - *s ^= *s >> 7; - *s ^= *s << 17; - *s - } - - /// FAILS IF: a quarter of the table is picked by the wrong magnitude bits, - /// a sign bit survives on a zero result, the final partial chunk writes - /// rows past the end, or bits/words past the last row are left set. - #[test] - fn odd_lut_u8_matches_the_table_at_every_lane_and_length() { - let mut s = 0x9E37_79B9_7F4A_7C15u64; - // Random table with table[0] = 0 and some zero entries elsewhere, so - // the sign-clearing rule is exercised away from magnitude 0 as well. - let mut table = [0u8; 256]; - for t in table.iter_mut().skip(1) { - let r = odd_lut_xorshift(&mut s) as u8; - *t = if r.is_multiple_of(7) { 0 } else { r }; - } - for &n in &[0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300, 1024 + 17] { - for shift in [0usize, 1, 37, 101] { - // Every magnitude appears at every lane position across shifts. - let mags: Vec = (0..n).map(|i| ((i * 37 + shift) % 256) as u8).collect(); - let words = n.div_ceil(64); - let mut signs: Vec = (0..words).map(|_| odd_lut_xorshift(&mut s)).collect(); - // Junk sign bits past the last row must not leak into the output. - if let Some(last) = signs.last_mut() { - *last |= !live64(n - (words - 1) * 64); - } - let (want, want_signs) = odd_lut_reference(&mags, &signs, &table); - let mut out = vec![0xAAu8; n + 5]; - let mut out_signs = vec![u64::MAX; words + 2]; - odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); - assert_eq!(&out[..n], &want[..], "n={n} shift={shift}"); - assert!(out[n..].iter().all(|&b| b == 0xAA), "rows past the end written, n={n}"); - assert_eq!(&out_signs[..words], &want_signs[..], "n={n} shift={shift}"); - assert!(out_signs[words..].iter().all(|&w| w == 0), "surplus words, n={n}"); - } - } - } - - /// FAILS IF: a negative value whose result rounds to zero keeps its sign - /// (a stored `-0`), or an input `-0` loses its sign. Silence half: a - /// nonzero negative result keeps its sign, and `+0` stays `+0`. - #[test] - fn odd_lut_u8_rounds_to_plus_zero_and_keeps_an_input_minus_zero() { - let mut table = [0u8; 256]; - for (m, t) in table.iter_mut().enumerate() { - *t = (m / 4) as u8; - } - // -0, -3 (rounds to 0), -4 (-> -1), +3, +200, +0 - let mags = [0u8, 3, 4, 3, 200, 0]; - let signs = [0b000111u64]; - let mut out = [0u8; 6]; - let mut out_signs = [u64::MAX; 1]; - odd_lut_u8(&mags, &signs, &table, &mut out, &mut out_signs); - assert_eq!(out, [0, 0, 1, 0, 50, 0]); - assert_eq!(out_signs[0], 0b000101); - } - - #[test] - #[should_panic(expected = "table[0] must be 0")] - fn odd_lut_u8_refuses_a_table_that_moves_zero() { - let mut table = [0u8; 256]; - table[0] = 1; - odd_lut_u8(&[0], &[0], &table, &mut [0], &mut [0]); - } - - #[test] - #[should_panic(expected = "signs holds 1 words, need 2")] - fn odd_lut_u8_refuses_a_short_sign_plane() { - let table = [0u8; 256]; - odd_lut_u8(&[0; 65], &[0], &table, &mut [0; 65], &mut [0; 2]); - } - // ── Packed-bitmask predicates + mask algebra ──────────────────────────── // // Every test compares the shipped path against an INDEPENDENT scalar diff --git a/src/simd_stride.rs b/src/simd_stride.rs index f3914cce..d54c1027 100644 --- a/src/simd_stride.rs +++ b/src/simd_stride.rs @@ -5,9 +5,11 @@ //! moves a byte to read it: no shuffle, no permute, no table lookup //! assembled from shuffles and masks (CLAUDE.md, FACET STRIDE). //! -//! This module holds the reusable form of that shape so callers do not -//! rebuild it. Like [`crate::simd_masking_ops`] it names no ISA and owns no -//! backend; it is slice ergonomics over the position view. +//! This module holds the reusable forms of that shape: [`lut_u8`] for a +//! plain byte table and [`lut9`] for an odd function over a sign-magnitude +//! 9-bit (SM9) lane, u8 magnitude plus sign plane, so callers do not +//! rebuild them. Like [`crate::simd_masking_ops`] it names no ISA and owns +//! no backend; it is slice ergonomics over the position view. /// Bytes per facet: one 128-bit value read as sixteen u8 positions. pub const FACET_BYTES: usize = 16; @@ -52,10 +54,209 @@ pub fn lut_u8(values: &[u8], table: &[u8; 256], out: &mut [u8]) { } } +// ──────────────────────────────────────────────────────────────────────── +// lut9: an odd-function table over a sign-magnitude 9-bit (SM9) lane. +// +// A sign-magnitude 9-bit lane stores each value as a `u8` magnitude plus one +// bit in a `u64` sign plane per 64 rows; whether the row holds a value at all +// is a separate presence plane, and the writer stores `+0` (magnitude 0, sign +// clear) for zero and for missing rows. For an odd function +// (`f(-x) = -f(x)`, e.g. tanh, sin, x³, sign-preserving clamps) one table +// over the 256 magnitudes covers both signs: the magnitude is looked up and +// the sign plane passes through. The presence plane is not an input — a +// missing row is magnitude 0, `f(0) = 0`, so it stays `+0`. +// ──────────────────────────────────────────────────────────────────────── + +/// Applies an odd function, given as a 256-entry magnitude table, to a +/// sign-magnitude lane. +/// +/// For every row `i < magnitudes.len()`: +/// +/// * `out_magnitudes[i] = table[magnitudes[i]]`; +/// * the output sign bit is the input sign bit, cleared where a nonzero +/// input gives a zero result. A negative value whose result rounds to +/// zero is stored as `+0`, never `-0`. +/// * an input `-0` (magnitude 0, sign set) comes out as `-0`. A writer +/// that only stores `+0` never produces it, so under the plain numeric +/// reading this changes nothing. A class whose reading gives `-0` a +/// meaning of its own, such as "observed, value NaN" beside a clear +/// presence bit for "not observed", keeps that meaning through the +/// function, the way IEEE arithmetic carries NaN through. Clearing it +/// would turn that reading's NaN into the value 0. +/// +/// Rows past `magnitudes.len()` are not written in `out_magnitudes`. In +/// `out_signs` the bits past `magnitudes.len()` and any surplus words are +/// written as `0`. The presence plane is not read or written: because +/// `table[0]` must be `0`, a missing row (stored as `+0`) stays `+0`. +/// +/// Facet stride: every 16 magnitudes are one 128-bit value whose positions +/// `{0}..{15}` are read where they sit, and each position addresses the table +/// directly. No byte is shuffled. The sign plane is 64 rows per +/// [`crate::simd::U8x64::cmpeq_mask`]. The final partial chunk is padded with +/// magnitude 0 and goes through the same path, with the padding lanes masked +/// off the sign plane. +/// +/// # Panics +/// +/// * if `table[0] != 0`, since an odd function maps 0 to 0; +/// * if `out_magnitudes.len() < magnitudes.len()`; +/// * if `signs.len()` or `out_signs.len()` is less than +/// `magnitudes.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::lut9; +/// +/// // f(m) = m / 2, an odd function on the signed value. +/// let mut table = [0u8; 256]; +/// for (m, t) in table.iter_mut().enumerate() { +/// *t = (m / 2) as u8; +/// } +/// // Values: +10, -10, -1, +0 +/// let magnitudes = [10u8, 10, 1, 0]; +/// let signs = [0b0110u64]; +/// let mut out = [0u8; 4]; +/// let mut out_signs = [0u64; 1]; +/// lut9(&magnitudes, &signs, &table, &mut out, &mut out_signs); +/// assert_eq!(out, [5, 5, 0, 0]); +/// // -1 / 2 rounds to zero and is stored as +0: only row 1 keeps its sign. +/// assert_eq!(out_signs[0], 0b0010); +/// ``` +pub fn lut9(magnitudes: &[u8], signs: &[u64], table: &[u8; 256], out_magnitudes: &mut [u8], out_signs: &mut [u64]) { + use crate::simd::U8x64; + let n = magnitudes.len(); + let words = n.div_ceil(64); + assert_eq!(table[0], 0, "lut9: an odd function maps 0 to 0, so table[0] must be 0"); + assert!(out_magnitudes.len() >= n, "lut9: out_magnitudes holds {} rows, need {n}", out_magnitudes.len()); + assert!(signs.len() >= words, "lut9: signs holds {} words, need {words}", signs.len()); + assert!(out_signs.len() >= words, "lut9: out_signs holds {} words, need {words}", out_signs.len()); + + // FACET STRIDE: the magnitude lookup reads positions where they sit + // ([`lut_u8`]); no byte is shuffled. + let fill = |chunk: &[u8; 64], out: &mut [u8; 64]| lut_u8(chunk, table, out); + let zero = U8x64::splat(0); + let (in_chunks, tail) = magnitudes.as_chunks::<64>(); + let (out_chunks, out_tail) = out_magnitudes[..n].as_chunks_mut::<64>(); + for (w, (chunk, out)) in in_chunks.iter().zip(out_chunks).enumerate() { + fill(chunk, out); + let (m, r) = (U8x64::from_array(*chunk), U8x64::from_array(*out)); + // Keep the sign where the result is nonzero or the input is `-0`. + out_signs[w] = signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)); + } + let mut w = in_chunks.len(); + if !tail.is_empty() { + let mut padded = [0u8; 64]; + padded[..tail.len()].copy_from_slice(tail); + let mut looked_up = [0u8; 64]; + fill(&padded, &mut looked_up); + out_tail.copy_from_slice(&looked_up[..tail.len()]); + let (m, r) = (U8x64::from_array(padded), U8x64::from_array(looked_up)); + // Padding lanes look like input `-0` (magnitude 0), so the keep-`-0` + // rule would pass their junk sign bits; mask them off. + out_signs[w] = + signs[w] & (!r.cmpeq_mask(zero) | m.cmpeq_mask(zero)) & crate::simd_masking_ops::live64(tail.len()); + w += 1; + } + for s in &mut out_signs[w..] { + *s = 0; + } +} + #[cfg(test)] mod tests { use super::*; + // ── lut9 (SM9 odd table) ────────────────────────────────────────────────────────── + + fn lut9_reference(mags: &[u8], signs: &[u64], table: &[u8; 256]) -> (Vec, Vec) { + let out: Vec = mags.iter().map(|&m| table[m as usize]).collect(); + let mut out_signs = vec![0u64; mags.len().div_ceil(64)]; + for (i, &o) in out.iter().enumerate() { + let neg = (signs[i / 64] >> (i % 64)) & 1 == 1; + if neg && (o != 0 || mags[i] == 0) { + out_signs[i / 64] |= 1 << (i % 64); + } + } + (out, out_signs) + } + + fn lut9_xorshift(s: &mut u64) -> u64 { + *s ^= *s << 13; + *s ^= *s >> 7; + *s ^= *s << 17; + *s + } + + /// FAILS IF: a quarter of the table is picked by the wrong magnitude bits, + /// a sign bit survives on a zero result, the final partial chunk writes + /// rows past the end, or bits/words past the last row are left set. + #[test] + fn lut9_matches_the_table_at_every_lane_and_length() { + let mut s = 0x9E37_79B9_7F4A_7C15u64; + // Random table with table[0] = 0 and some zero entries elsewhere, so + // the sign-clearing rule is exercised away from magnitude 0 as well. + let mut table = [0u8; 256]; + for t in table.iter_mut().skip(1) { + let r = lut9_xorshift(&mut s) as u8; + *t = if r.is_multiple_of(7) { 0 } else { r }; + } + for &n in &[0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 300, 1024 + 17] { + for shift in [0usize, 1, 37, 101] { + // Every magnitude appears at every lane position across shifts. + let mags: Vec = (0..n).map(|i| ((i * 37 + shift) % 256) as u8).collect(); + let words = n.div_ceil(64); + let mut signs: Vec = (0..words).map(|_| lut9_xorshift(&mut s)).collect(); + // Junk sign bits past the last row must not leak into the output. + if let Some(last) = signs.last_mut() { + *last |= !crate::simd_masking_ops::live64(n - (words - 1) * 64); + } + let (want, want_signs) = lut9_reference(&mags, &signs, &table); + let mut out = vec![0xAAu8; n + 5]; + let mut out_signs = vec![u64::MAX; words + 2]; + lut9(&mags, &signs, &table, &mut out, &mut out_signs); + assert_eq!(&out[..n], &want[..], "n={n} shift={shift}"); + assert!(out[n..].iter().all(|&b| b == 0xAA), "rows past the end written, n={n}"); + assert_eq!(&out_signs[..words], &want_signs[..], "n={n} shift={shift}"); + assert!(out_signs[words..].iter().all(|&w| w == 0), "surplus words, n={n}"); + } + } + } + + /// FAILS IF: a negative value whose result rounds to zero keeps its sign + /// (a stored `-0`), or an input `-0` loses its sign. Silence half: a + /// nonzero negative result keeps its sign, and `+0` stays `+0`. + #[test] + fn lut9_rounds_to_plus_zero_and_keeps_an_input_minus_zero() { + let mut table = [0u8; 256]; + for (m, t) in table.iter_mut().enumerate() { + *t = (m / 4) as u8; + } + // -0, -3 (rounds to 0), -4 (-> -1), +3, +200, +0 + let mags = [0u8, 3, 4, 3, 200, 0]; + let signs = [0b000111u64]; + let mut out = [0u8; 6]; + let mut out_signs = [u64::MAX; 1]; + lut9(&mags, &signs, &table, &mut out, &mut out_signs); + assert_eq!(out, [0, 0, 1, 0, 50, 0]); + assert_eq!(out_signs[0], 0b000101); + } + + #[test] + #[should_panic(expected = "table[0] must be 0")] + fn lut9_refuses_a_table_that_moves_zero() { + let mut table = [0u8; 256]; + table[0] = 1; + lut9(&[0], &[0], &table, &mut [0], &mut [0]); + } + + #[test] + #[should_panic(expected = "signs holds 1 words, need 2")] + fn lut9_refuses_a_short_sign_plane() { + let table = [0u8; 256]; + lut9(&[0; 65], &[0], &table, &mut [0; 65], &mut [0; 2]); + } + fn table() -> [u8; 256] { // A non-monotone, non-affine table, so a position read at the wrong // index cannot land on the right value by accident.