From 27a3baafe52f2b1f63a1e2810bb1e1c2b2821581 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sat, 26 Sep 2026 17:37:01 -0700 Subject: [PATCH 1/2] feat: Support SQL-NULL rows when shredding variant columns Add Infer and Shred overloads taking IEnumerable, where a null entry is a SQL-NULL row (distinct from a present VariantValue.Null). ShreddedVariantArrayBuilder.Build now turns null entries in its rows into null elements of the storage struct. Co-Authored-By: Claude Opus 5.5 --- .../Shredding/ShredSchemaInferer.cs | 22 +++ .../Shredding/ShreddedVariantArrayBuilder.cs | 35 +++- .../Shredding/VariantShredder.cs | 33 ++++ .../Shredding/ShredNullRowTests.cs | 151 ++++++++++++++++++ 4 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs diff --git a/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs index 4537c3cb..ed295324 100644 --- a/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs +++ b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs @@ -53,6 +53,28 @@ public ShredSchema Infer(IEnumerable values, ShredOptions options return BuildSchema(stats, totalCount, options, 0); } + /// + /// Infers a shredding schema by analyzing the given nullable values. + /// null entries are SQL-NULL rows and are ignored; they carry no type + /// information and do not count toward frequency thresholds. + /// + /// The variant values to analyze. + /// Options controlling depth, frequency, and type consistency thresholds. + /// An inferred . + public ShredSchema Infer(IEnumerable values, ShredOptions options = null) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + return Infer(NonNull(values), options); + } + + private static IEnumerable NonNull(IEnumerable values) + { + foreach (VariantValue? value in values) + { + if (value.HasValue) yield return value.Value; + } + } + private void CollectStats(VariantValue value, TypeStats stats, int depth, int maxDepth) { ShredType type = VariantShredder.GetShredType(value); diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs index 44b6de59..3428fa18 100644 --- a/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs +++ b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs @@ -36,7 +36,10 @@ public static class ShreddedVariantArrayBuilder /// /// The shredding schema applied to each row. /// The column-level variant metadata (shared across rows). - /// Per-row shred results whose residual bytes reference . + /// + /// Per-row shred results whose residual bytes reference . + /// A null entry produces a null (SQL-NULL) element in the resulting array. + /// /// Arrow memory allocator, or default if null. public static VariantArray Build( ShredSchema schema, @@ -50,6 +53,33 @@ public static VariantArray Build( int rowCount = rows.Count; + // Top-level validity. Children of a null row are emitted as missing + // (value and typed_value both null); metadata is still populated + // because the metadata field is non-nullable. + ArrowBuffer.BitmapBuilder validity = new ArrowBuffer.BitmapBuilder(rowCount); + int nullCount = 0; + ShredResult[] childRows = null; + for (int i = 0; i < rowCount; i++) + { + if (rows[i] == null) + { + if (childRows == null) + { + childRows = new ShredResult[rowCount]; + for (int j = 0; j < i; j++) childRows[j] = rows[j]; + } + childRows[i] = ShredResult.Missing; + validity.Append(false); + nullCount++; + } + else + { + if (childRows != null) childRows[i] = rows[i]; + validity.Append(true); + } + } + if (childRows != null) rows = childRows; + // metadata column: emit the shared bytes once per row. (A dictionary-encoded // or run-end-encoded representation would compress this; VariantArray's reader // already handles those, but for simplicity we emit the plain binary form.) @@ -81,8 +111,9 @@ public static VariantArray Build( } StructType structType = new StructType(fields); + ArrowBuffer nullBitmap = nullCount > 0 ? validity.Build(allocator) : ArrowBuffer.Empty; StructArray structArr = new StructArray( - structType, rowCount, children, ArrowBuffer.Empty, nullCount: 0); + structType, rowCount, children, nullBitmap, nullCount); // The public VariantArray(IArrowArray) constructor infers the VariantType // from the struct's shape (including detecting the shredded layout). return new VariantArray(structArr); diff --git a/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs index ef2ed11d..501e028f 100644 --- a/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs +++ b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs @@ -66,6 +66,39 @@ public static (byte[] Metadata, IReadOnlyList Rows) Shred( return (metadataBytes, results); } + /// + /// Shreds a column of nullable variant values. A null entry is a + /// SQL-NULL row, as opposed to , which is a + /// present variant null. SQL-NULL rows produce a null entry in the + /// returned rows, which turns + /// into a null element of the resulting array. + /// + public static (byte[] Metadata, IReadOnlyList Rows) Shred( + IEnumerable values, + ShredSchema schema) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + if (schema == null) throw new ArgumentNullException(nameof(schema)); + + List rows = values as List ?? new List(values); + + VariantMetadataBuilder metadata = new VariantMetadataBuilder(); + foreach (VariantValue? row in rows) + { + if (row.HasValue) CollectFieldNames(row.Value, metadata); + } + byte[] metadataBytes = metadata.Build(out int[] idRemap); + + ShredResult[] results = new ShredResult[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + VariantValue? row = rows[i]; + results[i] = row.HasValue ? Shred(row.Value, schema, metadata, idRemap) : null; + } + + return (metadataBytes, results); + } + /// /// Shreds a single variant value against a caller-managed metadata dictionary. /// Use this when combining shredded columns with external metadata, or when diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs new file mode 100644 index 00000000..2b924946 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Tests for SQL-NULL rows (a null ?) flowing through + /// inference, shredding and array assembly, and staying distinct from a present + /// . + /// + public class ShredNullRowTests + { + private static VariantValue Obj(int a) => VariantValue.FromObject( + new Dictionary { ["a"] = VariantValue.FromInt32(a) }); + + private static VariantArray ShredAndBuild(IReadOnlyList values, ShredSchema schema) + { + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, schema); + return ShreddedVariantArrayBuilder.Build(schema, metadata, rows); + } + + private static void AssertRows(IReadOnlyList expected, VariantArray array) + { + Assert.Equal(expected.Count, array.Length); + for (int i = 0; i < expected.Count; i++) + { + if (expected[i].HasValue) + { + Assert.False(array.IsNull(i), $"row {i} should be valid"); + Assert.Equal(expected[i].Value, array.GetLogicalVariantValue(i)); + } + else + { + Assert.True(array.IsNull(i), $"row {i} should be null"); + } + } + } + + [Fact] + public void SqlNullRow_IsDistinctFromVariantNull() + { + var values = new List { Obj(1), null, VariantValue.Null, Obj(3) }; + + ShredSchema schema = new ShredSchemaInferer().Infer(values, ShredOptions.Default); + VariantArray array = ShredAndBuild(values, schema); + + Assert.Equal(ShredType.Object, schema.TypedValueType); + Assert.Equal(1, array.NullCount); + Assert.Equal(1, array.StorageArray.NullCount); + AssertRows(values, array); + Assert.False(array.IsNull(2)); + Assert.True(array.GetLogicalVariantValue(2).IsNull); + } + + [Fact] + public void SqlNullRow_Unshredded() + { + var values = new List { VariantValue.FromString("x"), null, VariantValue.FromInt32(5) }; + AssertRows(values, ShredAndBuild(values, ShredSchema.Unshredded())); + } + + [Fact] + public void SqlNullRow_Primitive() + { + var values = new List { null, VariantValue.FromInt32(1), VariantValue.FromString("residual"), null }; + VariantArray array = ShredAndBuild(values, ShredSchema.Primitive(ShredType.Int32)); + Assert.Equal(2, array.NullCount); + AssertRows(values, array); + } + + [Fact] + public void SqlNullRow_Array() + { + var values = new List + { + VariantValue.FromArray(VariantValue.FromInt32(1), VariantValue.FromInt32(2)), + null, + VariantValue.FromArray(VariantValue.FromInt32(3)), + }; + AssertRows(values, ShredAndBuild(values, ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)))); + } + + [Fact] + public void AllRowsNull() + { + var values = new List { null, null }; + ShredSchema schema = new ShredSchemaInferer().Infer(values); + Assert.Equal(ShredType.None, schema.TypedValueType); + + VariantArray array = ShredAndBuild(values, schema); + Assert.Equal(2, array.NullCount); + AssertRows(values, array); + } + + [Fact] + public void NoNullRows_ProducesNoValidityBuffer() + { + var values = new List { Obj(1), Obj(2) }; + VariantArray array = ShredAndBuild(values, new ShredSchemaInferer().Infer(values)); + Assert.Equal(0, array.NullCount); + Assert.True(array.StorageArray.NullBitmapBuffer.IsEmpty); + } + + [Fact] + public void SlicedArray_KeepsValidityAligned() + { + var values = new List { Obj(1), null, Obj(3), null, Obj(5) }; + VariantArray array = ShredAndBuild(values, new ShredSchemaInferer().Infer(values)); + + var sliced = (VariantArray)ArrowArrayFactory.Slice(array, 1, 3); + AssertRows(new List { null, Obj(3), null }, sliced); + } + + [Fact] + public void Infer_IgnoresNullRows() + { + // With nulls counted, Int32 would appear in only a third of the rows. + var values = new List { VariantValue.FromInt32(1), null, null, null, VariantValue.FromInt32(2), null }; + ShredSchema schema = new ShredSchemaInferer().Infer(values); + Assert.Equal(ShredType.Int32, schema.TypedValueType); + } + + [Fact] + public void Shred_ReturnsNullEntryForNullRow() + { + var values = new List { Obj(1), null }; + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, ShredSchema.Unshredded()); + Assert.NotNull(metadata); + Assert.NotNull(rows[0]); + Assert.Null(rows[1]); + } + } +} From 9155656a377ff4c1ad0a1c5f00d3ba2c39632b25 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sat, 26 Sep 2026 20:24:58 -0700 Subject: [PATCH 2/2] fix: Let VariantUnshredder.Reconstruct accept SQL-NULL rows The nullable Shred overload returns a null ShredResult for SQL-NULL rows; Reconstruct now returns null for those instead of throwing. Co-Authored-By: Claude Opus 5.5 --- .../Shredding/VariantUnshredder.cs | 12 +++++++----- .../Shredding/ShredNullRowTests.cs | 13 +++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs index 346e57d8..ecc9e101 100644 --- a/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs +++ b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs @@ -30,19 +30,21 @@ public static class VariantUnshredder /// /// Reconstructs a variant value from a shredded result. /// - /// The shredded (value, typed_value) pair. + /// + /// The shredded (value, typed_value) pair, or null for a SQL-NULL row as produced by + /// . + /// /// The shredding schema that was used to produce the result. /// The column-level variant metadata bytes. /// - /// The reconstructed , or null if the field is missing - /// (both value and typed_value are null). + /// The reconstructed , or null if + /// is null (a SQL-NULL row) or the field is missing (both value and typed_value are null). /// public static VariantValue? Reconstruct(ShredResult shredded, ShredSchema schema, ReadOnlySpan metadata) { - if (shredded == null) throw new ArgumentNullException(nameof(shredded)); if (schema == null) throw new ArgumentNullException(nameof(schema)); - if (shredded.IsMissing) + if (shredded == null || shredded.IsMissing) { return null; } diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs index 2b924946..f275a3f6 100644 --- a/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs @@ -147,5 +147,18 @@ public void Shred_ReturnsNullEntryForNullRow() Assert.NotNull(rows[0]); Assert.Null(rows[1]); } + + [Fact] + public void Reconstruct_RoundTripsNullRows() + { + var values = new List { Obj(1), null, VariantValue.Null, VariantValue.FromString("s") }; + ShredSchema schema = new ShredSchemaInferer().Infer(values); + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, schema); + + for (int i = 0; i < values.Count; i++) + { + Assert.Equal(values[i], VariantUnshredder.Reconstruct(rows[i], schema, metadata)); + } + } } }