feat: Support SQL-NULL rows when shredding variant columns

src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs

@@ -53,6 +53,28 @@ public ShredSchema Infer(IEnumerable<VariantValue> values, ShredOptions options
             return BuildSchema(stats, totalCount, options, 0);
         }
 
+        /// <summary>
+        /// Infers a shredding schema by analyzing the given nullable values.
+        /// <c>null</c> entries are SQL-NULL rows and are ignored; they carry no type
+        /// information and do not count toward frequency thresholds.
+        /// </summary>
+        /// <param name="values">The variant values to analyze.</param>
+        /// <param name="options">Options controlling depth, frequency, and type consistency thresholds.</param>
+        /// <returns>An inferred <see cref="ShredSchema"/>.</returns>
+        public ShredSchema Infer(IEnumerable<VariantValue?> values, ShredOptions options = null)
+        {
+            if (values == null) throw new ArgumentNullException(nameof(values));
+            return Infer(NonNull(values), options);
+        }
+
+        private static IEnumerable<VariantValue> NonNull(IEnumerable<VariantValue?> values)
+        {
+            foreach (VariantValue? value in values)
+            {
+                if (value.HasValue) yield return value.Value;
+            }
+        }
+
         private void CollectStats(VariantValue value, TypeStats stats, int depth, int maxDepth)
         {
             ShredType type = VariantShredder.GetShredType(value);

src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs

@@ -36,7 +36,10 @@ public static class ShreddedVariantArrayBuilder
         /// </summary>
         /// <param name="schema">The shredding schema applied to each row.</param>
         /// <param name="metadata">The column-level variant metadata (shared across rows).</param>
-        /// <param name="rows">Per-row shred results whose residual bytes reference <paramref name="metadata"/>.</param>
+        /// <param name="rows">
+        /// Per-row shred results whose residual bytes reference <paramref name="metadata"/>.
+        /// A <c>null</c> entry produces a null (SQL-NULL) element in the resulting array.
+        /// </param>
         /// <param name="allocator">Arrow memory allocator, or default if null.</param>
         public static VariantArray Build(
             ShredSchema schema,
@@ -50,6 +53,33 @@ public static VariantArray Build(
 
             int rowCount = rows.Count;
 
+            // Top-level validity. Children of a null row are emitted as missing
+            // (value and typed_value both null); metadata is still populated
+            // because the metadata field is non-nullable.
+            ArrowBuffer.BitmapBuilder validity = new ArrowBuffer.BitmapBuilder(rowCount);
+            int nullCount = 0;
+            ShredResult[] childRows = null;
+            for (int i = 0; i < rowCount; i++)
+            {
+                if (rows[i] == null)
+                {
+                    if (childRows == null)
+                    {
+                        childRows = new ShredResult[rowCount];
+                        for (int j = 0; j < i; j++) childRows[j] = rows[j];
+                    }
+                    childRows[i] = ShredResult.Missing;
+                    validity.Append(false);
+                    nullCount++;
+                }
+                else
+                {
+                    if (childRows != null) childRows[i] = rows[i];
+                    validity.Append(true);
+                }
+            }
+            if (childRows != null) rows = childRows;
+
             // metadata column: emit the shared bytes once per row. (A dictionary-encoded
             // or run-end-encoded representation would compress this; VariantArray's reader
             // already handles those, but for simplicity we emit the plain binary form.)
@@ -81,8 +111,9 @@ public static VariantArray Build(
             }
 
             StructType structType = new StructType(fields);
+            ArrowBuffer nullBitmap = nullCount > 0 ? validity.Build(allocator) : ArrowBuffer.Empty;
             StructArray structArr = new StructArray(
-                structType, rowCount, children, ArrowBuffer.Empty, nullCount: 0);
+                structType, rowCount, children, nullBitmap, nullCount);
             // The public VariantArray(IArrowArray) constructor infers the VariantType
             // from the struct's shape (including detecting the shredded layout).
             return new VariantArray(structArr);

src/Apache.Arrow.Operations/Shredding/VariantShredder.cs

@@ -66,6 +66,39 @@ public static (byte[] Metadata, IReadOnlyList<ShredResult> Rows) Shred(
             return (metadataBytes, results);
         }
 
+        /// <summary>
+        /// Shreds a column of nullable variant values. A <c>null</c> entry is a
+        /// SQL-NULL row, as opposed to <see cref="VariantValue.Null"/>, which is a
+        /// present variant null. SQL-NULL rows produce a <c>null</c> entry in the
+        /// returned rows, which <see cref="ShreddedVariantArrayBuilder.Build"/> turns
+        /// into a null element of the resulting array.
+        /// </summary>
+        public static (byte[] Metadata, IReadOnlyList<ShredResult> Rows) Shred(
+            IEnumerable<VariantValue?> values,
+            ShredSchema schema)
+        {
+            if (values == null) throw new ArgumentNullException(nameof(values));
+            if (schema == null) throw new ArgumentNullException(nameof(schema));
+
+            List<VariantValue?> rows = values as List<VariantValue?> ?? new List<VariantValue?>(values);
+
+            VariantMetadataBuilder metadata = new VariantMetadataBuilder();
+            foreach (VariantValue? row in rows)
+            {
+                if (row.HasValue) CollectFieldNames(row.Value, metadata);
+            }
+            byte[] metadataBytes = metadata.Build(out int[] idRemap);
+
+            ShredResult[] results = new ShredResult[rows.Count];
+            for (int i = 0; i < rows.Count; i++)
+            {
+                VariantValue? row = rows[i];
+                results[i] = row.HasValue ? Shred(row.Value, schema, metadata, idRemap) : null;
+            }
+
+            return (metadataBytes, results);
+        }
+
         /// <summary>
         /// Shreds a single variant value against a caller-managed metadata dictionary.
         /// Use this when combining shredded columns with external metadata, or when

src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs

@@ -30,19 +30,21 @@ public static class VariantUnshredder
         /// <summary>
         /// Reconstructs a variant value from a shredded result.
         /// </summary>
-        /// <param name="shredded">The shredded (value, typed_value) pair.</param>
+        /// <param name="shredded">
+        /// The shredded (value, typed_value) pair, or null for a SQL-NULL row as produced by
+        /// <see cref="VariantShredder.Shred(System.Collections.Generic.IEnumerable{VariantValue?}, ShredSchema)"/>.
+        /// </param>
         /// <param name="schema">The shredding schema that was used to produce the result.</param>
         /// <param name="metadata">The column-level variant metadata bytes.</param>
         /// <returns>
-        /// The reconstructed <see cref="VariantValue"/>, or null if the field is missing
-        /// (both value and typed_value are null).
+        /// The reconstructed <see cref="VariantValue"/>, or null if <paramref name="shredded"/>
+        /// is null (a SQL-NULL row) or the field is missing (both value and typed_value are null).
         /// </returns>
         public static VariantValue? Reconstruct(ShredResult shredded, ShredSchema schema, ReadOnlySpan<byte> metadata)
         {
-            if (shredded == null) throw new ArgumentNullException(nameof(shredded));
             if (schema == null) throw new ArgumentNullException(nameof(schema));
 
-            if (shredded.IsMissing)
+            if (shredded == null || shredded.IsMissing)
             {
                 return null;
             }

test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs

@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Collections.Generic;
+using Apache.Arrow;
+using Apache.Arrow.Operations.Shredding;
+using Apache.Arrow.Scalars.Variant;
+using Xunit;
+
+namespace Apache.Arrow.Operations.Tests.Shredding
+{
+    /// <summary>
+    /// Tests for SQL-NULL rows (a null <see cref="VariantValue"/>?) flowing through
+    /// inference, shredding and array assembly, and staying distinct from a present
+    /// <see cref="VariantValue.Null"/>.
+    /// </summary>
+    public class ShredNullRowTests
+    {
+        private static VariantValue Obj(int a) => VariantValue.FromObject(
+            new Dictionary<string, VariantValue> { ["a"] = VariantValue.FromInt32(a) });
+
+        private static VariantArray ShredAndBuild(IReadOnlyList<VariantValue?> values, ShredSchema schema)
+        {
+            (byte[] metadata, IReadOnlyList<ShredResult> rows) = VariantShredder.Shred(values, schema);
+            return ShreddedVariantArrayBuilder.Build(schema, metadata, rows);
+        }
+
+        private static void AssertRows(IReadOnlyList<VariantValue?> expected, VariantArray array)
+        {
+            Assert.Equal(expected.Count, array.Length);
+            for (int i = 0; i < expected.Count; i++)
+            {
+                if (expected[i].HasValue)
+                {
+                    Assert.False(array.IsNull(i), $"row {i} should be valid");
+                    Assert.Equal(expected[i].Value, array.GetLogicalVariantValue(i));
+                }
+                else
+                {
+                    Assert.True(array.IsNull(i), $"row {i} should be null");
+                }
+            }
+        }
+
+        [Fact]
+        public void SqlNullRow_IsDistinctFromVariantNull()
+        {
+            var values = new List<VariantValue?> { Obj(1), null, VariantValue.Null, Obj(3) };
+
+            ShredSchema schema = new ShredSchemaInferer().Infer(values, ShredOptions.Default);
+            VariantArray array = ShredAndBuild(values, schema);
+
+            Assert.Equal(ShredType.Object, schema.TypedValueType);
+            Assert.Equal(1, array.NullCount);
+            Assert.Equal(1, array.StorageArray.NullCount);
+            AssertRows(values, array);
+            Assert.False(array.IsNull(2));
+            Assert.True(array.GetLogicalVariantValue(2).IsNull);
+        }
+
+        [Fact]
+        public void SqlNullRow_Unshredded()
+        {
+            var values = new List<VariantValue?> { VariantValue.FromString("x"), null, VariantValue.FromInt32(5) };
+            AssertRows(values, ShredAndBuild(values, ShredSchema.Unshredded()));
+        }
+
+        [Fact]
+        public void SqlNullRow_Primitive()
+        {
+            var values = new List<VariantValue?> { null, VariantValue.FromInt32(1), VariantValue.FromString("residual"), null };
+            VariantArray array = ShredAndBuild(values, ShredSchema.Primitive(ShredType.Int32));
+            Assert.Equal(2, array.NullCount);
+            AssertRows(values, array);
+        }
+
+        [Fact]
+        public void SqlNullRow_Array()
+        {
+            var values = new List<VariantValue?>
+            {
+                VariantValue.FromArray(VariantValue.FromInt32(1), VariantValue.FromInt32(2)),
+                null,
+                VariantValue.FromArray(VariantValue.FromInt32(3)),
+            };
+            AssertRows(values, ShredAndBuild(values, ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32))));
+        }
+
+        [Fact]
+        public void AllRowsNull()
+        {
+            var values = new List<VariantValue?> { null, null };
+            ShredSchema schema = new ShredSchemaInferer().Infer(values);
+            Assert.Equal(ShredType.None, schema.TypedValueType);
+
+            VariantArray array = ShredAndBuild(values, schema);
+            Assert.Equal(2, array.NullCount);
+            AssertRows(values, array);
+        }
+
+        [Fact]
+        public void NoNullRows_ProducesNoValidityBuffer()
+        {
+            var values = new List<VariantValue?> { Obj(1), Obj(2) };
+            VariantArray array = ShredAndBuild(values, new ShredSchemaInferer().Infer(values));
+            Assert.Equal(0, array.NullCount);
+            Assert.True(array.StorageArray.NullBitmapBuffer.IsEmpty);
+        }
+
+        [Fact]
+        public void SlicedArray_KeepsValidityAligned()
+        {
+            var values = new List<VariantValue?> { Obj(1), null, Obj(3), null, Obj(5) };
+            VariantArray array = ShredAndBuild(values, new ShredSchemaInferer().Infer(values));
+
+            var sliced = (VariantArray)ArrowArrayFactory.Slice(array, 1, 3);
+            AssertRows(new List<VariantValue?> { null, Obj(3), null }, sliced);
+        }
+
+        [Fact]
+        public void Infer_IgnoresNullRows()
+        {
+            // With nulls counted, Int32 would appear in only a third of the rows.
+            var values = new List<VariantValue?> { VariantValue.FromInt32(1), null, null, null, VariantValue.FromInt32(2), null };
+            ShredSchema schema = new ShredSchemaInferer().Infer(values);
+            Assert.Equal(ShredType.Int32, schema.TypedValueType);
+        }
+
+        [Fact]
+        public void Shred_ReturnsNullEntryForNullRow()
+        {
+            var values = new List<VariantValue?> { Obj(1), null };
+            (byte[] metadata, IReadOnlyList<ShredResult> rows) = VariantShredder.Shred(values, ShredSchema.Unshredded());
+            Assert.NotNull(metadata);
+            Assert.NotNull(rows[0]);
+            Assert.Null(rows[1]);
+        }
+
+        [Fact]
+        public void Reconstruct_RoundTripsNullRows()
+        {
+            var values = new List<VariantValue?> { Obj(1), null, VariantValue.Null, VariantValue.FromString("s") };
+            ShredSchema schema = new ShredSchemaInferer().Infer(values);
+            (byte[] metadata, IReadOnlyList<ShredResult> rows) = VariantShredder.Shred(values, schema);
+
+            for (int i = 0; i < values.Count; i++)
+            {
+                Assert.Equal(values[i], VariantUnshredder.Reconstruct(rows[i], schema, metadata));
+            }
+        }
+    }
+}