From 26462834b22b8b21bea662527b29eafbdab09298 Mon Sep 17 00:00:00 2001 From: Aarush Shetty <182738217+AarushShetty346@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:25:18 +0530 Subject: [PATCH] Skip NULL values in SQL aggregates #891 --- .../converter/functions/AggregateAddCols.java | 8 ++- .../functions/AggregateFunction.java | 40 +++++------- .../functions/AggregateGetResult.java | 16 ++++- .../wayang/api/sql/SqlToWayangRelTest.java | 61 +++++++++++++++++++ .../test/resources/data/exampleNullGroups.csv | 7 +++ 5 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 wayang-api/wayang-api-sql/src/test/resources/data/exampleNullGroups.csv diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateAddCols.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateAddCols.java index 468c87f2d..7372bcd6e 100644 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateAddCols.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateAddCols.java @@ -50,7 +50,13 @@ public Record apply(final Record record) { switch (kind) { case COUNT: - resValues[i] = 1; + // COUNT(*) has no arguments, COUNT(col) only counts rows where col is not null + resValues[i] = argList.stream().allMatch(index -> record.getField(index) != null) ? 1 : 0; + break; + case AVG: + // running sum and number of non-null values, divided in AggregateGetResult + final Object value = record.getField(argList.get(0)); + resValues[i] = new Object[] { value, value == null ? 0 : 1 }; break; default: resValues[i] = record.getField(argList.get(0)); diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateFunction.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateFunction.java index f69379313..f39d722c3 100644 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateFunction.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateFunction.java @@ -19,7 +19,6 @@ import java.math.BigDecimal; import java.util.List; -import java.util.Optional; import java.util.function.BiFunction; import org.apache.calcite.rel.core.AggregateCall; @@ -42,7 +41,6 @@ public AggregateFunction(final List aggregateCalls) { public Record apply(final Record record1, final Record record2) { final int l = record1.size(); final Object[] resValues = new Object[l]; - boolean countDone = false; for (int i = 0; i < l - aggregateKinds.size() - 1; i++) { resValues[i] = record1.getField(i); @@ -75,14 +73,12 @@ public Record apply(final Record record1, final Record record2) { resValues[counter] = count; break; case AVG: - final Object avg = this.castAndMap(field1, field2, null, Long::sum, Integer::sum, Double::sum, BigDecimal::add); - - resValues[counter] = avg; - - if (!countDone) { - resValues[l - 1] = record1.getInt(l - 1) + record2.getInt(l - 1); - countDone = true; - } + // running sum and number of non-null values, see AggregateAddCols + final Object[] avg1 = (Object[]) field1; + final Object[] avg2 = (Object[]) field2; + resValues[counter] = new Object[] { + this.castAndMap(avg1[0], avg2[0], null, Long::sum, Integer::sum, Double::sum, BigDecimal::add), + (Integer) avg1[1] + (Integer) avg2[1] }; break; default: throw new IllegalStateException("Unsupported operation: " + kind); @@ -110,26 +106,22 @@ private Object castAndMap(final Object a, final Object b, final BiFunction integerMap, final BiFunction doubleMap, final BiFunction bigDecimalMap) { - // support operations between null and any - // class - if ((a == null || b == null) || (a.getClass() == b.getClass())) { - // objects can be null in this if statement due to - // condition above - final Optional aWrapped = Optional.ofNullable(a); - final Optional bWrapped = Optional.ofNullable(b); - - // force .getClass() to be safe so - // we can pass null objects to - // .apply methods. - return switch (aWrapped.orElse(bWrapped.orElse("")).getClass().getSimpleName()) { + // SQL aggregates skip null values, so a null on one side keeps the other side + if (a == null) { + return b; + } + if (b == null) { + return a; + } + if (a.getClass() == b.getClass()) { + return switch (a.getClass().getSimpleName()) { case "String" -> stringMap.apply((String) a, (String) b); case "Long" -> longMap.apply((Long) a, (Long) b); case "Integer" -> integerMap.apply((Integer) a, (Integer) b); case "Double" -> doubleMap.apply((Double) a, (Double) b); case "BigDecimal" -> bigDecimalMap.apply((BigDecimal) a, (BigDecimal) b); default -> throw new IllegalStateException("Unsupported operation between: " - + aWrapped.getClass().toString() - + " and: " + bWrapped.getClass().toString()); + + a.getClass().getSimpleName() + " and: " + b.getClass().getSimpleName()); }; } throw new IllegalStateException("Unsupported operation between: " + a.getClass().getSimpleName() + " and: " diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateGetResult.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateGetResult.java index 451db8d56..fc850e5c6 100644 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateGetResult.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/AggregateGetResult.java @@ -29,6 +29,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.wayang.basic.data.Record; import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.util.ReflectionUtils; public class AggregateGetResult implements FunctionDescriptor.SerializableFunction { private final List aggregateKindList; @@ -52,8 +53,7 @@ public Record apply(final Record record) { final Object[] aggregateCallFields = IntStream.range(0, aggregateKindList.size()) .mapToObj(i -> aggregateKindList.get(i).equals(SqlKind.AVG) - ? record.getDouble(i + aggregateCallOffset) - / record.getDouble(recordSize - 1) + ? average(record.getField(i + aggregateCallOffset)) : record.getField(i + aggregateCallOffset)) .toArray(); @@ -62,4 +62,16 @@ public Record apply(final Record record) { return new Record(combinedFields); } + + /** + * Divides the running sum of an AVG by its number of non-null values, see {@link AggregateAddCols}. + * + * @param sumAndCount the running sum and the number of non-null values + * @return the average, or null if all values were null + */ + private static Object average(final Object sumAndCount) { + final Object[] partial = (Object[]) sumAndCount; + final int count = (Integer) partial[1]; + return count == 0 ? null : ReflectionUtils.toDouble(partial[0]) / count; + } } \ No newline at end of file diff --git a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java index 91c1d7aef..fe3dfc1db 100755 --- a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java +++ b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java @@ -420,6 +420,46 @@ void javaAverage() throws Exception { result.stream().findFirst().orElseThrow(() -> new RuntimeException("No record found")).getDouble(0)); } + @Test + void javaAggregatesSkipNulls() throws Exception { + // col1 is 1, null, 3, null, 5, null: aggregates over col1 only see 1, 3 and 5 + assertEquals(List.of(new Record(6)), this.executeOnNullGroups("SELECT COUNT(*) FROM fs.exampleNullGroups")); + assertEquals(List.of(new Record(3)), this.executeOnNullGroups("SELECT COUNT(col1) FROM fs.exampleNullGroups")); + assertEquals(List.of(new Record(9)), this.executeOnNullGroups("SELECT SUM(col1) FROM fs.exampleNullGroups")); + assertEquals(List.of(new Record(3.0)), this.executeOnNullGroups("SELECT AVG(col1) FROM fs.exampleNullGroups")); + assertEquals(List.of(new Record(1)), this.executeOnNullGroups("SELECT MIN(col1) FROM fs.exampleNullGroups")); + assertEquals(List.of(new Record(5)), this.executeOnNullGroups("SELECT MAX(col1) FROM fs.exampleNullGroups")); + } + + private List executeOnNullGroups(final String sql) throws Exception { + final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv"); + + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, sql); + sqlContext.execute(t.field1); + + return new ArrayList<>(t.field0); + } + + @Test + void javaGroupedAggregatesSkipNulls() throws Exception { + final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv"); + + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, + "SELECT col2, COUNT(col1), SUM(col1), AVG(col1) FROM fs.exampleNullGroups GROUP BY col2"); + final Collection result = t.field0; + final WayangPlan wayangPlan = t.field1; + + sqlContext.execute(wayangPlan); + + // group c only has a null, so its SUM and AVG are null as well + final List expected = List.of( + new Record("a", 1, 1, 1.0), + new Record("b", 2, 8, 4.0), + new Record("c", 0, null, null)); + assertEquals(expected.size(), result.size()); + assertTrue(result.containsAll(expected), "expected " + expected + ", got: " + result); + } + @Test void filterNotEqualsValue() throws Exception { final SqlContext sqlContext = this.createSqlContext("/data/largeLeftTableIndex.csv"); @@ -665,6 +705,27 @@ void sparkFilter() throws Exception { assertTrue(result.stream().anyMatch(rec -> rec.equals(new Record("test1", "test1", "test2")))); } + @Test + void sparkGroupedAggregatesSkipNulls() throws Exception { + final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv"); + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, + "SELECT col2, COUNT(col1), SUM(col1), AVG(col1) FROM fs.exampleNullGroups GROUP BY col2"); + final Collection result = t.field0; + final WayangPlan wayangPlan = t.field1; + + PlanTraversal.upstream().traverse(wayangPlan.getSinks()).getTraversedNodes() + .forEach(node -> node.addTargetPlatform(Spark.platform())); + + sqlContext.execute(wayangPlan); + + final List expected = List.of( + new Record("a", 1, 1, 1.0), + new Record("b", 2, 8, 4.0), + new Record("c", 0, null, null)); + assertEquals(expected.size(), result.size()); + assertTrue(result.containsAll(expected), "expected " + expected + ", got: " + result); + } + @Test void sparkAggregate() throws Exception { final SqlContext sqlContext = this.createSqlContext("/data/largeLeftTableIndex.csv"); diff --git a/wayang-api/wayang-api-sql/src/test/resources/data/exampleNullGroups.csv b/wayang-api/wayang-api-sql/src/test/resources/data/exampleNullGroups.csv new file mode 100644 index 000000000..d424da44d --- /dev/null +++ b/wayang-api/wayang-api-sql/src/test/resources/data/exampleNullGroups.csv @@ -0,0 +1,7 @@ +col1:int,col2:string +1;a +;a +3;b +;b +5;b +;c