diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangSortVisitor.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangSortVisitor.java index fec635f47..41706b2da 100644 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangSortVisitor.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangSortVisitor.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; @@ -61,6 +62,12 @@ Operator visit(final WayangSort wayangRelNode) { .map(fieldCol -> fieldCol.getDirection()) .collect(Collectors.toList()); + final List nullDirections = collation.getFieldCollations().stream() + .map(fieldCol -> fieldCol.nullDirection == NullDirection.UNSPECIFIED + ? fieldCol.getDirection().defaultNullDirection() + : fieldCol.nullDirection) + .collect(Collectors.toList()); + final List collationIndexes = collation.getFieldCollations().stream() .map(fieldCol -> fieldCol.getFieldIndex()) .collect(Collectors.toList()); @@ -68,6 +75,7 @@ Operator visit(final WayangSort wayangRelNode) { final TransformationDescriptor td = new TransformationDescriptor( new SortKeyExtractor( collationDirections, + nullDirections, collationIndexes), Record.class, Record.class); diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/SortKeyExtractor.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/SortKeyExtractor.java index 440152f29..b544f57fd 100644 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/SortKeyExtractor.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/SortKeyExtractor.java @@ -19,18 +19,29 @@ package org.apache.wayang.api.sql.calcite.converter.functions; import java.util.List; +import java.util.stream.Collectors; import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.wayang.basic.data.Record; import org.apache.wayang.core.function.FunctionDescriptor; public class SortKeyExtractor implements FunctionDescriptor.SerializableFunction { final List directions; + final List nullDirections; final List collationIndexes; public SortKeyExtractor(final List collationDirections, final List collationIndexes) { + this(collationDirections, + collationDirections.stream().map(Direction::defaultNullDirection).collect(Collectors.toList()), + collationIndexes); + } + + public SortKeyExtractor(final List collationDirections, final List nullDirections, + final List collationIndexes) { this.directions = collationDirections; + this.nullDirections = nullDirections; this.collationIndexes = collationIndexes; } @@ -45,7 +56,14 @@ public int compareTo(final Record that) throws IllegalStateException { final Direction direction = directions.get(i); final Comparable thisField = (Comparable) this.getValues()[i]; final Comparable thatField = (Comparable) that.getValues()[i]; - + + // NULLS FIRST / NULLS LAST places nulls regardless of the sort direction + if (thisField == null || thatField == null) { + if (thisField == thatField) continue; + final boolean nullsFirst = nullDirections.get(i) == NullDirection.FIRST; + return (thisField == null) == nullsFirst ? -1 : 1; + } + // == 0, < -1, > 1 final int comparison = direction.isDescending() ? -thisField.compareTo(thatField) : thisField.compareTo(thatField); diff --git a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java index 91c1d7aef..92259afdd 100755 --- a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java +++ b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java @@ -598,6 +598,56 @@ void javaSort() throws Exception { assertEquals(new Record(0, "b", "b", 1), result.get(6)); } + @Test + void javaSortWithNullsAscending() throws Exception { + final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv"); + + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, + "SELECT col1, col2 FROM fs.exampleNull ORDER BY col1"); + + final Collection r = t.field0; + final WayangPlan wayangPlan = t.field1; + + sqlContext.execute(wayangPlan); + + // ascending order puts nulls last by default + assertEquals(Arrays.asList(new Record(1, "a"), new Record(3, "c"), new Record(null, "b")), + new ArrayList<>(r)); + } + + @Test + void javaSortWithNullsDescending() throws Exception { + final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv"); + + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, + "SELECT col1, col2 FROM fs.exampleNull ORDER BY col1 DESC"); + + final Collection r = t.field0; + final WayangPlan wayangPlan = t.field1; + + sqlContext.execute(wayangPlan); + + // descending order puts nulls first by default + assertEquals(Arrays.asList(new Record(null, "b"), new Record(3, "c"), new Record(1, "a")), + new ArrayList<>(r)); + } + + @Test + void javaSortWithNullsFirst() throws Exception { + final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv"); + + final Tuple2, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, + "SELECT col1, col2 FROM fs.exampleNull ORDER BY col1 NULLS FIRST"); + + final Collection r = t.field0; + final WayangPlan wayangPlan = t.field1; + + sqlContext.execute(wayangPlan); + + assertEquals(Arrays.asList(new Record(null, "b"), new Record(1, "a"), new Record(3, "c")), + new ArrayList<>(r)); + } + @Test void joinWithLargeLeftTableIndexCorrect() throws Exception { final SqlContext sqlContext = createSqlContext("/data/largeLeftTableIndex.csv"); diff --git a/wayang-api/wayang-api-sql/src/test/resources/data/exampleNull.csv b/wayang-api/wayang-api-sql/src/test/resources/data/exampleNull.csv new file mode 100644 index 000000000..0810853eb --- /dev/null +++ b/wayang-api/wayang-api-sql/src/test/resources/data/exampleNull.csv @@ -0,0 +1,4 @@ +col1:int,col2:string +1;a +;b +3;c