Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.calcite.rel.RelFieldCollation.NullDirection;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;

Expand Down Expand Up @@ -61,13 +62,20 @@ Operator visit(final WayangSort wayangRelNode) {
.map(fieldCol -> fieldCol.getDirection())
.collect(Collectors.toList());

final List<NullDirection> nullDirections = collation.getFieldCollations().stream()
.map(fieldCol -> fieldCol.nullDirection == NullDirection.UNSPECIFIED
? fieldCol.getDirection().defaultNullDirection()
: fieldCol.nullDirection)
.collect(Collectors.toList());

final List<Integer> collationIndexes = collation.getFieldCollations().stream()
.map(fieldCol -> fieldCol.getFieldIndex())
.collect(Collectors.toList());

final TransformationDescriptor<Record, Record> td = new TransformationDescriptor<Record, Record>(
new SortKeyExtractor(
collationDirections,
nullDirections,
collationIndexes),
Record.class, Record.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,29 @@
package org.apache.wayang.api.sql.calcite.converter.functions;

import java.util.List;
import java.util.stream.Collectors;

import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.calcite.rel.RelFieldCollation.NullDirection;

import org.apache.wayang.basic.data.Record;
import org.apache.wayang.core.function.FunctionDescriptor;

public class SortKeyExtractor implements FunctionDescriptor.SerializableFunction<Record, Record> {
final List<Direction> directions;
final List<NullDirection> nullDirections;
final List<Integer> collationIndexes;

public SortKeyExtractor(final List<Direction> collationDirections, final List<Integer> collationIndexes) {
this(collationDirections,
collationDirections.stream().map(Direction::defaultNullDirection).collect(Collectors.toList()),
collationIndexes);
}

public SortKeyExtractor(final List<Direction> collationDirections, final List<NullDirection> nullDirections,
final List<Integer> collationIndexes) {
this.directions = collationDirections;
this.nullDirections = nullDirections;
this.collationIndexes = collationIndexes;
}

Expand All @@ -45,7 +56,14 @@ public int compareTo(final Record that) throws IllegalStateException {
final Direction direction = directions.get(i);
final Comparable thisField = (Comparable) this.getValues()[i];
final Comparable thatField = (Comparable) that.getValues()[i];


// NULLS FIRST / NULLS LAST places nulls regardless of the sort direction
if (thisField == null || thatField == null) {
if (thisField == thatField) continue;
final boolean nullsFirst = nullDirections.get(i) == NullDirection.FIRST;
return (thisField == null) == nullsFirst ? -1 : 1;
}

// == 0, < -1, > 1
final int comparison = direction.isDescending() ? -thisField.compareTo(thatField) : thisField.compareTo(thatField);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,56 @@ void javaSort() throws Exception {
assertEquals(new Record(0, "b", "b", 1), result.get(6));
}

@Test
void javaSortWithNullsAscending() throws Exception {
final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv");

final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext,
"SELECT col1, col2 FROM fs.exampleNull ORDER BY col1");

final Collection<Record> r = t.field0;
final WayangPlan wayangPlan = t.field1;

sqlContext.execute(wayangPlan);

// ascending order puts nulls last by default
assertEquals(Arrays.asList(new Record(1, "a"), new Record(3, "c"), new Record(null, "b")),
new ArrayList<>(r));
}

@Test
void javaSortWithNullsDescending() throws Exception {
final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv");

final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext,
"SELECT col1, col2 FROM fs.exampleNull ORDER BY col1 DESC");

final Collection<Record> r = t.field0;
final WayangPlan wayangPlan = t.field1;

sqlContext.execute(wayangPlan);

// descending order puts nulls first by default
assertEquals(Arrays.asList(new Record(null, "b"), new Record(3, "c"), new Record(1, "a")),
new ArrayList<>(r));
}

@Test
void javaSortWithNullsFirst() throws Exception {
final SqlContext sqlContext = createSqlContext("/data/exampleNull.csv");

final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext,
"SELECT col1, col2 FROM fs.exampleNull ORDER BY col1 NULLS FIRST");

final Collection<Record> r = t.field0;
final WayangPlan wayangPlan = t.field1;

sqlContext.execute(wayangPlan);

assertEquals(Arrays.asList(new Record(null, "b"), new Record(1, "a"), new Record(3, "c")),
new ArrayList<>(r));
}

@Test
void joinWithLargeLeftTableIndexCorrect() throws Exception {
final SqlContext sqlContext = createSqlContext("/data/largeLeftTableIndex.csv");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
col1:int,col2:string
1;a
;b
3;c