Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,13 @@ public Record apply(final Record record) {

switch (kind) {
case COUNT:
resValues[i] = 1;
// COUNT(*) has no arguments, COUNT(col) only counts rows where col is not null
resValues[i] = argList.stream().allMatch(index -> record.getField(index) != null) ? 1 : 0;
break;
case AVG:
// running sum and number of non-null values, divided in AggregateGetResult
final Object value = record.getField(argList.get(0));
resValues[i] = new Object[] { value, value == null ? 0 : 1 };
break;
default:
resValues[i] = record.getField(argList.get(0));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import java.math.BigDecimal;
import java.util.List;
import java.util.Optional;
import java.util.function.BiFunction;

import org.apache.calcite.rel.core.AggregateCall;
Expand All @@ -42,7 +41,6 @@ public AggregateFunction(final List<AggregateCall> aggregateCalls) {
public Record apply(final Record record1, final Record record2) {
final int l = record1.size();
final Object[] resValues = new Object[l];
boolean countDone = false;

for (int i = 0; i < l - aggregateKinds.size() - 1; i++) {
resValues[i] = record1.getField(i);
Expand Down Expand Up @@ -75,14 +73,12 @@ public Record apply(final Record record1, final Record record2) {
resValues[counter] = count;
break;
case AVG:
final Object avg = this.castAndMap(field1, field2, null, Long::sum, Integer::sum, Double::sum, BigDecimal::add);

resValues[counter] = avg;

if (!countDone) {
resValues[l - 1] = record1.getInt(l - 1) + record2.getInt(l - 1);
countDone = true;
}
// running sum and number of non-null values, see AggregateAddCols
final Object[] avg1 = (Object[]) field1;
final Object[] avg2 = (Object[]) field2;
resValues[counter] = new Object[] {
this.castAndMap(avg1[0], avg2[0], null, Long::sum, Integer::sum, Double::sum, BigDecimal::add),
(Integer) avg1[1] + (Integer) avg2[1] };
break;
default:
throw new IllegalStateException("Unsupported operation: " + kind);
Expand Down Expand Up @@ -110,26 +106,22 @@ private Object castAndMap(final Object a, final Object b,
final BiFunction<Integer, Integer, Object> integerMap,
final BiFunction<Double, Double, Object> doubleMap,
final BiFunction<BigDecimal, BigDecimal, Object> bigDecimalMap) {
// support operations between null and any
// class
if ((a == null || b == null) || (a.getClass() == b.getClass())) {
// objects can be null in this if statement due to
// condition above
final Optional<Object> aWrapped = Optional.ofNullable(a);
final Optional<Object> bWrapped = Optional.ofNullable(b);

// force .getClass() to be safe so
// we can pass null objects to
// .apply methods.
return switch (aWrapped.orElse(bWrapped.orElse("")).getClass().getSimpleName()) {
// SQL aggregates skip null values, so a null on one side keeps the other side
if (a == null) {
return b;
}
if (b == null) {
return a;
}
if (a.getClass() == b.getClass()) {
return switch (a.getClass().getSimpleName()) {
case "String" -> stringMap.apply((String) a, (String) b);
case "Long" -> longMap.apply((Long) a, (Long) b);
case "Integer" -> integerMap.apply((Integer) a, (Integer) b);
case "Double" -> doubleMap.apply((Double) a, (Double) b);
case "BigDecimal" -> bigDecimalMap.apply((BigDecimal) a, (BigDecimal) b);
default -> throw new IllegalStateException("Unsupported operation between: "
+ aWrapped.getClass().toString()
+ " and: " + bWrapped.getClass().toString());
+ a.getClass().getSimpleName() + " and: " + b.getClass().getSimpleName());
};
}
throw new IllegalStateException("Unsupported operation between: " + a.getClass().getSimpleName() + " and: "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.calcite.sql.SqlKind;
import org.apache.wayang.basic.data.Record;
import org.apache.wayang.core.function.FunctionDescriptor;
import org.apache.wayang.core.util.ReflectionUtils;

public class AggregateGetResult implements FunctionDescriptor.SerializableFunction<Record, Record> {
private final List<SqlKind> aggregateKindList;
Expand All @@ -52,8 +53,7 @@ public Record apply(final Record record) {

final Object[] aggregateCallFields = IntStream.range(0, aggregateKindList.size())
.mapToObj(i -> aggregateKindList.get(i).equals(SqlKind.AVG)
? record.getDouble(i + aggregateCallOffset)
/ record.getDouble(recordSize - 1)
? average(record.getField(i + aggregateCallOffset))
: record.getField(i + aggregateCallOffset))
.toArray();

Expand All @@ -62,4 +62,16 @@ public Record apply(final Record record) {

return new Record(combinedFields);
}

/**
* Divides the running sum of an AVG by its number of non-null values, see {@link AggregateAddCols}.
*
* @param sumAndCount the running sum and the number of non-null values
* @return the average, or null if all values were null
*/
private static Object average(final Object sumAndCount) {
final Object[] partial = (Object[]) sumAndCount;
final int count = (Integer) partial[1];
return count == 0 ? null : ReflectionUtils.toDouble(partial[0]) / count;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,46 @@ void javaAverage() throws Exception {
result.stream().findFirst().orElseThrow(() -> new RuntimeException("No record found")).getDouble(0));
}

@Test
void javaAggregatesSkipNulls() throws Exception {
// col1 is 1, null, 3, null, 5, null: aggregates over col1 only see 1, 3 and 5
assertEquals(List.of(new Record(6)), this.executeOnNullGroups("SELECT COUNT(*) FROM fs.exampleNullGroups"));
assertEquals(List.of(new Record(3)), this.executeOnNullGroups("SELECT COUNT(col1) FROM fs.exampleNullGroups"));
assertEquals(List.of(new Record(9)), this.executeOnNullGroups("SELECT SUM(col1) FROM fs.exampleNullGroups"));
assertEquals(List.of(new Record(3.0)), this.executeOnNullGroups("SELECT AVG(col1) FROM fs.exampleNullGroups"));
assertEquals(List.of(new Record(1)), this.executeOnNullGroups("SELECT MIN(col1) FROM fs.exampleNullGroups"));
assertEquals(List.of(new Record(5)), this.executeOnNullGroups("SELECT MAX(col1) FROM fs.exampleNullGroups"));
}

private List<Record> executeOnNullGroups(final String sql) throws Exception {
final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv");

final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext, sql);
sqlContext.execute(t.field1);

return new ArrayList<>(t.field0);
}

@Test
void javaGroupedAggregatesSkipNulls() throws Exception {
final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv");

final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext,
"SELECT col2, COUNT(col1), SUM(col1), AVG(col1) FROM fs.exampleNullGroups GROUP BY col2");
final Collection<Record> result = t.field0;
final WayangPlan wayangPlan = t.field1;

sqlContext.execute(wayangPlan);

// group c only has a null, so its SUM and AVG are null as well
final List<Record> expected = List.of(
new Record("a", 1, 1, 1.0),
new Record("b", 2, 8, 4.0),
new Record("c", 0, null, null));
assertEquals(expected.size(), result.size());
assertTrue(result.containsAll(expected), "expected " + expected + ", got: " + result);
}

@Test
void filterNotEqualsValue() throws Exception {
final SqlContext sqlContext = this.createSqlContext("/data/largeLeftTableIndex.csv");
Expand Down Expand Up @@ -665,6 +705,27 @@ void sparkFilter() throws Exception {
assertTrue(result.stream().anyMatch(rec -> rec.equals(new Record("test1", "test1", "test2"))));
}

@Test
void sparkGroupedAggregatesSkipNulls() throws Exception {
final SqlContext sqlContext = this.createSqlContext("/data/exampleNullGroups.csv");
final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(sqlContext,
"SELECT col2, COUNT(col1), SUM(col1), AVG(col1) FROM fs.exampleNullGroups GROUP BY col2");
final Collection<Record> result = t.field0;
final WayangPlan wayangPlan = t.field1;

PlanTraversal.upstream().traverse(wayangPlan.getSinks()).getTraversedNodes()
.forEach(node -> node.addTargetPlatform(Spark.platform()));

sqlContext.execute(wayangPlan);

final List<Record> expected = List.of(
new Record("a", 1, 1, 1.0),
new Record("b", 2, 8, 4.0),
new Record("c", 0, null, null));
assertEquals(expected.size(), result.size());
assertTrue(result.containsAll(expected), "expected " + expected + ", got: " + result);
}

@Test
void sparkAggregate() throws Exception {
final SqlContext sqlContext = this.createSqlContext("/data/largeLeftTableIndex.csv");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
col1:int,col2:string
1;a
;a
3;b
;b
5;b
;c