Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.wayang.api.sql.calcite.converter;

import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;

import org.apache.wayang.api.sql.calcite.rel.WayangTableScan;
Expand All @@ -32,8 +33,8 @@
import org.apache.wayang.basic.data.Record;
import org.apache.wayang.basic.operators.TableSource;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

//TODO: create tablesource with column types
//TODO: support other sources
Expand Down Expand Up @@ -64,25 +65,11 @@ Operator visit(final WayangTableScan wayangRelNode) {
"Could not initialize calcite model parser from current Wayang configuration");
}

final List<CsvType> fieldTypes = wayangRelNode.getRowType().getFieldList().stream()
.map(RelDataTypeField::getType)
.map(type -> {
switch (type.getSqlTypeName()) {
case BOOLEAN: return CsvType.BOOLEAN;
case TINYINT: return CsvType.TINYINT;
case SMALLINT: return CsvType.SMALLINT;
case INTEGER: return CsvType.INTEGER;
case BIGINT: return CsvType.BIGINT;
case FLOAT: return CsvType.FLOAT;
case DOUBLE: return CsvType.DOUBLE;
case DECIMAL: return CsvType.decimal(type.getPrecision(), type.getScale());
case DATE: return CsvType.DATE;
case TIME: return CsvType.TIME;
case TIMESTAMP: return CsvType.TIMESTAMP;
default: return CsvType.STRING;
}
})
.collect(Collectors.toList());
final List<RelDataTypeField> allFields = wayangRelNode.getRowType().getFieldList();
final List<CsvType> fieldTypes = new ArrayList<>(wayangRelNode.getFields().length);
for (int fieldIndex : wayangRelNode.getFields()) {
fieldTypes.add(csvType(allFields.get(fieldIndex).getType()));
}

final String url = String.format("file:/%s/%s.csv", modelParser.getFsPath(), wayangRelNode.getTableName());

Expand All @@ -92,7 +79,7 @@ Operator visit(final WayangTableScan wayangRelNode) {
} else if (wayangRelNode.getTable().getQualifiedName().size() == 1) {
// we assume that it is coming from a test environement or in memory db.

return new JdbcTableSource(wayangRelNode.getTable().getQualifiedName().get(0), wayangRelNode.getRowType().getFieldNames().toArray(String[]::new)) {
return new JdbcTableSource(wayangRelNode.getTable().getQualifiedName().get(0), wayangRelNode.getColumnNames().toArray(String[]::new)) {

@Override
public JdbcPlatformTemplate getPlatform() {
Expand All @@ -103,4 +90,21 @@ public JdbcPlatformTemplate getPlatform() {
throw new RuntimeException(
"Source not supported, got: " + tableSource + ", expected either postgres or filesystem (fs).");
}

private static CsvType csvType(RelDataType type) {
switch (type.getSqlTypeName()) {
case BOOLEAN: return CsvType.BOOLEAN;
case TINYINT: return CsvType.TINYINT;
case SMALLINT: return CsvType.SMALLINT;
case INTEGER: return CsvType.INTEGER;
case BIGINT: return CsvType.BIGINT;
case FLOAT: return CsvType.FLOAT;
case DOUBLE: return CsvType.DOUBLE;
case DECIMAL: return CsvType.decimal(type.getPrecision(), type.getScale());
case DATE: return CsvType.DATE;
case TIME: return CsvType.TIME;
case TIMESTAMP: return CsvType.TIMESTAMP;
default: return CsvType.STRING;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@
import org.apache.calcite.schema.Table;
import org.apache.wayang.api.sql.calcite.convention.WayangConvention;

import java.util.ArrayList;
import java.util.List;

public class WayangTableScan extends TableScan implements WayangRel {

//TODO: fields are never queried, why?
/**
* Ordinals of the columns this scan reads. Defaults to every column of the table.
*/
private final int[] fields;

public WayangTableScan(RelOptCluster cluster,
Expand Down Expand Up @@ -94,7 +97,19 @@ public String getTableName() {
: table.getQualifiedName().get(1);
}

/**
* @return the columns selected by {@link #fields}, in that order
*/
public List<String> getColumnNames() {
return table.getRowType().getFieldNames();
final List<String> names = table.getRowType().getFieldNames();
final List<String> projected = new ArrayList<>(this.fields.length);
for (int field : this.fields) {
projected.add(names.get(field));
}
return projected;
}

public int[] getFields() {
return this.fields.clone();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.wayang.basic.types.RecordType;
import org.apache.wayang.core.plan.wayangplan.UnarySource;
import org.apache.wayang.core.types.DataSetType;
import org.apache.wayang.core.types.DataUnitType;

/**
* {@link UnarySource} that provides the tuples from a database table.
Expand All @@ -30,16 +31,31 @@ public class TableSource extends UnarySource<Record> {

private final String tableName;

/**
* Columns this source reads. Empty when the caller did not name any columns.
*/
private final String[] columnNames;

public String getTableName() {
return tableName;
}

/**
* @return a copy of the column names, or an empty array when none were given
*/
public String[] getColumnNames() {
if (this.columnNames == null) {
return new String[0];
}
return this.columnNames.clone();
}

/**
* Creates a new instance.
*
* @param tableName name of the table to be read
* @param columnNames names of the columns in the tables; can be omitted but allows to inject schema information
* into Wayang, so as to allow specific optimizations
* @param columnNames names of the columns to read; can be omitted. When they are plain SQL identifiers,
* JDBC scans select these columns instead of {@code *}
*/
public TableSource(String tableName, String... columnNames) {
this(tableName, createOutputDataSetType(columnNames));
Expand All @@ -48,6 +64,22 @@ public TableSource(String tableName, String... columnNames) {
public TableSource(String tableName, DataSetType<Record> type) {
super(type);
this.tableName = tableName;
this.columnNames = columnNamesFrom(type);
}

private static String[] columnNamesFrom(DataSetType<Record> type) {
if (type == null) {
return new String[0];
}
final DataUnitType<Record> dataUnitType = type.getDataUnitType();
if (!(dataUnitType instanceof RecordType)) {
return new String[0];
}
final String[] names = ((RecordType) dataUnitType).getFieldNames();
if (names == null || names.length == 0) {
return new String[0];
}
return names.clone();
}

/**
Expand All @@ -71,6 +103,7 @@ private static DataSetType<Record> createOutputDataSetType(String[] columnNames)
public TableSource(TableSource that) {
super(that);
this.tableName = that.getTableName();
this.columnNames = that.getColumnNames();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.wayang.basic.operators;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

/**
* Tests for {@link TableSource}.
*/
class TableSourceTest {

@Test
void keepsColumnNames() {
TableSource source = new TableSource("customer", "name", "age");
assertArrayEquals(new String[]{"name", "age"}, source.getColumnNames());
}

@Test
void copyKeepsColumnNames() {
TableSource source = new TableSource("customer", "name", "age");
TableSource copy = new TableSource(source);
assertArrayEquals(new String[]{"name", "age"}, copy.getColumnNames());
copy.getColumnNames()[0] = "changed";
assertArrayEquals(new String[]{"name", "age"}, source.getColumnNames());
}

@Test
void omitsColumnNamesWhenUnknown() {
TableSource source = new TableSource("customer");
assertEquals(0, source.getColumnNames().length);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.wayang.basic.operators.JoinOperator;
import org.apache.wayang.basic.operators.SpatialFilterOperator;
import org.apache.wayang.basic.operators.SpatialJoinOperator;
import org.apache.wayang.basic.util.SqlIdentifierUtils;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.Job;
import org.apache.wayang.core.api.exception.WayangException;
Expand All @@ -58,6 +59,7 @@
import org.apache.wayang.jdbc.operators.JdbcSortOperator;
import org.apache.wayang.jdbc.operators.JdbcSourceOperator;
import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator;
import org.apache.wayang.jdbc.operators.JdbcTableSource;
import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate;

/**
Expand Down Expand Up @@ -105,6 +107,8 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin
jdbcExecutor.functionCompiler,
configuration
);
} else if (sourceOp instanceof JdbcTableSource tableSource) {
selectClause = selectClauseFor(tableSource);
} else {
selectClause = "*";
}
Expand Down Expand Up @@ -138,6 +142,29 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin
return sb;
}

/**
* Columns declared on the table source become the {@code SELECT} list.
* {@link JdbcTableSource#createSqlClause} stays the table name, because it is placed after {@code FROM}.
* Names that are not plain identifiers (for example {@code ST_AsText(geom)}) are schema hints, so the
* clause stays {@code *}.
*/
private static String selectClauseFor(JdbcTableSource tableOp) {
final String[] columnNames = tableOp.getColumnNames();
if (columnNames.length == 0 || !arePlainIdentifiers(columnNames)) {
return "*";
}
return String.join(", ", columnNames);
}

private static boolean arePlainIdentifiers(String[] columnNames) {
try {
SqlIdentifierUtils.validateColumnNames(columnNames);
return true;
} catch (IllegalArgumentException ex) {
return false;
}
}

/**
* Creates a query channel and the sql statement
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,52 @@ void testExecuteWithProjectionAndFilters() throws SQLException {
);
}

@Test
void testSelectsDeclaredColumns() {
JdbcTableSource tableSource = new HsqldbTableSource("customer", "name", "age");
StringBuilder query = JdbcExecutor.createSqlString(
mock(JdbcExecutor.class),
tableSource,
Collections.emptyList(),
null,
null,
null,
null,
Collections.emptyList());
assertEquals("SELECT name, age FROM customer", query.toString());
}

@Test
void testKeepsStarWhenColumnNameIsNotAnIdentifier() {
JdbcTableSource tableSource = new HsqldbTableSource("places", "ST_AsText(geom)");
StringBuilder query = JdbcExecutor.createSqlString(
mock(JdbcExecutor.class),
tableSource,
Collections.emptyList(),
null,
null,
null,
null,
Collections.emptyList());
assertEquals("SELECT * FROM places", query.toString());
}

@Test
void testProjectionOverridesSourceColumns() {
JdbcTableSource tableSource = new HsqldbTableSource("customer", "name", "age");
JdbcProjectionOperator projection = new HsqldbProjectionOperator("name");
StringBuilder query = JdbcExecutor.createSqlString(
mock(JdbcExecutor.class),
tableSource,
Collections.emptyList(),
projection,
null,
null,
null,
Collections.emptyList());
assertEquals("SELECT name FROM customer", query.toString());
}

@Test
void testExecuteWithNoTerminalConsumers() {
assertQueryWithTerminalConsumers(0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ void tableSource() {

assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE),
"TableSource should return all orders");
assertSqlReachedPresto("SELECT * FROM " + ORDERS);
assertSqlReachedPresto("SELECT order_id, customer_id, region, amount FROM " + ORDERS);
}

/** Filter: WHERE region = 'AMER' pushed to Presto. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ void tableSource() {

assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE),
"TableSource should return all orders");
assertSqlReachedTrino("SELECT * FROM " + ORDERS);
assertSqlReachedTrino("SELECT order_id, customer_id, region, amount FROM " + ORDERS);
}

/** Filter: WHERE region = 'AMER' pushed to Trino. */
Expand Down