From ceaa460aebd0835ab5dd21794a33e48e1e088973 Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Sat, 31 Oct 2026 01:34:26 +0530 Subject: [PATCH] Select declared table columns in JDBC scans. Column names on a table source were dropped when the operator was copied, and JDBC scans always used SELECT * because the table clause is only the FROM target. Keep the names and use them as the SELECT list when they are plain identifiers. --- .../converter/WayangTableScanVisitor.java | 46 +++++++++-------- .../api/sql/calcite/rel/WayangTableScan.java | 19 ++++++- .../wayang/basic/operators/TableSource.java | 37 +++++++++++++- .../basic/operators/TableSourceTest.java | 51 +++++++++++++++++++ .../wayang/jdbc/execution/JdbcExecutor.java | 26 +++++++++- .../jdbc/execution/JdbcExecutorTest.java | 46 +++++++++++++++++ .../wayang/presto/PrestoOperatorsIT.java | 2 +- .../apache/wayang/trino/TrinoOperatorsIT.java | 2 +- 8 files changed, 201 insertions(+), 28 deletions(-) create mode 100644 wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/TableSourceTest.java diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangTableScanVisitor.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangTableScanVisitor.java index e6ec4085c..bf7371f74 100755 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangTableScanVisitor.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangTableScanVisitor.java @@ -18,6 +18,7 @@ package org.apache.wayang.api.sql.calcite.converter; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.wayang.api.sql.calcite.rel.WayangTableScan; @@ -32,8 +33,8 @@ import org.apache.wayang.basic.data.Record; import org.apache.wayang.basic.operators.TableSource; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; //TODO: create tablesource with column types //TODO: support other sources @@ -64,25 +65,11 @@ Operator visit(final WayangTableScan wayangRelNode) { "Could not initialize calcite model parser from current Wayang configuration"); } - final List fieldTypes = wayangRelNode.getRowType().getFieldList().stream() - .map(RelDataTypeField::getType) - .map(type -> { - switch (type.getSqlTypeName()) { - case BOOLEAN: return CsvType.BOOLEAN; - case TINYINT: return CsvType.TINYINT; - case SMALLINT: return CsvType.SMALLINT; - case INTEGER: return CsvType.INTEGER; - case BIGINT: return CsvType.BIGINT; - case FLOAT: return CsvType.FLOAT; - case DOUBLE: return CsvType.DOUBLE; - case DECIMAL: return CsvType.decimal(type.getPrecision(), type.getScale()); - case DATE: return CsvType.DATE; - case TIME: return CsvType.TIME; - case TIMESTAMP: return CsvType.TIMESTAMP; - default: return CsvType.STRING; - } - }) - .collect(Collectors.toList()); + final List allFields = wayangRelNode.getRowType().getFieldList(); + final List fieldTypes = new ArrayList<>(wayangRelNode.getFields().length); + for (int fieldIndex : wayangRelNode.getFields()) { + fieldTypes.add(csvType(allFields.get(fieldIndex).getType())); + } final String url = String.format("file:/%s/%s.csv", modelParser.getFsPath(), wayangRelNode.getTableName()); @@ -92,7 +79,7 @@ Operator visit(final WayangTableScan wayangRelNode) { } else if (wayangRelNode.getTable().getQualifiedName().size() == 1) { // we assume that it is coming from a test environement or in memory db. - return new JdbcTableSource(wayangRelNode.getTable().getQualifiedName().get(0), wayangRelNode.getRowType().getFieldNames().toArray(String[]::new)) { + return new JdbcTableSource(wayangRelNode.getTable().getQualifiedName().get(0), wayangRelNode.getColumnNames().toArray(String[]::new)) { @Override public JdbcPlatformTemplate getPlatform() { @@ -103,4 +90,21 @@ public JdbcPlatformTemplate getPlatform() { throw new RuntimeException( "Source not supported, got: " + tableSource + ", expected either postgres or filesystem (fs)."); } + + private static CsvType csvType(RelDataType type) { + switch (type.getSqlTypeName()) { + case BOOLEAN: return CsvType.BOOLEAN; + case TINYINT: return CsvType.TINYINT; + case SMALLINT: return CsvType.SMALLINT; + case INTEGER: return CsvType.INTEGER; + case BIGINT: return CsvType.BIGINT; + case FLOAT: return CsvType.FLOAT; + case DOUBLE: return CsvType.DOUBLE; + case DECIMAL: return CsvType.decimal(type.getPrecision(), type.getScale()); + case DATE: return CsvType.DATE; + case TIME: return CsvType.TIME; + case TIMESTAMP: return CsvType.TIMESTAMP; + default: return CsvType.STRING; + } + } } diff --git a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/rel/WayangTableScan.java b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/rel/WayangTableScan.java index e002dbcb1..229feed74 100755 --- a/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/rel/WayangTableScan.java +++ b/wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/rel/WayangTableScan.java @@ -28,11 +28,14 @@ import org.apache.calcite.schema.Table; import org.apache.wayang.api.sql.calcite.convention.WayangConvention; +import java.util.ArrayList; import java.util.List; public class WayangTableScan extends TableScan implements WayangRel { - //TODO: fields are never queried, why? + /** + * Ordinals of the columns this scan reads. Defaults to every column of the table. + */ private final int[] fields; public WayangTableScan(RelOptCluster cluster, @@ -94,7 +97,19 @@ public String getTableName() { : table.getQualifiedName().get(1); } + /** + * @return the columns selected by {@link #fields}, in that order + */ public List getColumnNames() { - return table.getRowType().getFieldNames(); + final List names = table.getRowType().getFieldNames(); + final List projected = new ArrayList<>(this.fields.length); + for (int field : this.fields) { + projected.add(names.get(field)); + } + return projected; + } + + public int[] getFields() { + return this.fields.clone(); } } diff --git a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/TableSource.java b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/TableSource.java index 6c1a057f5..13372de3d 100644 --- a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/TableSource.java +++ b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/TableSource.java @@ -22,6 +22,7 @@ import org.apache.wayang.basic.types.RecordType; import org.apache.wayang.core.plan.wayangplan.UnarySource; import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; /** * {@link UnarySource} that provides the tuples from a database table. @@ -30,16 +31,31 @@ public class TableSource extends UnarySource { private final String tableName; + /** + * Columns this source reads. Empty when the caller did not name any columns. + */ + private final String[] columnNames; + public String getTableName() { return tableName; } + /** + * @return a copy of the column names, or an empty array when none were given + */ + public String[] getColumnNames() { + if (this.columnNames == null) { + return new String[0]; + } + return this.columnNames.clone(); + } + /** * Creates a new instance. * * @param tableName name of the table to be read - * @param columnNames names of the columns in the tables; can be omitted but allows to inject schema information - * into Wayang, so as to allow specific optimizations + * @param columnNames names of the columns to read; can be omitted. When they are plain SQL identifiers, + * JDBC scans select these columns instead of {@code *} */ public TableSource(String tableName, String... columnNames) { this(tableName, createOutputDataSetType(columnNames)); @@ -48,6 +64,22 @@ public TableSource(String tableName, String... columnNames) { public TableSource(String tableName, DataSetType type) { super(type); this.tableName = tableName; + this.columnNames = columnNamesFrom(type); + } + + private static String[] columnNamesFrom(DataSetType type) { + if (type == null) { + return new String[0]; + } + final DataUnitType dataUnitType = type.getDataUnitType(); + if (!(dataUnitType instanceof RecordType)) { + return new String[0]; + } + final String[] names = ((RecordType) dataUnitType).getFieldNames(); + if (names == null || names.length == 0) { + return new String[0]; + } + return names.clone(); } /** @@ -71,6 +103,7 @@ private static DataSetType createOutputDataSetType(String[] columnNames) public TableSource(TableSource that) { super(that); this.tableName = that.getTableName(); + this.columnNames = that.getColumnNames(); } } diff --git a/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/TableSourceTest.java b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/TableSourceTest.java new file mode 100644 index 000000000..eb1377f6f --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/TableSourceTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.basic.operators; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for {@link TableSource}. + */ +class TableSourceTest { + + @Test + void keepsColumnNames() { + TableSource source = new TableSource("customer", "name", "age"); + assertArrayEquals(new String[]{"name", "age"}, source.getColumnNames()); + } + + @Test + void copyKeepsColumnNames() { + TableSource source = new TableSource("customer", "name", "age"); + TableSource copy = new TableSource(source); + assertArrayEquals(new String[]{"name", "age"}, copy.getColumnNames()); + copy.getColumnNames()[0] = "changed"; + assertArrayEquals(new String[]{"name", "age"}, source.getColumnNames()); + } + + @Test + void omitsColumnNamesWhenUnknown() { + TableSource source = new TableSource("customer"); + assertEquals(0, source.getColumnNames().length); + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 05504569f..1a7211ed0 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -34,6 +34,7 @@ import org.apache.wayang.basic.operators.SpatialFilterOperator; import org.apache.wayang.basic.operators.SpatialJoinOperator; import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.basic.util.SqlIdentifierUtils; import org.apache.wayang.core.api.Job; import org.apache.wayang.core.api.exception.WayangException; import org.apache.wayang.core.optimizer.OptimizationContext; @@ -92,7 +93,7 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin jdbcExecutor.functionCompiler ); } else { - selectClause = "*"; + selectClause = selectClauseFor(tableOp); } final StringBuilder sb = new StringBuilder(1000); @@ -123,6 +124,29 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin return sb; } + /** + * Columns declared on the table source become the {@code SELECT} list. + * {@link JdbcTableSource#createSqlClause} stays the table name, because it is placed after {@code FROM}. + * Names that are not plain identifiers (for example {@code ST_AsText(geom)}) are schema hints, so the + * clause stays {@code *}. + */ + private static String selectClauseFor(JdbcTableSource tableOp) { + final String[] columnNames = tableOp.getColumnNames(); + if (columnNames.length == 0 || !arePlainIdentifiers(columnNames)) { + return "*"; + } + return String.join(", ", columnNames); + } + + private static boolean arePlainIdentifiers(String[] columnNames) { + try { + SqlIdentifierUtils.validateColumnNames(columnNames); + return true; + } catch (IllegalArgumentException ex) { + return false; + } + } + /** * Creates a query channel and the sql statement * diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index ca73e7663..c780863bc 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -246,6 +246,52 @@ void testExecuteWithProjectionAndFilters() throws SQLException { ); } + @Test + void testSelectsDeclaredColumns() { + JdbcTableSource tableSource = new HsqldbTableSource("customer", "name", "age"); + StringBuilder query = JdbcExecutor.createSqlString( + mock(JdbcExecutor.class), + tableSource, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList()); + assertEquals("SELECT name, age FROM customer", query.toString()); + } + + @Test + void testKeepsStarWhenColumnNameIsNotAnIdentifier() { + JdbcTableSource tableSource = new HsqldbTableSource("places", "ST_AsText(geom)"); + StringBuilder query = JdbcExecutor.createSqlString( + mock(JdbcExecutor.class), + tableSource, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList()); + assertEquals("SELECT * FROM places", query.toString()); + } + + @Test + void testProjectionOverridesSourceColumns() { + JdbcTableSource tableSource = new HsqldbTableSource("customer", "name", "age"); + JdbcProjectionOperator projection = new HsqldbProjectionOperator("name"); + StringBuilder query = JdbcExecutor.createSqlString( + mock(JdbcExecutor.class), + tableSource, + Collections.emptyList(), + projection, + null, + null, + null, + Collections.emptyList()); + assertEquals("SELECT name FROM customer", query.toString()); + } + @Test void testExecuteWithNoTerminalConsumers() { assertQueryWithTerminalConsumers(0); diff --git a/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoOperatorsIT.java b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoOperatorsIT.java index 2ab989a2a..956e92a05 100644 --- a/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoOperatorsIT.java +++ b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoOperatorsIT.java @@ -191,7 +191,7 @@ void tableSource() { assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), "TableSource should return all orders"); - assertSqlReachedPresto("SELECT * FROM " + ORDERS); + assertSqlReachedPresto("SELECT order_id, customer_id, region, amount FROM " + ORDERS); } /** Filter: WHERE region = 'AMER' pushed to Presto. */ diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java index fcdb2bdca..8de1da8f1 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java @@ -187,7 +187,7 @@ void tableSource() { assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), "TableSource should return all orders"); - assertSqlReachedTrino("SELECT * FROM " + ORDERS); + assertSqlReachedTrino("SELECT order_id, customer_id, region, amount FROM " + ORDERS); } /** Filter: WHERE region = 'AMER' pushed to Trino. */