diff --git a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java index c5df31937..00e2b50ff 100644 --- a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java +++ b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java @@ -167,9 +167,28 @@ public CardinalityEstimate estimate(OptimizationContext optimizationContext, Car CardinalityEstimate cardinalityEstimate = optimizationContext.queryJobCache(jobCacheKey, CardinalityEstimate.class); if (cardinalityEstimate != null) return cardinalityEstimate; + if (ParquetSource.this.metadata == null) { + ParquetSource.this.logger.warn("Could not inspect metadata of {}... deliver fallback estimate.", + ParquetSource.this.inputUrl); + timeMeasurement.stop(); + return this.FALLBACK_ESTIMATE; + } + // Otherwise calculate the cardinality. // First, inspect the size of the file and its line sizes. - OptionalLong fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl); + OptionalLong fileSize; + try { + fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl); + } catch (Exception e) { + ParquetSource.this.logger.warn( + "Could not determine size of {} ({}: {})... deliver fallback estimate.", + ParquetSource.this.inputUrl, + e.getClass().getSimpleName(), + e.getMessage() + ); + timeMeasurement.stop(); + return this.FALLBACK_ESTIMATE; + } if (fileSize.isEmpty()) { ParquetSource.this.logger.warn("Could not determine size of {}... deliver fallback estimate.", ParquetSource.this.inputUrl); diff --git a/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java new file mode 100644 index 000000000..6f233a243 --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.basic.operators; + +import org.apache.wayang.commons.util.profiledb.instrumentation.StopWatch; +import org.apache.wayang.commons.util.profiledb.model.Experiment; +import org.apache.wayang.commons.util.profiledb.model.Subject; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.Job; +import org.apache.wayang.core.optimizer.DefaultOptimizationContext; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test suite for {@link ParquetSource}. + */ +class ParquetSourceTest { + + @TempDir + Path tempDir; + + @Test + void cardinalityEstimationFallsBackWhenMetadataIsMissing() throws Exception { + Path parquetPath = this.tempDir.resolve("orders.parquet"); + Files.write(parquetPath, new byte[]{1, 2, 3, 4}); + + Job job = mock(Job.class); + DefaultOptimizationContext optimizationContext = mock(DefaultOptimizationContext.class); + when(job.getOptimizationContext()).thenReturn(optimizationContext); + when(optimizationContext.getJob()).thenReturn(job); + when(job.getStopWatch()).thenReturn(new StopWatch(new Experiment("mock", new Subject("mock", "mock")))); + when(optimizationContext.getConfiguration()).thenReturn(new Configuration()); + + ParquetSource source = new ParquetSource(parquetPath.toUri().toString(), null, "order_id"); + Optional estimator = source.createCardinalityEstimator( + 0, + optimizationContext.getConfiguration() + ); + + assertTrue(estimator.isPresent()); + CardinalityEstimate estimate = estimator.get().estimate(optimizationContext); + assertEquals(1000L, estimate.getLowerEstimate()); + assertEquals(100000000L, estimate.getUpperEstimate()); + } +} diff --git a/wayang-platforms/wayang-bigquery/pom.xml b/wayang-platforms/wayang-bigquery/pom.xml index 699da0b56..26cb7977f 100644 --- a/wayang-platforms/wayang-bigquery/pom.xml +++ b/wayang-platforms/wayang-bigquery/pom.xml @@ -39,7 +39,7 @@ - diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java index 715b4f1cd..10087fc4c 100644 --- a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java @@ -32,6 +32,7 @@ public class Mappings { new FilterMapping(), new GlobalReduceMapping(), new JoinMapping(), + new ParquetSourceMapping(), new ProjectionMapping(), new ReduceByMapping(), new SortMapping(), diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..1ff124b9c --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link BigQueryParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java new file mode 100644 index 000000000..9007080ca --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * BigQuery implementation for Parquet-backed external tables. + */ +public class BigQueryParquetSource extends JdbcParquetSource implements BigQueryExecutionOperator { + + public BigQueryParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public BigQueryParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties index c31477789..d26447217 100644 --- a/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties +++ b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties @@ -18,6 +18,18 @@ # JDBC driver (loaded via reflection; no compile-time dependency needed) wayang.bigquery.jdbc.driverName = com.google.cloud.bigquery.jdbc.BigQueryDriver +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a BigQuery-visible external table during execution. +# Format: parquet-uri=bigquery-relation;parquet-uri-2=bigquery-relation-2 +# wayang.bigquery.parquetsource.mappings = gs://bucket/orders/*.parquet=`project.dataset.orders` +# +# Optional auto-create support. Keep disabled unless the configured dataset is +# available and the URI is readable by BigQuery. +# wayang.bigquery.parquetsource.auto-create = false +# wayang.bigquery.parquetsource.auto-create.relation-prefix = project.dataset.wayang_parquet_ +# wayang.bigquery.parquetsource.auto-create.template = CREATE EXTERNAL TABLE IF NOT EXISTS ${relation} OPTIONS (format = 'PARQUET', uris = ['${uri}']) + # Connection URL and credentials are deployment-specific. # Set these in your wayang.properties or programmatically via Configuration. # diff --git a/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java new file mode 100644 index 000000000..e4a7c4ce4 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link BigQueryParquetSource}. + * + *

Set {@code -Dbigquery.parquet.source=`project.dataset.table`} or + * {@code BIGQUERY_PARQUET_SOURCE} for an existing external table. Alternatively, + * set {@code -Dbigquery.parquet.uri=gs://bucket/path/*.parquet} to let the test + * create a temporary BigQuery external table. + */ +class BigQueryParquetSourceIT { + + private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL", + "wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com"); + private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json"); + private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + private static final String DATASET = cfg("bigquery.dataset", "BIGQUERY_DATASET", "wayang_it"); + private static final String PARQUET_URI = cfg("bigquery.parquet.uri", "BIGQUERY_PARQUET_URI", ""); + private static final String CONFIGURED_SOURCE = cfg("bigquery.parquet.source", "BIGQUERY_PARQUET_SOURCE", ""); + private static final String SOURCE = CONFIGURED_SOURCE.isEmpty() ? tableName("parquet_source") : CONFIGURED_SOURCE; + private static final String SINK = cfg("bigquery.parquet.sink", "BIGQUERY_PARQUET_SINK", + tableName("parquet_source_copy")); + + private static final String JDBC_URL = String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION); + + private static boolean available = false; + private static long sourceCount = -1; + + @BeforeAll + static void setUp() { + if (CONFIGURED_SOURCE.isEmpty() && PARQUET_URI.isEmpty()) { + return; + } + try { + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS `" + PROJECT_ID + "." + DATASET + "` " + + "OPTIONS(location='" + LOCATION + "')"); + statement.execute("DROP TABLE IF EXISTS " + SINK); + if (!PARQUET_URI.isEmpty()) { + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + statement.execute("CREATE EXTERNAL TABLE " + SOURCE + + " OPTIONS (format = 'PARQUET', uris = ['" + PARQUET_URI + "'])"); + } + sourceCount = queryLong(statement, "SELECT count(*) FROM " + SOURCE); + available = true; + } + } catch (Exception e) { + System.err.println("[BigQueryParquetSourceIT] BigQuery Parquet source unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + if (!PARQUET_URI.isEmpty()) { + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + } + } catch (Exception e) { + System.err.println("[BigQueryParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsConfiguredParquetExternalTableIntoBigQuerySink() throws Exception { + Assumptions.assumeTrue(!CONFIGURED_SOURCE.isEmpty() || !PARQUET_URI.isEmpty(), + "No BigQuery Parquet source or URI configured"); + Assumptions.assumeTrue(available, "BigQuery Parquet source unavailable"); + + BigQueryParquetSource source = new BigQueryParquetSource(SOURCE, null); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK); + source.connectTo(0, sink, 0); + + wayangContext().execute("BQ-ParquetSource", new WayangPlan(sink)); + + assertEquals(sourceCount, queryLong("SELECT count(*) FROM " + SINK)); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + return new WayangContext(configuration).withPlugin(BigQuery.plugin()); + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) value = System.getenv(envVar); + return value == null || value.isEmpty() ? dflt : value; + } + + private static String tableName(String table) { + return "`" + PROJECT_ID + "." + DATASET + "." + table + "`"; + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + return queryLong(statement, sql); + } + } + + private static long queryLong(Statement statement, String sql) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index cd0268f3e..c5e19317f 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -18,14 +18,6 @@ package org.apache.wayang.jdbc.execution; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Set; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.wayang.basic.data.Tuple2; @@ -60,20 +52,60 @@ import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + /** * {@link Executor} implementation for the {@link JdbcPlatformTemplate}. */ public class JdbcExecutor extends ExecutorTemplate { - public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSourceOperator sourceOp, - final Collection filterTasks, final JdbcProjectionOperator projectionTask, final JdbcGlobalReduceOperator globalReduceTask, final JdbcReduceByOperator reduceByTask, final JdbcSortOperator sortTask, - final Collection joinTasks) { - return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, reduceByTask, - sortTask, joinTasks, null); + + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks) { + return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, + reduceByTask, sortTask, joinTasks, null, Map.of()); + } + + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks, + final Configuration configuration) { + final Map sourceNames = Map.of( + sourceOp.getSourceName(), + sourceOp.getSourceName(configuration) + ); + return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, + reduceByTask, sortTask, joinTasks, configuration, sourceNames); } - public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSourceOperator sourceOp, - final Collection filterTasks, final JdbcProjectionOperator projectionTask, final JdbcGlobalReduceOperator globalReduceTask, final JdbcReduceByOperator reduceByTask, final JdbcSortOperator sortTask, - final Collection joinTasks, final Configuration configuration) { + private static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks, + final Configuration configuration, + final Map sourceNames) { final String sourceName = sourceOp.createSqlClause( jdbcExecutor.connection, jdbcExecutor.functionCompiler, @@ -83,52 +115,55 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final Collection joins = joinTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) + .map(op -> op instanceof JdbcJoinOperator + ? ((JdbcJoinOperator) op).createSqlClause( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + sourceNames + ) + : op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); - + final String selectClause; - if (globalReduceTask != null) { - selectClause = globalReduceTask.createSqlClause( + if (globalReduceTask != null) { + selectClause = globalReduceTask.createSqlClause( jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration - ); - } else if (reduceByTask != null) { - selectClause = reduceByTask.createSqlClause( + ); + } else if (reduceByTask != null) { + selectClause = reduceByTask.createSqlClause( jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration - ); - } else if (projectionTask != null) { - selectClause = projectionTask.createSqlClause( + ); + } else if (projectionTask != null) { + selectClause = projectionTask.createSqlClause( jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration - ); - } else { - selectClause = "*"; - } + ); + } else { + selectClause = "*"; + } final StringBuilder sb = new StringBuilder(1000); sb.append("SELECT ").append(selectClause).append(" FROM ").append(sourceName); - if (!joins.isEmpty()) { - final String separator = " "; - for (final String join : joins) { - sb.append(separator).append(join); - } + for (final String join : joins) { + sb.append(" ").append(join); } if (!conditions.isEmpty()) { sb.append(" WHERE "); sb.append(String.join(" AND ", conditions)); } if (reduceByTask != null) { - sb.append(" GROUP BY " + reduceByTask.getKeyDescriptor().getSqlImplementation().getField0()); - } + sb.append(" GROUP BY ").append(reduceByTask.getKeyDescriptor().getSqlImplementation().getField0()); + } if (sortTask != null) { sb.append(sortTask.createSqlClause( - jdbcExecutor.connection, - jdbcExecutor.functionCompiler, - configuration + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration )); } @@ -139,18 +174,20 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin } /** - * Creates a query channel and the sql statement - * - * @param stage - * @param context - * @return a tuple containing the sql statement + * Creates a query channel and the SQL statement. + * + * @param stage the JDBC execution stage + * @param context provides optimization information and the Wayang configuration + * @param jdbcExecutor the executor with the database connection + * @return a tuple containing the SQL statement and its output channel */ protected static Tuple2 createSqlQuery(final ExecutionStage stage, - final OptimizationContext context, final JdbcExecutor jdbcExecutor) { + final OptimizationContext context, + final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); + JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, context.getConfiguration()); // Verify that we can handle this instance. - JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, context.getConfiguration()); final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage, context.getConfiguration()); assert startTask.getOperator() instanceof JdbcSourceOperator : "Invalid JDBC stage: Start task has to be a JDBC source"; @@ -177,18 +214,18 @@ protected static Tuple2 createSqlQuery(final E assert projectionTask == null; // Allow one projection operator per stage for now. projectionTask = (JdbcProjectionOperator) operator; } else if (operator instanceof final JdbcGlobalReduceOperator globalReduce) { - assert globalReduceTask == null; // Allow one projection operator per stage for now. + assert globalReduceTask == null; // Allow one global reduce operator per stage for now. globalReduceTask = globalReduce; } else if (operator instanceof final JdbcReduceByOperator reduceBy) { - assert reduceByTask == null; // Allow one projection operator per stage for now. + assert reduceByTask == null; // Allow one reduce-by operator per stage for now. reduceByTask = reduceBy; } else if (operator instanceof final JdbcSortOperator sort) { - assert sortTask == null; // Allow one projection operator per stage for now. + assert sortTask == null; // Allow one sort operator per stage for now. sortTask = sort; - } else if (operator instanceof JoinOperator || (operator instanceof SpatialJoinOperator)) { + } else if (operator instanceof JoinOperator || operator instanceof SpatialJoinOperator) { joinTasks.add((JdbcExecutionOperator) operator); } else { - throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask.toString())); + throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask)); } // Move the tipChannelInstance. @@ -201,20 +238,16 @@ protected static Tuple2 createSqlQuery(final E // Create the SQL query. final StringBuilder query = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration()); - + globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration(), + resolveSourceNames(startTasks, context.getConfiguration())); return new Tuple2<>(query.toString(), tipChannelInstance); } /** - * Selects the table source that belongs on the left-hand side of a JDBC join. + * Selects the source that belongs on the left-hand side of a JDBC join. * Stage start tasks are not ordered, but {@link JdbcJoinOperator#createSqlClause} - * assumes its first key descriptor's table is used in the {@code FROM} clause. + * assumes its first key descriptor's source is used in the {@code FROM} clause. */ - private static ExecutionTask selectStartTask(final Collection startTasks, final ExecutionStage stage) { - return selectStartTask(startTasks, stage, null); - } - private static ExecutionTask selectStartTask(final Collection startTasks, final ExecutionStage stage, final Configuration configuration) { @@ -225,13 +258,13 @@ private static ExecutionTask selectStartTask(final Collection startTasks, for (ExecutionTask task : stage.getAllTasks()) { if (task.getOperator() instanceof JdbcJoinOperator) { final JdbcJoinOperator joinOperator = (JdbcJoinOperator) task.getOperator(); - final String leftTableName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; + final String leftSourceName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; for (Object startTaskObject : startTasks) { final ExecutionTask startTask = (ExecutionTask) startTaskObject; if (startTask.getOperator() instanceof JdbcSourceOperator) { final JdbcSourceOperator sourceOperator = (JdbcSourceOperator) startTask.getOperator(); - if (sourceOperator.getSourceName().equals(leftTableName) - || sourceOperator.getSourceName(configuration).equals(leftTableName)) { + if (sourceOperator.getSourceName().equals(leftSourceName) + || sourceOperator.getSourceName(configuration).equals(leftSourceName)) { return startTask; } } @@ -239,7 +272,7 @@ private static ExecutionTask selectStartTask(final Collection startTasks, } } - throw new WayangException("Could not determine the left table source for JDBC stage."); + throw new WayangException("Could not determine the left source for JDBC stage."); } private static void prepareSourceTasks(final Collection startTasks, @@ -257,14 +290,30 @@ private static void prepareSourceTasks(final Collection startTasks, } } + private static Map resolveSourceNames(final Collection startTasks, + final Configuration configuration) { + final Map sourceNames = new HashMap<>(); + for (Object startTaskObject : startTasks) { + final ExecutionTask startTask = (ExecutionTask) startTaskObject; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + final JdbcSourceOperator sourceOperator = (JdbcSourceOperator) startTask.getOperator(); + sourceNames.put( + sourceOperator.getSourceName(), + sourceOperator.getSourceName(configuration) + ); + } + } + return sourceNames; + } + /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. * Composes a SQL query from the stage's operators and executes it directly on * the database connection, keeping all data within the database. * - * @param stage the execution stage ending with a sink + * @param stage the execution stage ending with a sink * @param optimizationContext provides optimization information - * @param jdbcExecutor the executor with the database connection + * @param jdbcExecutor the executor with the database connection */ private static long executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { @@ -284,7 +333,6 @@ private static long executeSinkStage(final ExecutionStage stage, final Optimizat assert termTask.getOperator() instanceof JdbcTableSinkOperator : "Invalid JDBC stage: Terminal task has to be a JdbcTableSinkOperator"; - // Extract operators from the stage final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); final JdbcTableSinkOperator sinkOp = (JdbcTableSinkOperator) termTask.getOperator(); final Collection filterTasks = new ArrayList<>(4); @@ -294,7 +342,6 @@ private static long executeSinkStage(final ExecutionStage stage, final Optimizat JdbcSortOperator sortTask = null; final Collection joinTasks = new ArrayList<>(); - // Walk through intermediate operators, stopping at the sink ExecutionTask nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(startTask, stage); while (nextTask != null && !(nextTask.getOperator() instanceof JdbcTableSinkOperator)) { if (nextTask.getOperator() instanceof final JdbcFilterOperator filterOperator) { @@ -314,26 +361,20 @@ private static long executeSinkStage(final ExecutionStage stage, final Optimizat } else if (nextTask.getOperator() instanceof final JdbcJoinOperator joinOperator) { joinTasks.add(joinOperator); } else { - throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask.toString())); + throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask)); } nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(nextTask, stage); } - // Compose the SELECT query final String selectSql = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration()).toString(); - - // Get the sink's SQL clause + globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration(), + resolveSourceNames(startTasks, optimizationContext.getConfiguration())).toString(); final String sinkClause = sinkOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); - // Execute on the database try (Statement stmt = jdbcExecutor.connection.createStatement()) { - // Handle overwrite: drop existing table first if ("overwrite".equals(sinkOp.getMode())) { stmt.execute("DROP TABLE IF EXISTS " + sinkOp.getQuotedTableName(jdbcExecutor.connection)); } - // Execute the composed query: CREATE TABLE x AS SELECT ... or INSERT INTO x - // SELECT ... final String fullSql = sinkClause + " " + selectSql + sinkOp.createSqlSuffix(); final long startTime = System.currentTimeMillis(); stmt.execute(fullSql); @@ -382,7 +423,7 @@ private Collection createExecutionLineageNodes( * @return the said follow-up {@link ExecutionTask} or {@code null} if none */ private static ExecutionTask findJdbcExecutionOperatorTaskInStage(final ExecutionTask task, - final ExecutionStage stage) { + final ExecutionStage stage) { if (task.getNumOuputChannels() != 1) { return null; } @@ -407,7 +448,8 @@ private static ExecutionTask findJdbcExecutionOperatorTaskInStage(final Executio * @return the {@link SqlQueryChannel.Instance} */ private static SqlQueryChannel.Instance instantiateOutboundChannel(final ExecutionTask task, - final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { + final OptimizationContext optimizationContext, + final JdbcExecutor jdbcExecutor) { assert task.getNumOuputChannels() == 1 : String.format("Illegal task: %s.", task); assert task.getOutputChannel(0) instanceof SqlQueryChannel : String.format("Illegal task: %s.", task); @@ -430,8 +472,9 @@ private static SqlQueryChannel.Instance instantiateOutboundChannel(final Executi * @return the {@link SqlQueryChannel.Instance} */ private static SqlQueryChannel.Instance instantiateOutboundChannel(final ExecutionTask task, - final OptimizationContext optimizationContext, final SqlQueryChannel.Instance predecessorChannelInstance, - final JdbcExecutor jdbcExecutor) { + final OptimizationContext optimizationContext, + final SqlQueryChannel.Instance predecessorChannelInstance, + final JdbcExecutor jdbcExecutor) { final SqlQueryChannel.Instance newInstance = JdbcExecutor.instantiateOutboundChannel(task, optimizationContext, jdbcExecutor); newInstance.getLineage().addPredecessor(predecessorChannelInstance.getLineage()); @@ -454,8 +497,7 @@ public JdbcExecutor(final JdbcPlatformTemplate platform, final Job job) { @Override public void execute(final ExecutionStage stage, final OptimizationContext optimizationContext, - final ExecutionState executionState) { - // Check if this stage ends with a sink operator + final ExecutionState executionState) { final Collection termTasks = stage.getTerminalTasks(); assert termTasks.size() == 1 : "Invalid JDBC stage: multiple terminal tasks are not currently supported."; final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; @@ -472,8 +514,6 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi } } } else { - // If it is normal stage: compose SQL and store in channel for downstream - // consumption final Tuple2 pair = JdbcExecutor.createSqlQuery(stage, optimizationContext, this); final String query = pair.field0; diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java index c3d631b66..21150de6c 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java @@ -20,15 +20,16 @@ import org.apache.wayang.core.util.Tuple; import org.apache.wayang.basic.data.Record; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.types.DataSetType; import org.apache.wayang.basic.operators.JoinOperator; import org.apache.wayang.core.function.TransformationDescriptor; import org.apache.wayang.jdbc.compiler.FunctionCompiler; -import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator; import org.apache.wayang.core.optimizer.costs.LoadProfileEstimators; import java.sql.Connection; +import java.util.Map; import java.util.Optional; /** @@ -64,17 +65,17 @@ public JdbcJoinOperator(JoinOperator that) { @Override public String createSqlClause(Connection connection, FunctionCompiler compiler) { - return this.createSqlClause(connection, compiler, null); + return this.createSqlClause(connection, compiler, Map.of()); } - @Override - public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + public String createSqlClause(Connection connection, + FunctionCompiler compiler, + Map sourceNames) { final Tuple left = this.keyDescriptor0.getSqlImplementation(); final Tuple right = this.keyDescriptor1.getSqlImplementation(); - final String platformId = this.getPlatform().getPlatformId(); - final String leftTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, left.field0); + final String leftTableName = sourceNames.getOrDefault(left.field0, left.field0); final String leftKeys = left.field1; - final String rightTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, right.field0); + final String rightTableName = sourceNames.getOrDefault(right.field0, right.field0); final String rightKeys = right.field1; if (leftKeys.contains(",") && rightKeys.contains(",")) { diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java index 4bea068df..4a8353ee4 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java @@ -19,9 +19,6 @@ package org.apache.wayang.jdbc.operators; import org.apache.logging.log4j.LogManager; -import org.apache.parquet.schema.LogicalTypeAnnotation; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; import org.apache.wayang.basic.operators.ParquetSource; import org.apache.wayang.commons.util.profiledb.model.measurement.TimeMeasurement; import org.apache.wayang.core.api.Configuration; @@ -29,6 +26,9 @@ import org.apache.wayang.core.optimizer.OptimizationContext; import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index ca73e7663..c3b7b1af7 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -29,6 +29,7 @@ import org.apache.wayang.core.profiling.NoInstrumentationStrategy; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.operators.JdbcFilterOperator; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; import org.apache.wayang.jdbc.operators.JdbcTableSource; import org.apache.wayang.jdbc.operators.SqlToStreamOperator; @@ -38,11 +39,15 @@ import org.apache.wayang.jdbc.test.HsqldbTableSource; import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Collections; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -51,6 +56,120 @@ */ class JdbcExecutorTest { + @Test + void testCreateSqlStringResolvesConfiguredParquetSourceMapping() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/orders=orders_external" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/orders", null, "id", "name"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + StringBuilder query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ); + + assertEquals("SELECT * FROM orders_external", query.toString()); + } + + @Test + void testCreateSqlStringAutoCreatesConfiguredParquetSourceRelation() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/auto-orders=auto_orders_external" + ); + configuration.setProperty("wayang.hsqldb.parquetsource.auto-create", "true"); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.template", + "CREATE TABLE IF NOT EXISTS ${relation} (id INTEGER)" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/auto-orders", null, "id"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + StringBuilder query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ); + + assertEquals("SELECT * FROM auto_orders_external", query.toString()); + try (Connection connection = HsqldbPlatform.getInstance() + .createDatabaseDescriptor(configuration) + .createJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM auto_orders_external")) { + resultSet.next(); + assertEquals(0, resultSet.getLong(1)); + } + } + + @Test + void testCreateSqlStringAutoCreatesGeneratedParquetSourceRelation() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.hsqldb.parquetsource.auto-create", "true"); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.relation-prefix", + "auto_parquet_" + ); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.template", + "CREATE TABLE IF NOT EXISTS ${relation} (id INTEGER)" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/generated-orders", null, "id"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + String query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ).toString(); + + assertTrue(query.startsWith("SELECT * FROM auto_parquet_")); + String relationName = query.substring("SELECT * FROM ".length()); + try (Connection connection = HsqldbPlatform.getInstance() + .createDatabaseDescriptor(configuration) + .createJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM " + relationName)) { + resultSet.next(); + assertEquals(0, resultSet.getLong(1)); + } + } + @Test void testExecuteWithPlainTableSource() throws SQLException { Configuration configuration = new Configuration(); @@ -303,4 +422,16 @@ public boolean isReusable() { executor.dispose(); } } + + private static class HsqldbParquetSource extends JdbcParquetSource { + + HsqldbParquetSource(String inputUrl, String[] projection, String... columnNames) { + super(inputUrl, projection, columnNames); + } + + @Override + public HsqldbPlatform getPlatform() { + return HsqldbPlatform.getInstance(); + } + } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index d56405b19..cea2c11aa 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -66,9 +66,81 @@ void initDB() throws SQLException { statement.execute("DROP TABLE IF EXISTS shipments"); } } + + @Test + void testParquetSourceJoinResolvesConfiguredSourceMappings() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/testA=testA;file:///warehouse/testB=testB" + ); + + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + SqlQueryChannel.Descriptor sqlChannelDescriptor = HsqldbPlatform.getInstance().getSqlQueryChannelDescriptor(); + + ExecutionStage sqlStage = mock(ExecutionStage.class); + + HsqldbParquetSource sourceA = new HsqldbParquetSource("file:///warehouse/testA", null, "a", "b"); + HsqldbParquetSource sourceB = new HsqldbParquetSource("file:///warehouse/testB", null, "a", "b"); + + ExecutionTask sourceATask = new ExecutionTask(sourceA); + sourceATask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, sourceA.getOutput(0))); + sourceATask.setStage(sqlStage); + + ExecutionTask sourceBTask = new ExecutionTask(sourceB); + sourceBTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, sourceB.getOutput(0))); + sourceBTask.setStage(sqlStage); + + final ExecutionOperator joinOperator = new HsqldbJoinOperator( + new TransformationDescriptor( + (record) -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation("file:///warehouse/testA", "a"), + new TransformationDescriptor( + (record) -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation("file:///warehouse/testB", "a") + ); + + ExecutionTask joinTask = new ExecutionTask(joinOperator); + sourceATask.getOutputChannel(0).addConsumer(joinTask, 0); + sourceBTask.getOutputChannel(0).addConsumer(joinTask, 1); + joinTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, joinOperator.getOutput(0))); + joinTask.setStage(sqlStage); + + when(sqlStage.getStartTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList(sourceBTask, sourceATask))); + when(sqlStage.getAllTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList(sourceBTask, sourceATask, joinTask))); + when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(joinTask)); + + ExecutionStage nextStage = mock(ExecutionStage.class); + + SqlToStreamOperator sqlToStreamOperator = new SqlToStreamOperator(HsqldbPlatform.getInstance()); + ExecutionTask sqlToStreamTask = new ExecutionTask(sqlToStreamOperator); + joinTask.getOutputChannel(0).addConsumer(sqlToStreamTask, 0); + sqlToStreamTask.setStage(nextStage); + + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + + SqlQueryChannel.Instance sqlQueryChannelInstance = + (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); + + assertEquals( + "SELECT * FROM testA JOIN testB ON testB.a=testA.a", + sqlQueryChannelInstance.getSqlQuery() + ); + } @Test void testWithHsqldb() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "testA=wrongParquetRelationA;testB=wrongParquetRelationB" + ); Job job = mock(Job.class); when(job.getConfiguration()).thenReturn(configuration); @@ -259,4 +331,16 @@ void testMultiConditionJoinWithHsqldb() throws SQLException { } } + private static class HsqldbParquetSource extends JdbcParquetSource { + + HsqldbParquetSource(String inputUrl, String[] projection, String... columnNames) { + super(inputUrl, projection, columnNames); + } + + @Override + public HsqldbPlatform getPlatform() { + return HsqldbPlatform.getInstance(); + } + } + } diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java index 9c8e0f42b..c72ad8433 100644 --- a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java @@ -32,6 +32,7 @@ public class Mappings { new FilterMapping(), new GlobalReduceMapping(), new JoinMapping(), + new ParquetSourceMapping(), new ProjectionMapping(), new ReduceByMapping(), new SortMapping(), diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..de25e7bab --- /dev/null +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.apache.wayang.presto.platform.PrestoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link PrestoParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + PrestoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new PrestoParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java new file mode 100644 index 000000000..77c0c72e6 --- /dev/null +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * Presto implementation for Parquet-backed SQL relations. + */ +public class PrestoParquetSource extends JdbcParquetSource implements PrestoExecutionOperator { + + public PrestoParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public PrestoParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties b/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties index 3c63f1621..18c55e6f3 100644 --- a/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties +++ b/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties @@ -21,6 +21,18 @@ # wayang.presto.jdbc.password = (optional; omit when no password is required) wayang.presto.jdbc.driverName = com.facebook.presto.jdbc.PrestoDriver +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a Presto-visible SQL relation during execution. +# Format: parquet-uri=presto-relation;parquet-uri-2=presto-relation-2 +# wayang.presto.parquetsource.mappings = file:///data/orders=hive.profile.orders +# +# Optional auto-create support. Keep disabled unless the configured catalog +# supports the selected DDL syntax. +# wayang.presto.parquetsource.auto-create = false +# wayang.presto.parquetsource.auto-create.relation-prefix = hive.profile.wayang_parquet_ +# wayang.presto.parquetsource.auto-create.template = CREATE TABLE IF NOT EXISTS ${relation} (${columns}) WITH (format = 'PARQUET', external_location = '${uri}') + # Hardware profile used by LoadProfileToTimeConverter. # Presto runs as a distributed MPP cluster; model a representative coordinator node. wayang.presto.cpu.mhz = 2700 diff --git a/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java new file mode 100644 index 000000000..e45d14bb8 --- /dev/null +++ b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link PrestoParquetSource}. + * + *

Set {@code -Dpresto.parquet.source=catalog.schema.table} or + * {@code PRESTO_PARQUET_SOURCE} to point at an existing Parquet-backed table. + */ +class PrestoParquetSourceIT { + + private static final String HOST = System.getenv().getOrDefault("PRESTO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("PRESTO_PORT", "8081")); + private static final String USER = System.getenv().getOrDefault("PRESTO_USER", "test"); + private static final String JDBC_URL = cfg("presto.jdbc.url", "PRESTO_JDBC_URL", + String.format("jdbc:presto://%s:%d/hive", HOST, PORT)); + private static final String SOURCE = cfg("presto.parquet.source", "PRESTO_PARQUET_SOURCE", ""); + private static final String SINK = cfg("presto.parquet.sink", "PRESTO_PARQUET_SINK", + defaultSinkName(SOURCE, "wayang_parquet_source_copy")); + + private static boolean available = false; + private static long sourceCount = -1; + + @BeforeAll + static void setUp() { + if (SOURCE.isEmpty()) { + return; + } + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + sourceCount = queryLong(statement, "SELECT count(*) FROM " + SOURCE); + statement.execute("DROP TABLE IF EXISTS " + SINK); + available = true; + } catch (Exception e) { + System.err.println("[PrestoParquetSourceIT] Presto Parquet source unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + } catch (Exception e) { + System.err.println("[PrestoParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsConfiguredParquetBackedRelationIntoPrestoSink() throws Exception { + Assumptions.assumeFalse(SOURCE.isEmpty(), "No Presto Parquet source configured"); + Assumptions.assumeTrue(available, "Presto Parquet source unavailable"); + + PrestoParquetSource source = new PrestoParquetSource(SOURCE, null); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK); + source.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(sourceCount, queryLong("SELECT count(*) FROM " + SINK)); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.presto.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.presto.jdbc.user", USER); + return new WayangContext(configuration).withPlugin(Presto.plugin()); + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) value = System.getenv(envVar); + return value == null || value.isEmpty() ? dflt : value; + } + + private static String defaultSinkName(String source, String tableName) { + int lastDot = source.lastIndexOf('.'); + if (lastDot < 0) { + return tableName; + } + return source.substring(0, lastDot + 1) + tableName; + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement()) { + return queryLong(statement, sql); + } + } + + private static long queryLong(Statement statement, String sql) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static Connection jdbc() throws Exception { + Properties properties = new Properties(); + properties.setProperty("user", USER); + return DriverManager.getConnection(JDBC_URL, properties); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java index 2912ed7ed..5f26b8405 100644 --- a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java @@ -32,6 +32,7 @@ public class Mappings { new FilterMapping(), new GlobalReduceMapping(), new JoinMapping(), + new ParquetSourceMapping(), new ProjectionMapping(), new ReduceByMapping(), new SortMapping(), diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..add76f800 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link TrinoParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new TrinoParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java new file mode 100644 index 000000000..ab7f277d2 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * Trino implementation for Parquet-backed SQL relations. + */ +public class TrinoParquetSource extends JdbcParquetSource implements TrinoExecutionOperator { + + public TrinoParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public TrinoParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties index 4c1785e7f..24d64a7cc 100644 --- a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties +++ b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties @@ -21,6 +21,18 @@ # wayang.trino.jdbc.password = wayang.trino.jdbc.driverName = io.trino.jdbc.TrinoDriver +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a Trino-visible SQL relation during execution. +# Format: parquet-uri=trino-relation;parquet-uri-2=trino-relation-2 +# wayang.trino.parquetsource.mappings = file:///data/orders=iceberg.profile.orders +# +# Optional auto-create support. Keep disabled unless the configured catalog +# supports the selected DDL syntax. +# wayang.trino.parquetsource.auto-create = false +# wayang.trino.parquetsource.auto-create.relation-prefix = iceberg.profile.wayang_parquet_ +# wayang.trino.parquetsource.auto-create.template = CREATE TABLE IF NOT EXISTS ${relation} (${columns}) WITH (format = 'PARQUET', external_location = '${uri}') + # Hardware profile used by LoadProfileToTimeConverter. # Trino runs as a distributed cluster; model a representative coordinator node. wayang.trino.cpu.mhz = 2700 diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java new file mode 100644 index 000000000..46455cf83 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link TrinoParquetSource}. + */ +class TrinoParquetSourceIT { + + private static final String HOST = System.getenv().getOrDefault("TRINO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("TRINO_PORT", "8080")); + private static final String USER = System.getenv().getOrDefault("TRINO_USER", "admin"); + private static final String JDBC_URL = String.format("jdbc:trino://%s:%d", HOST, PORT); + + private static final String SCHEMA = "iceberg.wayang_parquet_it"; + private static final String SOURCE = SCHEMA + ".orders_parquet"; + private static final String SINK = SCHEMA + ".orders_parquet_copy"; + private static final String[] COLUMNS = {"order_id", "region", "amount"}; + + private static boolean available = false; + + @BeforeAll + static void setUp() { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + statement.execute("DROP TABLE IF EXISTS " + SINK); + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + statement.execute("CREATE TABLE " + SOURCE + " WITH (format = 'PARQUET') AS " + + "SELECT * FROM (VALUES " + + "(CAST(1 AS BIGINT), 'AMER', CAST(10.0 AS DOUBLE)), " + + "(CAST(2 AS BIGINT), 'EMEA', CAST(20.0 AS DOUBLE)), " + + "(CAST(3 AS BIGINT), 'APAC', CAST(30.0 AS DOUBLE))" + + ") AS t(order_id, region, amount)"); + available = true; + } catch (Exception e) { + System.err.println("[TrinoParquetSourceIT] Trino Parquet setup unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + } catch (Exception e) { + System.err.println("[TrinoParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsParquetBackedRelationIntoTrinoSink() throws Exception { + Assumptions.assumeTrue(available, "Trino Parquet setup unavailable"); + + TrinoParquetSource source = new TrinoParquetSource(SOURCE, null, COLUMNS); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK, COLUMNS); + source.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK)); + assertEquals(60.0, queryDouble("SELECT sum(amount) FROM " + SINK), 0.01); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.trino.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.trino.jdbc.user", USER); + configuration.setProperty("wayang.trino.jdbc.password", ""); + return new WayangContext(configuration).withPlugin(Trino.plugin()); + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static double queryDouble(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getDouble(1); + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(JDBC_URL, USER, ""); + } +} diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml index 6b87cda96..a89586026 100644 --- a/wayang-tests-integration/pom.xml +++ b/wayang-tests-integration/pom.xml @@ -55,6 +55,24 @@ ${project.version} test + + org.apache.wayang + wayang-trino + 1.1.2-SNAPSHOT + test + + + org.apache.wayang + wayang-presto + 1.1.2-SNAPSHOT + test + + + org.apache.wayang + wayang-bigquery + 1.1.2-SNAPSHOT + test + org.apache.wayang wayang-giraph diff --git a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java new file mode 100644 index 000000000..7bb68c832 --- /dev/null +++ b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.tests; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.BigQuery; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.presto.Presto; +import org.apache.wayang.presto.operators.PrestoProjectionOperator; +import org.apache.wayang.presto.platform.PrestoPlatform; +import org.apache.wayang.trino.Trino; +import org.apache.wayang.trino.operators.TrinoProjectionOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Properties; +import java.util.function.BiFunction; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Real-service smoke test for Parquet-backed SQL join profiling. + * + *

The test runs on a single target SQL platform at a time. Logical + * {@link ParquetSource}s are mapped to relations that the selected platform can + * read from Parquet. + */ +class SqlParquetJoinProfilingIT { + + private static final String TARGET_PLATFORM = cfg( + "wayang.profile.parquet.target", + "WAYANG_PROFILE_PARQUET_TARGET", + "trino" + ).toLowerCase(Locale.ROOT); + private static final String ORDERS_URI = cfg( + "wayang.profile.parquet.orders.uri", + "WAYANG_PROFILE_PARQUET_ORDERS_URI", + "" + ); + private static final String CUSTOMERS_URI = cfg( + "wayang.profile.parquet.customers.uri", + "WAYANG_PROFILE_PARQUET_CUSTOMERS_URI", + "" + ); + private static final long EXPECTED_ROWS = Long.parseLong(cfg( + "wayang.profile.parquet.expectedRows", + "WAYANG_PROFILE_PARQUET_EXPECTED_ROWS", + "-1" + )); + private static final String JOIN_FLATTEN_NAME = "SQL Parquet profile join flatten"; + private static final String[] ORDERS_COLUMNS = {"order_id", "customer_id", "region", "amount"}; + private static final String[] CUSTOMERS_COLUMNS = {"cust_id", "tier"}; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "cust_id", "tier" + }; + private static final Path OUTPUT_DIR = Paths.get(cfg( + "wayang.profile.parquet.outputDir", + "WAYANG_PROFILE_PARQUET_OUTPUT_DIR", + "target/cost-profiling/parquet-sql" + )); + private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); + private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); + private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); + + private String sinkRelation; + + @AfterEach + void tearDown() throws Exception { + if (this.sinkRelation != null && isCleanupEnabled()) { + dropRelation(TARGET_PLATFORM, this.sinkRelation); + } + } + + @Test + void runsParquetJoinProfilingOnTargetSqlPlatform() throws Exception { + Assumptions.assumeTrue(isEnabled(), "Set wayang.profile.parquet.enabled=true to run this test."); + Assumptions.assumeFalse(ORDERS_URI.isEmpty(), "Configure wayang.profile.parquet.orders.uri."); + Assumptions.assumeFalse(CUSTOMERS_URI.isEmpty(), "Configure wayang.profile.parquet.customers.uri."); + Assumptions.assumeTrue(isSupportedTarget(TARGET_PLATFORM), "Unsupported target platform: " + TARGET_PLATFORM); + + this.sinkRelation = relation(TARGET_PLATFORM, "sink"); + Assumptions.assumeFalse(this.sinkRelation.isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".sink.relation."); + Assumptions.assumeFalse(relation(TARGET_PLATFORM, "orders").isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".orders.relation."); + Assumptions.assumeFalse(relation(TARGET_PLATFORM, "customers").isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".customers.relation."); + Assumptions.assumeTrue(isPlatformAvailable(TARGET_PLATFORM), + "Target platform is not reachable: " + TARGET_PLATFORM); + + Files.createDirectories(OUTPUT_DIR); + writeManifestHeaderIfNeeded(); + dropRelation(TARGET_PLATFORM, this.sinkRelation); + + Platform targetPlatform = targetPlatform(); + WayangPlan plan = createParquetJoinPlan(this.sinkRelation, targetPlatform); + long startedAt = System.currentTimeMillis(); + wayangContext(targetPlatform).execute("SQL-Parquet-Join-Profiling-" + TARGET_PLATFORM, plan); + long elapsedMillis = System.currentTimeMillis() - startedAt; + + long actualRows = queryLong(TARGET_PLATFORM, "SELECT count(*) FROM " + this.sinkRelation); + if (EXPECTED_ROWS >= 0) { + assertEquals(EXPECTED_ROWS, actualRows); + } + appendManifest(TARGET_PLATFORM, actualRows, elapsedMillis, this.sinkRelation); + } + + private static WayangPlan createParquetJoinPlan(String sinkRelation, Platform targetPlatform) { + ParquetSource orders = new ParquetSource(ORDERS_URI, null, ORDERS_COLUMNS); + ParquetSource customers = new ParquetSource(CUSTOMERS_URI, null, CUSTOMERS_COLUMNS); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(ORDERS_URI, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_URI, "cust_id") + ); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkRelation, JOIN_COLUMNS); + + orders.addTargetPlatform(targetPlatform); + customers.addTargetPlatform(targetPlatform); + join.addTargetPlatform(targetPlatform); + flatten.addTargetPlatform(targetPlatform); + sink.addTargetPlatform(targetPlatform); + + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + return new WayangPlan(sink); + } + + private static WayangContext wayangContext(Platform targetPlatform) { + Configuration configuration = new Configuration(); + configureJdbc(configuration, TARGET_PLATFORM); + configureParquetMappings(configuration, TARGET_PLATFORM); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.core.explain.enabled", "false"); + configuration.setProperty("wayang.core.log.executions", EXECUTIONS_PATH.toString().replace('\\', '/')); + configuration.setProperty("wayang.core.log.cardinalities", CARDINALITIES_PATH.toString().replace('\\', '/')); + configuration.getMappingProvider().addAllToWhitelist(Collections.singleton(new JoinFlattenMapping(targetPlatform))); + + WayangContext context = new WayangContext(configuration); + switch (TARGET_PLATFORM) { + case "trino": + return context.withPlugin(Trino.plugin()); + case "presto": + return context.withPlugin(Presto.plugin()); + case "bigquery": + return context.withPlugin(BigQuery.plugin()); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static void configureJdbc(Configuration configuration, String platform) { + switch (platform) { + case "trino": + configuration.setProperty("wayang.trino.jdbc.url", cfg( + "wayang.trino.jdbc.url", + "WAYANG_TRINO_JDBC_URL", + "jdbc:trino://localhost:8080" + )); + configuration.setProperty("wayang.trino.jdbc.user", cfg( + "wayang.trino.jdbc.user", + "WAYANG_TRINO_JDBC_USER", + "admin" + )); + configuration.setProperty("wayang.trino.jdbc.password", cfg( + "wayang.trino.jdbc.password", + "WAYANG_TRINO_JDBC_PASSWORD", + "" + )); + return; + case "presto": + configuration.setProperty("wayang.presto.jdbc.url", cfg( + "wayang.presto.jdbc.url", + "WAYANG_PRESTO_JDBC_URL", + "jdbc:presto://localhost:8081/hive" + )); + configuration.setProperty("wayang.presto.jdbc.user", cfg( + "wayang.presto.jdbc.user", + "WAYANG_PRESTO_JDBC_USER", + "test" + )); + return; + case "bigquery": + configuration.setProperty("wayang.bigquery.jdbc.url", bigQueryJdbcUrl()); + return; + default: + throw new IllegalArgumentException("Unsupported target platform: " + platform); + } + } + + private static void configureParquetMappings(Configuration configuration, String platform) { + String ordersRelation = relation(platform, "orders"); + String customersRelation = relation(platform, "customers"); + if (!ordersRelation.isEmpty() && !customersRelation.isEmpty()) { + configuration.setProperty( + String.format("wayang.%s.parquetsource.mappings", platform), + ORDERS_URI + "=" + ordersRelation + ";" + CUSTOMERS_URI + "=" + customersRelation + ); + } + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class) + ), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class) + ); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private static Platform targetPlatform() { + switch (TARGET_PLATFORM) { + case "trino": + return TrinoPlatform.getInstance(); + case "presto": + return PrestoPlatform.getInstance(); + case "bigquery": + return BigQueryPlatform.getInstance(); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static boolean isSupportedTarget(String platform) { + return "trino".equals(platform) || "presto".equals(platform) || "bigquery".equals(platform); + } + + private static boolean isPlatformAvailable(String platform) { + try (Connection connection = jdbc(platform); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT 1")) { + return resultSet.next(); + } catch (Exception e) { + return false; + } + } + + private static long queryLong(String platform, String sql) throws Exception { + try (Connection connection = jdbc(platform); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static void dropRelation(String platform, String relation) throws Exception { + try (Connection connection = jdbc(platform); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + relation); + } + } + + private static Connection jdbc(String platform) throws Exception { + switch (platform) { + case "trino": + return DriverManager.getConnection( + cfg("wayang.trino.jdbc.url", "WAYANG_TRINO_JDBC_URL", "jdbc:trino://localhost:8080"), + cfg("wayang.trino.jdbc.user", "WAYANG_TRINO_JDBC_USER", "admin"), + cfg("wayang.trino.jdbc.password", "WAYANG_TRINO_JDBC_PASSWORD", "") + ); + case "presto": + Properties prestoProperties = new Properties(); + prestoProperties.setProperty( + "user", + cfg("wayang.presto.jdbc.user", "WAYANG_PRESTO_JDBC_USER", "test") + ); + return DriverManager.getConnection( + cfg("wayang.presto.jdbc.url", "WAYANG_PRESTO_JDBC_URL", "jdbc:presto://localhost:8081/hive"), + prestoProperties + ); + case "bigquery": + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + return DriverManager.getConnection(bigQueryJdbcUrl()); + default: + throw new IllegalArgumentException("Unsupported platform: " + platform); + } + } + + private static String bigQueryJdbcUrl() { + String configuredUrl = cfg("wayang.bigquery.jdbc.url", "WAYANG_BIGQUERY_JDBC_URL", ""); + if (!configuredUrl.isEmpty()) { + return configuredUrl; + } + String projectId = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + String serviceAccount = cfg( + "bigquery.saEmail", + "BIGQUERY_SA_EMAIL", + "wayang-bq@" + projectId + ".iam.gserviceaccount.com" + ); + String keyPath = cfg( + "bigquery.keyPath", + "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json" + ); + String location = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + return String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + projectId, + serviceAccount, + keyPath, + location + ); + } + + private static String relation(String platform, String relationRole) { + return cfg( + String.format("wayang.profile.parquet.%s.%s.relation", platform, relationRole), + String.format( + "WAYANG_PROFILE_PARQUET_%s_%s_RELATION", + platform.toUpperCase(Locale.ROOT), + relationRole.toUpperCase(Locale.ROOT) + ), + "" + ); + } + + private static boolean isEnabled() { + return Boolean.parseBoolean(cfg( + "wayang.profile.parquet.enabled", + "WAYANG_PROFILE_PARQUET_ENABLED", + "false" + )); + } + + private static boolean isCleanupEnabled() { + return Boolean.parseBoolean(cfg( + "wayang.profile.parquet.cleanup", + "WAYANG_PROFILE_PARQUET_CLEANUP", + "true" + )); + } + + private static void writeManifestHeaderIfNeeded() throws Exception { + if (Files.exists(MANIFEST_PATH)) { + return; + } + try (BufferedWriter writer = Files.newBufferedWriter(MANIFEST_PATH, StandardCharsets.UTF_8)) { + writer.write("target_platform,orders_uri,customers_uri,actual_rows,expected_rows,elapsed_millis,sink_relation"); + writer.newLine(); + } + } + + private static void appendManifest(String platform, + long actualRows, + long elapsedMillis, + String sinkRelation) throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter( + MANIFEST_PATH, + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND)) { + writer.write(String.join(",", + platform, + ORDERS_URI, + CUSTOMERS_URI, + String.valueOf(actualRows), + String.valueOf(EXPECTED_ROWS), + String.valueOf(elapsedMillis), + sinkRelation)); + writer.newLine(); + } + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) { + value = System.getenv(envVar); + } + return value == null || value.isEmpty() ? dflt : value; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0), + right.getField(1) + ); + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + private final Platform targetPlatform; + + private JoinFlattenMapping(Platform targetPlatform) { + this.targetPlatform = targetPlatform; + } + + @Override + public Collection getTransformations() { + return Collections.singleton(createTransformation(this.targetPlatform, this::createProjectionOperator)); + } + + private org.apache.wayang.core.plan.wayangplan.Operator createProjectionOperator( + MapOperator matchedOperator, + Integer epoch) { + switch (TARGET_PLATFORM) { + case "trino": + return new TrinoProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + case "presto": + return new PrestoProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + case "bigquery": + return new BigQueryProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static PlanTransformation createTransformation( + Platform platform, + BiFunction factoryFunction) { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false + ).withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + factoryFunction::apply + ); + + return new PlanTransformation(SubplanPattern.createSingleton(pattern), factory, platform); + } + + private static MapOperator, Record> createProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + java.util.Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class) + ); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class) + ); + projection.setName(JOIN_FLATTEN_NAME); + return projection; + } + } +} diff --git a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java new file mode 100644 index 000000000..7fa42e40e --- /dev/null +++ b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.tests; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.bigquery.BigQuery; +import org.apache.wayang.bigquery.operators.BigQueryJoinOperator; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.plan.wayangplan.OperatorAlternative; +import org.apache.wayang.core.plan.wayangplan.PlanTraversal; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.util.WayangCollections; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; +import org.apache.wayang.presto.Presto; +import org.apache.wayang.presto.operators.PrestoJoinOperator; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.apache.wayang.trino.Trino; +import org.apache.wayang.trino.operators.TrinoJoinOperator; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Regression coverage for Parquet-backed SQL sources when several SQL platforms + * are registered in the same {@link WayangContext}. + */ +class SqlParquetMultiPlatformIT { + + private static final String ORDERS_PARQUET = "s3://wayang-profiler/orders.parquet"; + private static final String CUSTOMERS_PARQUET = "s3://wayang-profiler/customers.parquet"; + + @Test + void registersParquetJoinAlternativesForAllSqlPlatforms() { + WayangContext context = new WayangContext() + .withPlugin(Trino.plugin()) + .withPlugin(Presto.plugin()) + .withPlugin(BigQuery.plugin()); + Configuration configuration = context.getConfiguration(); + + WayangPlan plan = createParquetJoinPlan(); + plan.prepare(); + plan.applyTransformations(gatherTransformations(configuration)); + + Collection alternatives = PlanTraversal.upstream() + .traverse(plan.getSinks()) + .getTraversedNodesWith(operator -> operator instanceof OperatorAlternative) + .stream() + .map(operator -> (OperatorAlternative) operator) + .collect(Collectors.toList()); + + List sourceAlternatives = alternatives.stream() + .filter(alternative -> containsAlternative(alternative, ParquetSource.class)) + .collect(Collectors.toList()); + List joinAlternatives = alternatives.stream() + .filter(alternative -> containsAlternative(alternative, JoinOperator.class)) + .collect(Collectors.toList()); + + assertEquals(2, sourceAlternatives.size()); + sourceAlternatives.forEach(alternative -> { + assertTrue(containsAlternative(alternative, TrinoParquetSource.class)); + assertTrue(containsAlternative(alternative, PrestoParquetSource.class)); + assertTrue(containsAlternative(alternative, BigQueryParquetSource.class)); + }); + + assertEquals(1, joinAlternatives.size()); + OperatorAlternative joinAlternative = joinAlternatives.get(0); + assertTrue(containsAlternative(joinAlternative, TrinoJoinOperator.class)); + assertTrue(containsAlternative(joinAlternative, PrestoJoinOperator.class)); + assertTrue(containsAlternative(joinAlternative, BigQueryJoinOperator.class)); + } + + @Test + void resolvesSameParquetUriToPlatformSpecificRelations() { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.trino.parquetsource.mappings", + ORDERS_PARQUET + "=iceberg.profiler.orders" + ); + configuration.setProperty( + "wayang.presto.parquetsource.mappings", + ORDERS_PARQUET + "=hive.profiler.orders" + ); + configuration.setProperty( + "wayang.bigquery.parquetsource.mappings", + ORDERS_PARQUET + "=`project.profiler.orders_ext`" + ); + + assertEquals( + "iceberg.profiler.orders", + JdbcParquetSource.resolveSourceName(configuration, "trino", ORDERS_PARQUET) + ); + assertEquals( + "hive.profiler.orders", + JdbcParquetSource.resolveSourceName(configuration, "presto", ORDERS_PARQUET) + ); + assertEquals( + "`project.profiler.orders_ext`", + JdbcParquetSource.resolveSourceName(configuration, "bigquery", ORDERS_PARQUET) + ); + } + + private static WayangPlan createParquetJoinPlan() { + ParquetSource orders = new ParquetSource( + ORDERS_PARQUET, + null, + "order_id", + "customer_id", + "amount" + ); + ParquetSource customers = new ParquetSource( + CUSTOMERS_PARQUET, + null, + "customer_id", + "region" + ); + + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor( + record -> (Integer) record.getField(1), + Record.class, + Integer.class + ).withSqlImplementation(ORDERS_PARQUET, "customer_id"), + new TransformationDescriptor( + record -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation(CUSTOMERS_PARQUET, "customer_id") + ); + + @SuppressWarnings({"rawtypes", "unchecked"}) + DataSetType> joinOutputType = + (DataSetType) DataSetType.createDefaultUnchecked(Tuple2.class); + List> collector = new ArrayList<>(); + LocalCallbackSink> sink = + LocalCallbackSink.createCollectingSink(collector, joinOutputType); + + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, sink, 0); + + return new WayangPlan(sink); + } + + private static Collection gatherTransformations(Configuration configuration) { + Set platforms = WayangCollections.asSet(configuration.getPlatformProvider().provideAll()); + return configuration.getMappingProvider().provideAll().stream() + .flatMap((Mapping mapping) -> mapping.getTransformations().stream()) + .filter(transformation -> transformation.getTargetPlatforms().isEmpty() + || platforms.containsAll(transformation.getTargetPlatforms())) + .collect(Collectors.toList()); + } + + private static boolean containsAlternative(OperatorAlternative alternative, + Class operatorClass) { + return alternative.getAlternatives().stream() + .map(OperatorAlternative.Alternative::getContainedOperator) + .anyMatch(operatorClass::isInstance); + } +}