From ae9c1b88aec55179a5c7855f66d6722b8bc3c451 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 12 Aug 2026 17:08:38 +0800 Subject: [PATCH 1/4] Add SQL Parquet source support --- pom.xml | 2 +- .../wayang/api/sql/SqlToWayangRelTest.java | 2 +- .../wayang/basic/operators/ParquetSource.java | 21 +- .../basic/operators/ParquetSourceTest.java | 72 +++ wayang-platforms/pom.xml | 3 +- wayang-platforms/wayang-bigquery/pom.xml | 90 +++ .../org/apache/wayang/bigquery/BigQuery.java | 63 ++ .../bigquery/channels/ChannelConversions.java | 54 ++ .../bigquery/mapping/FilterMapping.java | 63 ++ .../bigquery/mapping/GlobalReduceMapping.java | 60 ++ .../wayang/bigquery/mapping/JoinMapping.java | 76 +++ .../wayang/bigquery/mapping/Mappings.java | 42 ++ .../mapping/ParquetSourceMapping.java | 59 ++ .../bigquery/mapping/ProjectionMapping.java | 66 ++ .../bigquery/mapping/ReduceByMapping.java | 63 ++ .../wayang/bigquery/mapping/SortMapping.java | 62 ++ .../bigquery/mapping/TableSinkMapping.java | 58 ++ .../operators/BigQueryExecutionOperator.java | 31 + .../operators/BigQueryFilterOperator.java | 52 ++ .../BigQueryGlobalReduceOperator.java | 50 ++ .../operators/BigQueryJoinOperator.java | 49 ++ .../operators/BigQueryParquetSource.java | 36 ++ .../operators/BigQueryProjectionOperator.java | 49 ++ .../operators/BigQueryReduceByOperator.java | 52 ++ .../operators/BigQuerySortOperator.java | 49 ++ .../operators/BigQueryTableSinkOperator.java | 48 ++ .../operators/BigQueryTableSource.java | 58 ++ .../bigquery/platform/BigQueryPlatform.java | 62 ++ .../plugin/BigQueryConversionsPlugin.java | 58 ++ .../bigquery/plugin/BigQueryPlugin.java | 58 ++ .../wayang-bigquery-defaults.properties | 198 ++++++ .../wayang/bigquery/BigQueryOperatorsIT.java | 599 ++++++++++++++++++ .../bigquery/BigQueryParquetSourceIT.java | 154 +++++ .../wayang/jdbc/execution/JdbcExecutor.java | 344 +++++----- .../jdbc/operators/JdbcExecutionOperator.java | 8 + .../jdbc/operators/JdbcJoinOperator.java | 10 +- .../jdbc/operators/JdbcParquetSource.java | 332 ++++++++++ .../jdbc/operators/JdbcSourceOperator.java | 66 ++ .../jdbc/operators/JdbcTableSource.java | 9 +- .../jdbc/execution/JdbcExecutorTest.java | 139 +++- .../JdbcGlobalReduceOperatorTest.java | 2 +- .../jdbc/operators/JdbcJoinOperatorTest.java | 84 ++- .../operators/JdbcReduceByOperatorTest.java | 2 +- .../jdbc/operators/JdbcSortOperatorTest.java | 2 +- .../wayang/presto/mapping/Mappings.java | 1 + .../presto/mapping/ParquetSourceMapping.java | 59 ++ .../presto/operators/PrestoParquetSource.java | 36 ++ .../wayang-presto-defaults.properties | 26 +- .../wayang/presto/PrestoParquetSourceIT.java | 138 ++++ .../apache/wayang/trino/mapping/Mappings.java | 1 + .../trino/mapping/ParquetSourceMapping.java | 59 ++ .../trino/operators/TrinoParquetSource.java | 36 ++ .../wayang-trino-defaults.properties | 28 +- .../wayang/trino/TrinoParquetSourceIT.java | 129 ++++ wayang-tests-integration/pom.xml | 18 + .../tests/SqlParquetJoinProfilingIT.java | 525 +++++++++++++++ .../tests/SqlParquetMultiPlatformIT.java | 195 ++++++ 57 files changed, 4476 insertions(+), 232 deletions(-) create mode 100644 wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java create mode 100644 wayang-platforms/wayang-bigquery/pom.xml create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java create mode 100644 wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties create mode 100644 wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java create mode 100644 wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java create mode 100644 wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java create mode 100644 wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java create mode 100644 wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java create mode 100644 wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java create mode 100644 wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java create mode 100644 wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java create mode 100644 wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java create mode 100644 wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java diff --git a/pom.xml b/pom.xml index f0d5cf77e..b01d2f93e 100644 --- a/pom.xml +++ b/pom.xml @@ -1075,7 +1075,7 @@ - ASF 2.0 | The Apache Software License, Version 2.0 | Apache License, Version 2.0 | Apache 2.0 License | Apache License Version 2.0 | Apache 2.0 | Apache-2.0 | The Apache License, Version 2.0 | Apache License Version 2 | Apache 2 | http://www.apache.org/licenses/LICENSE-2.0.txt | Apache License 2.0 | Apache Software License - Version 2.0 | Apache License, version 2.0 | Apache License v2.0 + ASF 2.0 | The Apache Software License, Version 2.0 | Apache License, Version 2.0 | Apache License V2.0 | Apache 2.0 License | Apache License Version 2.0 | Apache 2.0 | Apache-2.0 | The Apache License, Version 2.0 | Apache License Version 2 | Apache 2 | http://www.apache.org/licenses/LICENSE-2.0.txt | Apache License 2.0 | Apache Software License - Version 2.0 | Apache License, version 2.0 | Apache License v2.0 BSD 3-claus | 3-Clause BSD License | BSD 3 Clause License | BSD 3 Clause | BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) | BSD licence | BSD | New BSD License | Revised BSD | The BSD 3-Clause License | The BSD License | The New BSD License | New BSD license | BSD 3-clause | BSD 3-Clause | BSD-3-Clause | BSD New license | Go License MIT | MIT License | The MIT License | The MIT License (MIT) | MIT license BSD 2-claus | BSD 2-Clause License | BSD 2-Clause | BSD-2-Clause diff --git a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java index 77537826d..91c1d7aef 100755 --- a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java +++ b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java @@ -311,7 +311,7 @@ public RelDataType getRowType(final RelDataTypeFactory typeFactory) { final StringBuilder query = JdbcExecutor.createSqlString(jdbcExecutor, table, Arrays.asList(), projection, null, null, null, Arrays.asList()); - assertEquals("SELECT ID, NAME FROM T1;", query.toString()); + assertEquals("SELECT ID, NAME FROM T1", query.toString()); } @Test diff --git a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java index c5df31937..00e2b50ff 100644 --- a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java +++ b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSource.java @@ -167,9 +167,28 @@ public CardinalityEstimate estimate(OptimizationContext optimizationContext, Car CardinalityEstimate cardinalityEstimate = optimizationContext.queryJobCache(jobCacheKey, CardinalityEstimate.class); if (cardinalityEstimate != null) return cardinalityEstimate; + if (ParquetSource.this.metadata == null) { + ParquetSource.this.logger.warn("Could not inspect metadata of {}... deliver fallback estimate.", + ParquetSource.this.inputUrl); + timeMeasurement.stop(); + return this.FALLBACK_ESTIMATE; + } + // Otherwise calculate the cardinality. // First, inspect the size of the file and its line sizes. - OptionalLong fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl); + OptionalLong fileSize; + try { + fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl); + } catch (Exception e) { + ParquetSource.this.logger.warn( + "Could not determine size of {} ({}: {})... deliver fallback estimate.", + ParquetSource.this.inputUrl, + e.getClass().getSimpleName(), + e.getMessage() + ); + timeMeasurement.stop(); + return this.FALLBACK_ESTIMATE; + } if (fileSize.isEmpty()) { ParquetSource.this.logger.warn("Could not determine size of {}... deliver fallback estimate.", ParquetSource.this.inputUrl); diff --git a/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java new file mode 100644 index 000000000..6f233a243 --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ParquetSourceTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.basic.operators; + +import org.apache.wayang.commons.util.profiledb.instrumentation.StopWatch; +import org.apache.wayang.commons.util.profiledb.model.Experiment; +import org.apache.wayang.commons.util.profiledb.model.Subject; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.Job; +import org.apache.wayang.core.optimizer.DefaultOptimizationContext; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test suite for {@link ParquetSource}. + */ +class ParquetSourceTest { + + @TempDir + Path tempDir; + + @Test + void cardinalityEstimationFallsBackWhenMetadataIsMissing() throws Exception { + Path parquetPath = this.tempDir.resolve("orders.parquet"); + Files.write(parquetPath, new byte[]{1, 2, 3, 4}); + + Job job = mock(Job.class); + DefaultOptimizationContext optimizationContext = mock(DefaultOptimizationContext.class); + when(job.getOptimizationContext()).thenReturn(optimizationContext); + when(optimizationContext.getJob()).thenReturn(job); + when(job.getStopWatch()).thenReturn(new StopWatch(new Experiment("mock", new Subject("mock", "mock")))); + when(optimizationContext.getConfiguration()).thenReturn(new Configuration()); + + ParquetSource source = new ParquetSource(parquetPath.toUri().toString(), null, "order_id"); + Optional estimator = source.createCardinalityEstimator( + 0, + optimizationContext.getConfiguration() + ); + + assertTrue(estimator.isPresent()); + CardinalityEstimate estimate = estimator.get().estimate(optimizationContext); + assertEquals(1000L, estimate.getLowerEstimate()); + assertEquals(100000000L, estimate.getUpperEstimate()); + } +} diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml index 06b34ee77..51db92326 100644 --- a/wayang-platforms/pom.xml +++ b/wayang-platforms/pom.xml @@ -43,8 +43,9 @@ wayang-giraph wayang-flink wayang-generic-jdbc - wayang-presto wayang-trino + wayang-presto + wayang-bigquery wayang-tensorflow diff --git a/wayang-platforms/wayang-bigquery/pom.xml b/wayang-platforms/wayang-bigquery/pom.xml new file mode 100644 index 000000000..8c9f57b53 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/pom.xml @@ -0,0 +1,90 @@ + + + + 4.0.0 + + + wayang-platforms + org.apache.wayang + 1.1.2-SNAPSHOT + + + wayang-bigquery + + Wayang Platform BigQuery + + Wayang implementation of the operators to be working with the platform "BigQuery" + + + + org.apache.wayang.platform.bigquery + 0.6.0 + + + + + + com.google.cloud + google-cloud-bigquery-jdbc + ${bigquery-jdbc.version} + all + + + org.apache.wayang + wayang-basic + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-jdbc-template + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-spark + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-api-scala-java + 1.1.2-SNAPSHOT + test + + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + + diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java new file mode 100644 index 000000000..c07b2138e --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.bigquery.plugin.BigQueryConversionsPlugin; +import org.apache.wayang.bigquery.plugin.BigQueryPlugin; + +/** + * Register for relevant components of this module. + */ +public class BigQuery { + + private final static BigQueryPlugin PLUGIN = new BigQueryPlugin(); + + private final static BigQueryConversionsPlugin CONVERSIONS_PLUGIN = new BigQueryConversionsPlugin(); + + /** + * Retrieve the {@link BigQueryPlugin}. + * + * @return the {@link BigQueryPlugin} + */ + public static BigQueryPlugin plugin() { + return PLUGIN; + } + + /** + * Retrieve the {@link BigQueryConversionsPlugin}. + * + * @return the {@link BigQueryConversionsPlugin} + */ + public static BigQueryConversionsPlugin conversionPlugin() { + return CONVERSIONS_PLUGIN; + } + + + /** + * Retrieve the {@link BigQueryPlatform}. + * + * @return the {@link BigQueryPlatform} + */ + public static BigQueryPlatform platform() { + return BigQueryPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java new file mode 100644 index 000000000..7079d22ef --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.channels; + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.optimizer.channels.DefaultChannelConversion; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.jdbc.operators.SqlToRddOperator; +import org.apache.wayang.jdbc.operators.SqlToStreamOperator; +import org.apache.wayang.spark.channels.RddChannel; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link ChannelConversion}s supported for this platform. + */ +public class ChannelConversions { + + public static final ChannelConversion SQL_TO_STREAM_CONVERSION = new DefaultChannelConversion( + BigQueryPlatform.getInstance().getSqlQueryChannelDescriptor(), + StreamChannel.DESCRIPTOR, + () -> new SqlToStreamOperator(BigQueryPlatform.getInstance()) + ); + + public static final ChannelConversion SQL_TO_UNCACHED_RDD_CONVERSION = new DefaultChannelConversion( + BigQueryPlatform.getInstance().getSqlQueryChannelDescriptor(), + RddChannel.UNCACHED_DESCRIPTOR, + () -> new SqlToRddOperator(BigQueryPlatform.getInstance()) + ); + + public static final Collection ALL = Arrays.asList( + SQL_TO_STREAM_CONVERSION, + SQL_TO_UNCACHED_RDD_CONVERSION + ); + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java new file mode 100644 index 000000000..e109cb920 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.bigquery.operators.BigQueryFilterOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + + +/** + * Mapping from {@link FilterOperator} to {@link BigQueryFilterOperator}. + */ +@SuppressWarnings("unchecked") +public class FilterMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "filter", new FilterOperator<>(null, DataSetType.createDefault(Record.class)), false + ).withAdditionalTest(op -> op.getPredicateDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryFilterOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java new file mode 100644 index 000000000..4b20ff344 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.bigquery.operators.BigQueryGlobalReduceOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link GlobalReduceOperator} to {@link BigQueryGlobalReduceOperator}. + */ +@SuppressWarnings("unchecked") +public class GlobalReduceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduce", new GlobalReduceOperator(null, DataSetType.createDefault(Record.class)), false) + .withAdditionalTest(op -> op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryGlobalReduceOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java new file mode 100644 index 000000000..8db353600 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.bigquery.operators.BigQueryJoinOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link JoinOperator} to {@link BigQueryJoinOperator}. + */ +@SuppressWarnings("unchecked") +public class JoinMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "join", + new JoinOperator( + null, + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ), + false + ) + .withAdditionalTest(op -> op.getKeyDescriptor0() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor1() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor0().getSqlImplementation() != null) + .withAdditionalTest(op -> op.getKeyDescriptor1().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> { + return new BigQueryJoinOperator(matchedOperator).at(epoch); + } + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java new file mode 100644 index 000000000..10087fc4c --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.core.mapping.Mapping; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link Mapping}s supported for this platform. + */ +public class Mappings { + + public static final Collection ALL = Arrays.asList( + new FilterMapping(), + new GlobalReduceMapping(), + new JoinMapping(), + new ParquetSourceMapping(), + new ProjectionMapping(), + new ReduceByMapping(), + new SortMapping(), + new TableSinkMapping() + ); + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..1ff124b9c --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link BigQueryParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java new file mode 100644 index 000000000..2c26e3a4b --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link MapOperator} to {@link BigQueryProjectionOperator}. + */ +public class ProjectionMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "projection", + new MapOperator<>( + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getFunctionDescriptor() instanceof ProjectionDescriptor) + .withAdditionalTest(op -> op.getNumInputs() == 1); // No broadcasts. + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryProjectionOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java new file mode 100644 index 000000000..a20c9b3cc --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.bigquery.operators.BigQueryReduceByOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ReduceByOperator} to {@link BigQueryReduceByOperator}. + */ +@SuppressWarnings("unchecked") +public class ReduceByMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduceBy", + new ReduceByOperator(null, null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null + && op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryReduceByOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java new file mode 100644 index 000000000..e9f7a13e8 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.bigquery.operators.BigQuerySortOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link SortOperator} to {@link BigQuerySortOperator}. + */ +@SuppressWarnings("unchecked") +public class SortMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "sort", + new SortOperator(null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQuerySortOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java new file mode 100644 index 000000000..aafaed0c8 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.operators.BigQueryTableSinkOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link TableSink} to {@link BigQueryTableSinkOperator}. + */ +@SuppressWarnings("unchecked") +public class TableSinkMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern<>( + "sink", new TableSink<>(null, null, null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryTableSinkOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java new file mode 100644 index 000000000..a496042fb --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; + +public interface BigQueryExecutionOperator extends JdbcExecutionOperator { + + @Override + default BigQueryPlatform getPlatform() { + return BigQueryPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java new file mode 100644 index 000000000..ee246c93d --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.operators.JdbcFilterOperator; + + +/** + * BigQuery implementation of the {@link FilterOperator}. + */ +public class BigQueryFilterOperator extends JdbcFilterOperator implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + */ + public BigQueryFilterOperator(PredicateDescriptor predicateDescriptor) { + super(predicateDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryFilterOperator(FilterOperator that) { + super(that); + } + + @Override + protected BigQueryFilterOperator createCopy() { + return new BigQueryFilterOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java new file mode 100644 index 000000000..b6b115e10 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.jdbc.operators.JdbcGlobalReduceOperator; + +/** + * BigQuery implementation of the {@link GlobalReduceOperator}. The reduction is + * pushed down as a SQL aggregate (e.g. {@code SUM(amount)}) via its + * {@code sqlImplementation}. + */ +public class BigQueryGlobalReduceOperator extends JdbcGlobalReduceOperator implements BigQueryExecutionOperator { + + public BigQueryGlobalReduceOperator(ReduceDescriptor reduceDescriptor) { + super(reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryGlobalReduceOperator(GlobalReduceOperator that) { + super(that); + } + + @Override + protected BigQueryGlobalReduceOperator createCopy() { + return new BigQueryGlobalReduceOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java new file mode 100644 index 000000000..40d444c43 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcJoinOperator; + + +/** + * BigQuery implementation of the {@link JoinOperator}. + */ +public class BigQueryJoinOperator extends JdbcJoinOperator implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + */ + public BigQueryJoinOperator( + TransformationDescriptor keyDescriptor0, + TransformationDescriptor keyDescriptor1) { + super(keyDescriptor0,keyDescriptor1); + } + + public BigQueryJoinOperator(JoinOperator that) { + super(that); + } + + @Override + protected BigQueryJoinOperator createCopy() { + return new BigQueryJoinOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java new file mode 100644 index 000000000..9007080ca --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * BigQuery implementation for Parquet-backed external tables. + */ +public class BigQueryParquetSource extends JdbcParquetSource implements BigQueryExecutionOperator { + + public BigQueryParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public BigQueryParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java new file mode 100644 index 000000000..6cd0b538e --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; + +/** + * BigQuery implementation of the {@link FilterOperator}. + */ +public class BigQueryProjectionOperator extends JdbcProjectionOperator implements BigQueryExecutionOperator { + + public BigQueryProjectionOperator(String... fieldNames) { + super(fieldNames); + } + + public BigQueryProjectionOperator(ProjectionDescriptor functionDescriptor) { + super(functionDescriptor); + } + + public BigQueryProjectionOperator(MapOperator that) { + super(that); + } + + @Override + protected BigQueryProjectionOperator createCopy() { + return new BigQueryProjectionOperator(this); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java new file mode 100644 index 000000000..cacf9dcaa --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; + +/** + * BigQuery implementation of the {@link ReduceByOperator}. The grouping key and + * the reduction are pushed down as a SQL {@code GROUP BY} plus aggregate (e.g. + * {@code SELECT region, SUM(amount) ... GROUP BY region}). + */ +public class BigQueryReduceByOperator extends JdbcReduceByOperator implements BigQueryExecutionOperator { + + public BigQueryReduceByOperator(TransformationDescriptor keyDescriptor, + ReduceDescriptor reduceDescriptor) { + super(keyDescriptor, reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryReduceByOperator(ReduceByOperator that) { + super(that); + } + + @Override + protected BigQueryReduceByOperator createCopy() { + return new BigQueryReduceByOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java new file mode 100644 index 000000000..2aea82627 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcSortOperator; + +/** + * BigQuery implementation of the {@link SortOperator}. The sort key and direction + * are pushed down as a SQL {@code ORDER BY} clause via its {@code sqlImplementation}. + */ +public class BigQuerySortOperator extends JdbcSortOperator implements BigQueryExecutionOperator { + + public BigQuerySortOperator(TransformationDescriptor keyDescriptor) { + super(keyDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQuerySortOperator(SortOperator that) { + super(that); + } + + @Override + protected BigQuerySortOperator createCopy() { + return new BigQuerySortOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java new file mode 100644 index 000000000..c7c065013 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; + +/** + * BigQuery implementation of the {@link JdbcTableSinkOperator}. The sink stays + * entirely within BigQuery: the composed query is wrapped in a + * {@code CREATE TABLE ... AS} (mode {@code overwrite}) or {@code INSERT INTO ...} + * statement. + * + *

Table names follow BigQuery's backtick-quoted convention + * {@code `project.dataset.table`}. + */ +public class BigQueryTableSinkOperator extends JdbcTableSinkOperator implements BigQueryExecutionOperator { + + public BigQueryTableSinkOperator(String tableName, String[] columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryTableSinkOperator(TableSink that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java new file mode 100644 index 000000000..2d71d3746 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.operators.JdbcTableSource; + +import java.util.List; + +/** + * BigQuery implementation for the {@link TableSource}. + * + *

Table names must be backtick-quoted and fully qualified: + * {@code `project.dataset.table`}. Pass the backtick-quoted name as the + * {@code tableName} constructor argument. + */ +public class BigQueryTableSource extends JdbcTableSource implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + * + * @see TableSource#TableSource(String, String...) + */ + public BigQueryTableSource(String tableName, String... columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryTableSource(JdbcTableSource that) { + super(that); + } + + @Override + public List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("This operator has no input channels."); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java new file mode 100644 index 000000000..8ab7c036d --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.platform; + +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; + +/** + * {@link Platform} implementation for BigQuery. + * + *

BigQuery JDBC URL format: + *

+ *   jdbc:bigquery://https://www.googleapis.com/bigquery/v2;
+ *     ProjectId=my-project;
+ *     OAuthType=0;
+ *     OAuthServiceAcctEmail=sa@my-project.iam.gserviceaccount.com;
+ *     OAuthPvtKeyPath=/path/to/key.json
+ * 
+ * + *

Table names must be backtick-quoted: {@code `project.dataset.table`}. + */ +public class BigQueryPlatform extends JdbcPlatformTemplate { + + private static final String PLATFORM_NAME = "BigQuery"; + + private static final String CONFIG_NAME = "bigquery"; + + private static BigQueryPlatform instance = null; + + public static BigQueryPlatform getInstance() { + if (instance == null) { + instance = new BigQueryPlatform(); + } + return instance; + } + + protected BigQueryPlatform() { + super(PLATFORM_NAME, CONFIG_NAME); + } + + @Override + public String getJdbcDriverClassName() { + return "com.google.cloud.bigquery.jdbc.BigQueryDriver"; + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java new file mode 100644 index 000000000..d828489ad --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.plugin; + +import org.apache.wayang.bigquery.channels.ChannelConversions; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link BigQueryPlatform}. + */ +public class BigQueryConversionsPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(BigQueryPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Collections.emptyList(); + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java new file mode 100644 index 000000000..cf4dc3863 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.plugin; + +import org.apache.wayang.bigquery.channels.ChannelConversions; +import org.apache.wayang.bigquery.mapping.Mappings; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link BigQueryPlatform}. + */ +public class BigQueryPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(BigQueryPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Mappings.ALL; + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties new file mode 100644 index 000000000..7ebd3b4cb --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties @@ -0,0 +1,198 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JDBC driver (loaded via reflection; no compile-time dependency needed) +wayang.bigquery.jdbc.driverName = com.google.cloud.bigquery.jdbc.BigQueryDriver + +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a BigQuery-visible external table during execution. +# Format: parquet-uri=bigquery-relation;parquet-uri-2=bigquery-relation-2 +# wayang.bigquery.parquetsource.mappings = gs://bucket/orders/*.parquet=`project.dataset.orders` +# +# Optional auto-create support. Keep disabled unless the configured dataset is +# available and the URI is readable by BigQuery. +# wayang.bigquery.parquetsource.auto-create = false +# wayang.bigquery.parquetsource.auto-create.relation-prefix = project.dataset.wayang_parquet_ +# wayang.bigquery.parquetsource.auto-create.template = CREATE EXTERNAL TABLE IF NOT EXISTS ${relation} OPTIONS (format = 'PARQUET', uris = ['${uri}']) + +# Connection URL and credentials are deployment-specific. +# Set these in your wayang.properties or programmatically via Configuration. +# +# Example: +# wayang.bigquery.jdbc.url = jdbc:bigquery://https://www.googleapis.com/bigquery/v2;\ +# ProjectId=my-project;\ +# OAuthType=0;\ +# OAuthServiceAcctEmail=sa@my-project.iam.gserviceaccount.com;\ +# OAuthPvtKeyPath=/path/to/key.json +# +# wayang.bigquery.jdbc.url = (required; set per deployment) +# wayang.bigquery.jdbc.user = (optional) +# wayang.bigquery.jdbc.password = (optional) + +# Hardware profile. +# BigQuery is serverless and runs on Google's shared compute. +# Model enough cores for full parallelism; latency is dominated by network +# and query dispatch rather than raw CPU. +wayang.bigquery.cpu.mhz = 2700 +wayang.bigquery.cores = 8 +wayang.bigquery.costs.fix = 0.0 +wayang.bigquery.costs.per-ms = 1.0 + +# Cost model. +# +# Formula: cpu = alpha * rows + beta +# +# BigQuery is a serverless, massively parallel columnar engine. +# Per-row cost (alpha) is very low because scans run across thousands of slots +# in parallel. Fixed startup (beta) is high due to query dispatch, planning, +# billing overhead, and result serialisation back over the network. +# +# Compared to single-node sources: +# alpha = 5 - massively parallel; per-row overhead about 10x lower than Postgres +# beta = 2000000 - serverless dispatch + billing + network round-trip +# +# Optimizer crossover points: +# BigQuery vs Postgres: 5n+2M < 55n+380k => n > ~32k rows +# BigQuery vs Trino: 5n+2M < 10n+800k => n > ~240k rows +# +# These are initial estimates. Tune by running the .load.template variants +# and fitting measured data. +wayang.bigquery.tablesource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.tablesource.load = {\ + "in":0, "out":1,\ + "cpu":"${5*out0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.filter.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.filter.load = {\ + "in":1, "out":1,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.projection.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.projection.load = {\ + "in":1, "out":1,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.join.load.template = {\ + "type":"mathex", "in":2, "out":1,\ + "cpu":"?*in0 + ?*in1 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.join.load = {\ + "in":2, "out":1,\ + "cpu":"${5*in0 + 5*in1 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.globalreduce.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.globalreduce.load = {\ + "in":1, "out":1,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.reduceby.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.reduceby.load = {\ + "in":1, "out":1,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.sort.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.sort.load = {\ + "in":1, "out":1,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.tablesink.load.template = {\ + "type":"mathex", "in":1, "out":0,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.tablesink.load = {\ + "in":1, "out":0,\ + "cpu":"${5*in0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.bigquery.sqltostream.load.query.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0 + ?"\ +} +wayang.bigquery.sqltostream.load.query = {\ + "in":1, "out":1,\ + "cpu":"${5*out0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.sqltostream.load.output.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0"\ +} +wayang.bigquery.sqltostream.load.output = {\ + "in":1, "out":1,\ + "cpu":"${5*out0}",\ + "ram":"0",\ + "p":0.9\ +} diff --git a/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java new file mode 100644 index 000000000..bc96addf5 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java @@ -0,0 +1,599 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.api.DataQuantaBuilder; +import org.apache.wayang.api.JavaPlanBuilder; +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.operators.BigQueryTableSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Engine-only end-to-end integration tests for every operator the BigQuery + * platform implements, driven through the Wayang API against real BigQuery. + * + *

Coverage: {@code TableSource}, {@code Filter}, {@code Projection}, + * {@code Join}, {@code GlobalReduce}, {@code ReduceBy}, {@code Sort}, + * {@code TableSink}. Every Wayang plan ends in a BigQuery {@code TableSink} that + * compiles to {@code CREATE TABLE `proj.ds.t` AS SELECT ...} executed inside + * BigQuery. Only {@code BigQuery.plugin()} is registered; there is no + * {@code Java.basicPlugin()}, so the optimizer has no Java operators to fall back + * to and the whole plan necessarily runs in BigQuery. Assertions re-query the + * sink table via plain JDBC only after {@code execute(...)} returns; the sink + * table's existence + contents prove the CTAS ran in BigQuery. + * + *

The source tables are created from inline literals in {@link #setUp()}, so + * no external BigQuery dataset or table is required. Requires a live GCP project + * + service account (the JDBC driver mandates OAuth2; the local emulator cannot + * serve it). + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class BigQueryOperatorsIT { + + private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL", + "wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com"); + private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json"); + private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + private static final String DATASET = cfg("bigquery.dataset", "BIGQUERY_DATASET", "wayang_it"); + + private static final String TABLE = tableName("orders"); + private static final String SINK_TABLE = tableName("operator_result"); + private static final String JOIN_TABLE = tableName("regions"); + private static final String[] JOIN_COLUMNS = {"order_id", "region", "product", "amount", "region_name"}; + private static final String JOIN_FLATTEN_NAME = "BigQuery test-only join flatten"; + + private static final String JDBC_URL = String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION); + + private static boolean available = false; + + private static String cfg(String sysProp, String envVar, String dflt) { + String v = System.getProperty(sysProp); + if (v == null || v.isEmpty()) v = System.getenv(envVar); + return (v == null || v.isEmpty()) ? dflt : v; + } + + private static String tableName(String table) { + return "`" + PROJECT_ID + "." + DATASET + "." + table + "`"; + } + + private static void createFixtureTables(Connection conn) throws Exception { + try (Statement st = conn.createStatement()) { + st.execute("CREATE SCHEMA IF NOT EXISTS `" + PROJECT_ID + "." + DATASET + "` " + + "OPTIONS(location='" + LOCATION + "')"); + st.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + st.execute("DROP TABLE IF EXISTS " + JOIN_TABLE); + st.execute("DROP TABLE IF EXISTS " + TABLE); + st.execute("CREATE TABLE " + TABLE + " AS " + + "SELECT * FROM UNNEST([" + + "STRUCT(1 AS order_id, 'APAC' AS region, 'Widget A' AS product, 1500.0 AS amount)," + + "STRUCT(2 AS order_id, 'EMEA' AS region, 'Widget B' AS product, 800.5 AS amount)," + + "STRUCT(3 AS order_id, 'AMER' AS region, 'Widget A' AS product, 2200.0 AS amount)," + + "STRUCT(4 AS order_id, 'APAC' AS region, 'Widget C' AS product, 350.75 AS amount)," + + "STRUCT(5 AS order_id, 'EMEA' AS region, 'Widget A' AS product, 1100.0 AS amount)," + + "STRUCT(6 AS order_id, 'AMER' AS region, 'Widget B' AS product, 950.25 AS amount)," + + "STRUCT(7 AS order_id, 'APAC' AS region, 'Widget B' AS product, 1750.0 AS amount)," + + "STRUCT(8 AS order_id, 'EMEA' AS region, 'Widget C' AS product, 420.0 AS amount)," + + "STRUCT(9 AS order_id, 'AMER' AS region, 'Widget C' AS product, 680.5 AS amount)," + + "STRUCT(10 AS order_id, 'APAC' AS region, 'Widget A' AS product, 3000.0 AS amount)" + + "])"); + // Lookup table for the join tests; region_name avoids duplicate + // region columns in the flattened CREATE TABLE AS SELECT. + st.execute("CREATE TABLE " + JOIN_TABLE + + " AS SELECT DISTINCT region AS region_name FROM " + TABLE); + } + } + + // Lifecycle + + @BeforeAll + static void setUp() { + try { + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + ResultSet rs = conn.createStatement().executeQuery("SELECT 1"); + available = rs.next(); + createFixtureTables(conn); + System.out.println("[SETUP] Connected to BigQuery project: " + PROJECT_ID); + } + } catch (Exception e) { + System.err.println("[SETUP] BigQuery not available; all tests will be skipped: " + e.getMessage()); + } + } + + @AfterAll + static void cleanup() { + if (!available) return; + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + conn.createStatement().execute("DROP TABLE IF EXISTS " + SINK_TABLE); + conn.createStatement().execute("DROP TABLE IF EXISTS " + JOIN_TABLE); + conn.createStatement().execute("DROP TABLE IF EXISTS " + TABLE); + } catch (Exception e) { + System.err.println("[CLEANUP] failed: " + e.getMessage()); + } + } + + // Tests (one per operator) + + @Test + @Order(1) + @DisplayName("BigQuery engine-only: TableSource -> TableSink") + void tableSource() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + TableSink sink = tableSink("order_id", "region", "product", "amount"); + src.connectTo(0, sink, 0); + + wayangContext().execute("BQ-TableSource", new WayangPlan(sink)); + + assertEquals(10, queryLong("SELECT count(*) FROM " + SINK_TABLE), "all 10 orders expected"); + } + + @Test + @Order(2) + @DisplayName("BigQuery engine-only: Filter -> TableSink") + void filter() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "EMEA".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'EMEA'")); + TableSink sink = tableSink("order_id", "region", "product", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute("BQ-Filter", new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE), "3 EMEA orders expected"); + assertEquals(0, queryLong("SELECT COUNTIF(region != 'EMEA') FROM " + SINK_TABLE), "only EMEA rows"); + } + + @Test + @Order(3) + @DisplayName("BigQuery engine-only: Projection -> TableSink") + void projection() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "EMEA".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'EMEA'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "product", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayangContext().execute("BQ-Projection", new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE), "3 EMEA rows expected"); + assertEquals(2, columnCount(SINK_TABLE), "projection keeps only 2 columns"); + } + + @Test + @Order(4) + @DisplayName("BigQuery engine-only: Join -> TableSink") + void join() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource orders = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + BigQueryTableSource regions = new BigQueryTableSource(JOIN_TABLE, "region_name"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation(TABLE, "region"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation(JOIN_TABLE, "region_name")); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = tableSink(JOIN_COLUMNS); + orders.connectTo(0, join, 0); + regions.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + wayangContext().execute("BQ-Join", new WayangPlan(sink)); + + assertEquals(10, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join yields one row per order (every region exists)"); + assertEquals(0, queryLong("SELECT COUNTIF(region != region_name) FROM " + SINK_TABLE), + "joined regions should match"); + } + + @Test + @Order(5) + @DisplayName("BigQuery engine-only: GlobalReduce -> TableSink") + void globalReduce() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("total_amount"); + src.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + + wayangContext().execute("BQ-GlobalReduce", new WayangPlan(sink)); + + assertSingleDoubleResult(12752.0, "global reduce collapses to a single SUM row"); + } + + @Test + @Order(6) + @DisplayName("BigQuery engine-only: ReduceBy -> TableSink") + void reduceBy() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "total_amount"); + src.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + + wayangContext().execute("BQ-ReduceBy", new WayangPlan(sink)); + + Map sums = readRegionSums(); + assertEquals(3, sums.size(), "one row per region expected"); + assertEquals(3830.75, sums.get("AMER"), 0.01); + assertEquals(2320.5, sums.get("EMEA"), 0.01); + assertEquals(6600.75, sums.get("APAC"), 0.01); + } + + @Test + @Order(7) + @DisplayName("BigQuery engine-only: Sort -> TableSink") + void sort() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(3)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("order_id", "region", "product", "amount"); + src.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + + wayangContext().execute("BQ-Sort", new WayangPlan(sink)); + + assertEquals(10, queryLong("SELECT count(*) FROM " + SINK_TABLE), "sort preserves cardinality"); + assertEquals(350.75, queryDouble("SELECT min(amount) FROM " + SINK_TABLE), 0.001); + assertEquals(3000.0, queryDouble("SELECT max(amount) FROM " + SINK_TABLE), 0.001); + } + + @Test + @Order(8) + @DisplayName("BigQuery engine-only: TableSink (filter -> CREATE TABLE AS SELECT)") + void tableSink() { + Assumptions.assumeTrue(available, "BigQuery not available"); + BigQueryTableSource src = new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "EMEA".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'EMEA'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", SINK_TABLE, + "order_id", "region", "product", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute("BQ-TableSink", new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE), "sink holds all 3 EMEA orders"); + assertEquals(0, queryLong("SELECT COUNTIF(region != 'EMEA') FROM " + SINK_TABLE), "only EMEA rows"); + } + + // JavaPlanBuilder combination tests + + @Test + @Order(9) + @DisplayName("BigQuery engine-only JavaPlanBuilder: readTable -> filter -> projection -> tableSink") + void javaPlanBuilderReadTableFilterProjection() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + new JavaPlanBuilder(wayangContext(), "BigQuery JavaPlanBuilder filter projection test") + .readTable(new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount")) + .filter(record -> "EMEA".equals(record.getField(1))) + .withSqlUdf("region = 'EMEA'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable(SINK_TABLE, "overwrite", new String[]{"order_id", "amount"}, new Properties()); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE), "3 projected EMEA orders expected"); + assertEquals(2, columnCount(SINK_TABLE), "projection keeps only 2 columns"); + } + + @Test + @Order(10) + @DisplayName("BigQuery engine-only JavaPlanBuilder: readTable -> filter -> globalReduce -> tableSink") + void javaPlanBuilderReadTableFilterGlobalReduce() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + new JavaPlanBuilder(wayangContext(), "BigQuery JavaPlanBuilder global reduce test") + .readTable(new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount")) + .filter(record -> "EMEA".equals(record.getField(1))) + .withSqlUdf("region = 'EMEA'") + .reduce((left, right) -> left) + .withSqlUdf("SUM(amount) AS total_amount") + .writeTable(SINK_TABLE, "overwrite", new String[]{"total_amount"}, new Properties()); + + assertSingleDoubleResult(2320.5, "global reduction over EMEA should return one row"); + } + + @Test + @Order(11) + @DisplayName("BigQuery engine-only JavaPlanBuilder: readTable -> reduceByKey -> sort -> tableSink") + void javaPlanBuilderReadTableReduceBySort() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + new JavaPlanBuilder(wayangContext(), "BigQuery JavaPlanBuilder reduce-by sort test") + .readTable(new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount")) + .reduceByKey( + record -> new Record(record.getField(1)), + (left, right) -> left) + .withSqlUdfs("region", "SUM(amount) AS total_amount") + .sort(record -> new Record(record.getField(0))) + .withSqlUdf("region", "ASC") + .writeTable(SINK_TABLE, "overwrite", new String[]{"region", "total_amount"}, new Properties()); + + Map sums = readRegionSums(); + assertEquals(3, sums.size(), "one row per region expected"); + assertTrue(sums.containsKey("AMER") && sums.containsKey("APAC") && sums.containsKey("EMEA")); + } + + @Test + @Order(12) + @DisplayName("BigQuery engine-only JavaPlanBuilder: readTable -> filter -> projection -> writeTable") + void javaPlanBuilderReadTableFilterProjectionTableSink() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + new JavaPlanBuilder(wayangContext(), "BigQuery JavaPlanBuilder table sink test") + .readTable(new BigQueryTableSource(TABLE, "order_id", "region", "product", "amount")) + .filter(record -> "EMEA".equals(record.getField(1))) + .withSqlUdf("region = 'EMEA'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable(SINK_TABLE, "overwrite", new String[]{"order_id", "amount"}, new Properties()); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE), "sink holds 3 projected EMEA orders"); + } + + @Test + @Order(13) + @DisplayName("BigQuery engine-only JavaPlanBuilder: readTable + readTable -> join -> tableSink") + void javaPlanBuilderReadTableJoin() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + JavaPlanBuilder plan = new JavaPlanBuilder(wayangContext(), "BigQuery JavaPlanBuilder join test"); + DataQuantaBuilder orders = plan.readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")); + DataQuantaBuilder regions = plan.readTable(new BigQueryTableSource( + JOIN_TABLE, "region_name")); + + orders + .join( + record -> new Record(record.getField(1)), + regions, + record -> new Record(record.getField(0))) + .withSqlUdfs(TABLE, "region", JOIN_TABLE, "region_name") + .map(new JoinFlattenFunction()) + .withName(JOIN_FLATTEN_NAME) + .writeTable(SINK_TABLE, "overwrite", JOIN_COLUMNS, new Properties()); + + assertEquals(10, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join yields one row per order"); + assertEquals(0, queryLong("SELECT COUNTIF(region != region_name) FROM " + SINK_TABLE), + "joined regions should match"); + } + + // Helpers + + private WayangContext wayangContext() { + Configuration config = new Configuration(); + config.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + config.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(config) + .withPlugin(BigQuery.plugin()); + } + + private TableSink tableSink(String... columnNames) { + return new TableSink<>(new Properties(), "overwrite", SINK_TABLE, columnNames); + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), left.getField(1), left.getField(2), left.getField(3), + right.getField(0)); + } + + private long queryLong(String sql) { + try (Connection c = DriverManager.getConnection(JDBC_URL); ResultSet rs = c.createStatement().executeQuery(sql)) { + rs.next(); + return rs.getLong(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private double queryDouble(String sql) { + try (Connection c = DriverManager.getConnection(JDBC_URL); ResultSet rs = c.createStatement().executeQuery(sql)) { + rs.next(); + return rs.getDouble(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private int columnCount(String table) { + try (Connection c = DriverManager.getConnection(JDBC_URL); + ResultSet rs = c.createStatement().executeQuery("SELECT * FROM " + table + " LIMIT 1")) { + return rs.getMetaData().getColumnCount(); + } catch (Exception e) { + throw new RuntimeException("query failed: column count of " + table, e); + } + } + + private void assertSingleDoubleResult(double expected, String message) { + try (Connection c = DriverManager.getConnection(JDBC_URL); + ResultSet rs = c.createStatement().executeQuery("SELECT * FROM " + SINK_TABLE)) { + assertTrue(rs.next(), message); + assertEquals(expected, rs.getDouble(1), 0.01, message); + assertFalse(rs.next(), message); + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private Map readRegionSums() { + Map sums = new HashMap<>(); + try (Connection c = DriverManager.getConnection(JDBC_URL); + ResultSet rs = c.createStatement().executeQuery("SELECT * FROM " + SINK_TABLE)) { + while (rs.next()) { + sums.put(rs.getString(1), rs.getDouble(2)); + } + return sums; + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + /** Test-only mapping for the unresolved logical join Tuple-to-Record mismatch. */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createBigQueryProjection().at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + BigQueryPlatform.getInstance())); + } + + private static BigQueryProjectionOperator createBigQueryProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(JOIN_FLATTEN_NAME); + return new BigQueryProjectionOperator((MapOperator) (MapOperator) projection); + } + } +} diff --git a/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java new file mode 100644 index 000000000..e4a7c4ce4 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryParquetSourceIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link BigQueryParquetSource}. + * + *

Set {@code -Dbigquery.parquet.source=`project.dataset.table`} or + * {@code BIGQUERY_PARQUET_SOURCE} for an existing external table. Alternatively, + * set {@code -Dbigquery.parquet.uri=gs://bucket/path/*.parquet} to let the test + * create a temporary BigQuery external table. + */ +class BigQueryParquetSourceIT { + + private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL", + "wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com"); + private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json"); + private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + private static final String DATASET = cfg("bigquery.dataset", "BIGQUERY_DATASET", "wayang_it"); + private static final String PARQUET_URI = cfg("bigquery.parquet.uri", "BIGQUERY_PARQUET_URI", ""); + private static final String CONFIGURED_SOURCE = cfg("bigquery.parquet.source", "BIGQUERY_PARQUET_SOURCE", ""); + private static final String SOURCE = CONFIGURED_SOURCE.isEmpty() ? tableName("parquet_source") : CONFIGURED_SOURCE; + private static final String SINK = cfg("bigquery.parquet.sink", "BIGQUERY_PARQUET_SINK", + tableName("parquet_source_copy")); + + private static final String JDBC_URL = String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION); + + private static boolean available = false; + private static long sourceCount = -1; + + @BeforeAll + static void setUp() { + if (CONFIGURED_SOURCE.isEmpty() && PARQUET_URI.isEmpty()) { + return; + } + try { + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS `" + PROJECT_ID + "." + DATASET + "` " + + "OPTIONS(location='" + LOCATION + "')"); + statement.execute("DROP TABLE IF EXISTS " + SINK); + if (!PARQUET_URI.isEmpty()) { + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + statement.execute("CREATE EXTERNAL TABLE " + SOURCE + + " OPTIONS (format = 'PARQUET', uris = ['" + PARQUET_URI + "'])"); + } + sourceCount = queryLong(statement, "SELECT count(*) FROM " + SOURCE); + available = true; + } + } catch (Exception e) { + System.err.println("[BigQueryParquetSourceIT] BigQuery Parquet source unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + if (!PARQUET_URI.isEmpty()) { + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + } + } catch (Exception e) { + System.err.println("[BigQueryParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsConfiguredParquetExternalTableIntoBigQuerySink() throws Exception { + Assumptions.assumeTrue(!CONFIGURED_SOURCE.isEmpty() || !PARQUET_URI.isEmpty(), + "No BigQuery Parquet source or URI configured"); + Assumptions.assumeTrue(available, "BigQuery Parquet source unavailable"); + + BigQueryParquetSource source = new BigQueryParquetSource(SOURCE, null); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK); + source.connectTo(0, sink, 0); + + wayangContext().execute("BQ-ParquetSource", new WayangPlan(sink)); + + assertEquals(sourceCount, queryLong("SELECT count(*) FROM " + SINK)); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + return new WayangContext(configuration).withPlugin(BigQuery.plugin()); + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) value = System.getenv(envVar); + return value == null || value.isEmpty() ? dflt : value; + } + + private static String tableName(String table) { + return "`" + PROJECT_ID + "." + DATASET + "." + table + "`"; + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + return queryLong(statement, sql); + } + } + + private static long queryLong(Statement statement, String sql) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 4b816b2eb..8b0629495 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -1,4 +1,3 @@ -package org.apache.wayang.jdbc.execution; /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -17,48 +16,7 @@ * limitations under the License. */ -import org.apache.wayang.basic.channels.FileChannel; -import org.apache.wayang.basic.data.Tuple2; -import org.apache.wayang.basic.operators.SpatialFilterOperator; -import org.apache.wayang.basic.operators.SpatialJoinOperator; -import org.apache.wayang.basic.operators.FilterOperator; -import org.apache.wayang.basic.operators.JoinOperator; -import org.apache.wayang.basic.operators.TableSource; -import org.apache.wayang.core.api.Job; -import org.apache.wayang.core.api.exception.WayangException; -import org.apache.wayang.core.optimizer.OptimizationContext; -import org.apache.wayang.core.plan.executionplan.Channel; -import org.apache.wayang.core.plan.executionplan.ExecutionStage; -import org.apache.wayang.core.plan.executionplan.ExecutionTask; -import org.apache.wayang.core.platform.ExecutionState; -import org.apache.wayang.core.platform.ExecutorTemplate; -import org.apache.wayang.core.platform.Platform; -import org.apache.wayang.core.util.fs.FileSystem; -import org.apache.wayang.core.util.fs.FileSystems; -import org.apache.wayang.jdbc.channels.SqlQueryChannel; -import org.apache.wayang.jdbc.compiler.FunctionCompiler; - -import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; -import org.apache.wayang.jdbc.operators.JdbcFilterOperator; -import org.apache.wayang.jdbc.operators.JdbcJoinOperator; -import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSource; - -import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.UncheckedIOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Set; -import java.util.stream.Collectors; +package org.apache.wayang.jdbc.execution; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -67,7 +25,7 @@ import org.apache.wayang.basic.operators.JoinOperator; import org.apache.wayang.basic.operators.SpatialFilterOperator; import org.apache.wayang.basic.operators.SpatialJoinOperator; -import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.Job; import org.apache.wayang.core.api.exception.WayangException; import org.apache.wayang.core.optimizer.OptimizationContext; @@ -79,7 +37,6 @@ import org.apache.wayang.core.platform.Executor; import org.apache.wayang.core.platform.ExecutorTemplate; import org.apache.wayang.core.platform.Platform; -import org.apache.wayang.core.util.WayangCollections; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.compiler.FunctionCompiler; import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; @@ -89,100 +46,118 @@ import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; import org.apache.wayang.jdbc.operators.JdbcSortOperator; +import org.apache.wayang.jdbc.operators.JdbcSourceOperator; import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSource; import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; + /** * {@link Executor} implementation for the {@link JdbcPlatformTemplate}. */ - public class JdbcExecutor extends ExecutorTemplate { - public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcTableSource tableOp, - final Collection filterTasks, final JdbcProjectionOperator projectionTask, final JdbcGlobalReduceOperator globalReduceTask, final JdbcReduceByOperator reduceByTask, final JdbcSortOperator sortTask, - final Collection joinTasks) { - final String tableName = tableOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); + + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks) { + return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, + reduceByTask, sortTask, joinTasks, null); + } + + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks, + final Configuration configuration) { + final String sourceName = sourceOp.createSqlClause( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration + ); final Collection conditions = filterTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler)) + .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final Collection joins = joinTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler)) + .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); - + final String selectClause; - if (globalReduceTask != null) { - selectClause = globalReduceTask.createSqlClause( + if (globalReduceTask != null) { + selectClause = globalReduceTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler - ); - } else if (reduceByTask != null) { - selectClause = reduceByTask.createSqlClause( + jdbcExecutor.functionCompiler, + configuration + ); + } else if (reduceByTask != null) { + selectClause = reduceByTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler - ); - } else if (projectionTask != null) { - selectClause = projectionTask.createSqlClause( + jdbcExecutor.functionCompiler, + configuration + ); + } else if (projectionTask != null) { + selectClause = projectionTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler - ); - } else { - selectClause = "*"; - } + jdbcExecutor.functionCompiler, + configuration + ); + } else { + selectClause = "*"; + } final StringBuilder sb = new StringBuilder(1000); - sb.append("SELECT ").append(selectClause).append(" FROM ").append(tableName); - if (!joins.isEmpty()) { - final String separator = " "; - for (final String join : joins) { - sb.append(separator).append(join); - } + sb.append("SELECT ").append(selectClause).append(" FROM ").append(sourceName); + for (final String join : joins) { + sb.append(" ").append(join); } if (!conditions.isEmpty()) { sb.append(" WHERE "); sb.append(String.join(" AND ", conditions)); } if (reduceByTask != null) { - sb.append(" GROUP BY " + reduceByTask.getKeyDescriptor().getSqlImplementation().getField0()); - } + sb.append(" GROUP BY ").append(reduceByTask.getKeyDescriptor().getSqlImplementation().getField0()); + } if (sortTask != null) { sb.append(sortTask.createSqlClause( - jdbcExecutor.connection, - jdbcExecutor.functionCompiler + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration )); } - appendStatementTerminator(sb); + // Intentionally no trailing ';'. A trailing semicolon is unnecessary for a + // single-statement JDBC executeQuery and is rejected by strict SQL parsers + // such as Trino and BigQuery. Postgres/SQLite/HSQLDB accept its absence. return sb; } - private static void appendStatementTerminator(final StringBuilder query) { - int i = query.length() - 1; - while (i >= 0 && Character.isWhitespace(query.charAt(i))) { - i--; - } - if (i < 0 || query.charAt(i) != ';') { - query.append(';'); - } - } - /** - * Creates a query channel and the sql statement - * - * @param stage - * @param context - * @return a tuple containing the sql statement + * Creates a query channel and the SQL statement. */ protected static Tuple2 createSqlQuery(final ExecutionStage stage, - final OptimizationContext context, final JdbcExecutor jdbcExecutor) { + final OptimizationContext context, + final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); + JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, context.getConfiguration()); + final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage, context.getConfiguration()); + assert startTask.getOperator() instanceof JdbcSourceOperator + : "Invalid JDBC stage: Start task has to be a JDBC source"; - // Verify that we can handle this instance. - final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); - assert startTask.getOperator() instanceof TableSource - : "Invalid JDBC stage: Start task has to be a TableSource"; - - // Extract the different types of ExecutionOperators from the stage. - final JdbcTableSource tableOp = (JdbcTableSource) startTask.getOperator(); + final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); SqlQueryChannel.Instance tipChannelInstance = JdbcExecutor.instantiateOutboundChannel(startTask, context, jdbcExecutor); final Collection filterTasks = new ArrayList<>(4); @@ -195,51 +170,45 @@ protected static Tuple2 createSqlQuery(final E assert allTasks.size() <= 3; ExecutionTask nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(startTask, stage); while (nextTask != null) { - // Evaluate the nextTask. final ExecutionOperator operator = nextTask.getOperator(); if (operator instanceof FilterOperator || operator instanceof SpatialFilterOperator) { filterTasks.add((JdbcExecutionOperator) operator); } else if (operator instanceof JdbcProjectionOperator) { - assert projectionTask == null; // Allow one projection operator per stage for now. + assert projectionTask == null; projectionTask = (JdbcProjectionOperator) operator; } else if (operator instanceof JdbcGlobalReduceOperator) { - final JdbcGlobalReduceOperator globalReduce = (JdbcGlobalReduceOperator) operator; - assert globalReduceTask == null; // Allow one projection operator per stage for now. - globalReduceTask = globalReduce; + assert globalReduceTask == null; + globalReduceTask = (JdbcGlobalReduceOperator) operator; } else if (operator instanceof JdbcReduceByOperator) { - final JdbcReduceByOperator reduceBy = (JdbcReduceByOperator) operator; - assert reduceByTask == null; // Allow one projection operator per stage for now. - reduceByTask = reduceBy; + assert reduceByTask == null; + reduceByTask = (JdbcReduceByOperator) operator; } else if (operator instanceof JdbcSortOperator) { - final JdbcSortOperator sort = (JdbcSortOperator) operator; - assert sortTask == null; // Allow one projection operator per stage for now. - sortTask = sort; - } else if (operator instanceof JoinOperator || (operator instanceof SpatialJoinOperator)) { + assert sortTask == null; + sortTask = (JdbcSortOperator) operator; + } else if (operator instanceof JoinOperator || operator instanceof SpatialJoinOperator) { joinTasks.add((JdbcExecutionOperator) operator); } else { - throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask.toString())); + throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask)); } - // Move the tipChannelInstance. tipChannelInstance = JdbcExecutor.instantiateOutboundChannel(nextTask, context, tipChannelInstance, jdbcExecutor); - - // Go to the next nextTask. nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(nextTask, stage); } - // Create the SQL query. - final StringBuilder query = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, globalReduceTask, reduceByTask, sortTask, joinTasks); - + final StringBuilder query = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, + globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration()); return new Tuple2<>(query.toString(), tipChannelInstance); } /** - * Selects the table source that belongs on the left-hand side of a JDBC join. + * Selects the source that belongs on the left-hand side of a JDBC join. * Stage start tasks are not ordered, but {@link JdbcJoinOperator#createSqlClause} - * assumes its first key descriptor's table is used in the {@code FROM} clause. + * assumes its first key descriptor's source is used in the {@code FROM} clause. */ - private static ExecutionTask selectStartTask(final Collection startTasks, final ExecutionStage stage) { + private static ExecutionTask selectStartTask(final Collection startTasks, + final ExecutionStage stage, + final Configuration configuration) { if (startTasks.size() == 1) { return (ExecutionTask) startTasks.iterator().next(); } @@ -247,44 +216,60 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin for (ExecutionTask task : stage.getAllTasks()) { if (task.getOperator() instanceof JdbcJoinOperator) { final JdbcJoinOperator joinOperator = (JdbcJoinOperator) task.getOperator(); - final String leftTableName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; + final String leftSourceName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; for (Object startTaskObject : startTasks) { final ExecutionTask startTask = (ExecutionTask) startTaskObject; - if (startTask.getOperator() instanceof JdbcTableSource - && ((JdbcTableSource) startTask.getOperator()).getTableName().equals(leftTableName)) { - return startTask; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + final JdbcSourceOperator sourceOperator = (JdbcSourceOperator) startTask.getOperator(); + if (sourceOperator.getSourceName().equals(leftSourceName) + || sourceOperator.getSourceName(configuration).equals(leftSourceName)) { + return startTask; + } } } } } - throw new WayangException("Could not determine the left table source for JDBC stage."); + throw new WayangException("Could not determine the left source for JDBC stage."); + } + + private static void prepareSourceTasks(final Collection startTasks, + final JdbcExecutor jdbcExecutor, + final Configuration configuration) { + for (Object startTaskObject : startTasks) { + final ExecutionTask startTask = (ExecutionTask) startTaskObject; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + ((JdbcSourceOperator) startTask.getOperator()).prepareSource( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration + ); + } + } } /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. - * Composes a SQL query from the stage's operators and executes it directly on - * the database connection, keeping all data within the database. - * - * @param stage the execution stage ending with a sink - * @param optimizationContext provides optimization information - * @param jdbcExecutor the executor with the database connection */ private static void executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, - final JdbcExecutor jdbcExecutor) { + final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); final Collection termTasks = stage.getTerminalTasks(); + JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, optimizationContext.getConfiguration()); - final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); + final ExecutionTask startTask = JdbcExecutor.selectStartTask( + startTasks, + stage, + optimizationContext.getConfiguration() + ); assert termTasks.size() == 1 : "Invalid JDBC stage: multiple terminal tasks are not currently supported."; final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; - assert startTask.getOperator() instanceof TableSource - : "Invalid JDBC stage: Start task has to be a TableSource"; + assert startTask.getOperator() instanceof JdbcSourceOperator + : "Invalid JDBC stage: Start task has to be a JDBC source"; assert termTask.getOperator() instanceof JdbcTableSinkOperator : "Invalid JDBC stage: Terminal task has to be a JdbcTableSinkOperator"; - // Extract operators from the stage - final JdbcTableSource tableOp = (JdbcTableSource) startTask.getOperator(); + final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); final JdbcTableSinkOperator sinkOp = (JdbcTableSinkOperator) termTask.getOperator(); final Collection filterTasks = new ArrayList<>(4); JdbcProjectionOperator projectionTask = null; @@ -293,58 +278,38 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat JdbcSortOperator sortTask = null; final Collection joinTasks = new ArrayList<>(); - // Walk through intermediate operators, stopping at the sink ExecutionTask nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(startTask, stage); while (nextTask != null && !(nextTask.getOperator() instanceof JdbcTableSinkOperator)) { if (nextTask.getOperator() instanceof JdbcFilterOperator) { - final JdbcFilterOperator filterOperator = (JdbcFilterOperator) nextTask.getOperator(); - filterTasks.add(filterOperator); + filterTasks.add((JdbcFilterOperator) nextTask.getOperator()); } else if (nextTask.getOperator() instanceof JdbcProjectionOperator) { - final JdbcProjectionOperator projectionOperator = (JdbcProjectionOperator) nextTask.getOperator(); assert projectionTask == null; - projectionTask = projectionOperator; + projectionTask = (JdbcProjectionOperator) nextTask.getOperator(); } else if (nextTask.getOperator() instanceof JdbcGlobalReduceOperator) { - final JdbcGlobalReduceOperator globalReduceOperator = (JdbcGlobalReduceOperator) nextTask.getOperator(); assert globalReduceTask == null; - globalReduceTask = globalReduceOperator; + globalReduceTask = (JdbcGlobalReduceOperator) nextTask.getOperator(); } else if (nextTask.getOperator() instanceof JdbcReduceByOperator) { - final JdbcReduceByOperator reduceByOperator = (JdbcReduceByOperator) nextTask.getOperator(); assert reduceByTask == null; - reduceByTask = reduceByOperator; + reduceByTask = (JdbcReduceByOperator) nextTask.getOperator(); } else if (nextTask.getOperator() instanceof JdbcSortOperator) { - final JdbcSortOperator sortOperator = (JdbcSortOperator) nextTask.getOperator(); assert sortTask == null; - sortTask = sortOperator; + sortTask = (JdbcSortOperator) nextTask.getOperator(); } else if (nextTask.getOperator() instanceof JdbcJoinOperator) { - final JdbcJoinOperator joinOperator = (JdbcJoinOperator) nextTask.getOperator(); - joinTasks.add(joinOperator); + joinTasks.add((JdbcJoinOperator) nextTask.getOperator()); } else { - throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask.toString())); + throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask)); } nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(nextTask, stage); } - // Compose the SELECT query - final StringBuilder selectQuery = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks); - - // Remove trailing semicolon from SELECT - String selectSql = selectQuery.toString(); - if (selectSql.endsWith(";")) { - selectSql = selectSql.substring(0, selectSql.length() - 1); - } - - // Get the sink's SQL clause + final String selectSql = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, + globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration()).toString(); final String sinkClause = sinkOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); - // Execute on the database try (Statement stmt = jdbcExecutor.connection.createStatement()) { - // Handle overwrite: drop existing table first if ("overwrite".equals(sinkOp.getMode())) { stmt.execute("DROP TABLE IF EXISTS " + sinkOp.getTableName()); } - // Execute the composed query: CREATE TABLE x AS SELECT ... or INSERT INTO x - // SELECT ... final String fullSql = sinkClause + " " + selectSql + sinkOp.createSqlSuffix(); stmt.execute(fullSql); jdbcExecutor.logger.info("Executed SQL sink: {}", fullSql); @@ -352,56 +317,42 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat } catch (final SQLException e) { throw new WayangException("Failed to execute SQL sink on table: " + sinkOp.getTableName(), e); } - } - /** - * Retrieves the follow-up {@link ExecutionTask} of the given {@code task} - * unless it is not comprising a {@link JdbcExecutionOperator} and/or not in the - * given {@link ExecutionStage}. - * - * @param task whose follow-up {@link ExecutionTask} is requested; should have - * a single follower - * @param stage in which the follow-up {@link ExecutionTask} should be - * @return the said follow-up {@link ExecutionTask} or {@code null} if none - */ private static ExecutionTask findJdbcExecutionOperatorTaskInStage(final ExecutionTask task, - final ExecutionStage stage) { + final ExecutionStage stage) { assert task.getNumOuputChannels() == 1; final Channel outputChannel = task.getOutputChannel(0); - if (outputChannel.getConsumers().size() != 1) { return null; } final ExecutionTask consumer = outputChannel.getConsumers().iterator().next(); - - return consumer.getStage() == stage && consumer.getOperator() instanceof JdbcExecutionOperator ? consumer - : null; + return consumer.getStage() == stage && consumer.getOperator() instanceof JdbcExecutionOperator + ? consumer + : null; } private static SqlQueryChannel.Instance instantiateOutboundChannel(final ExecutionTask task, - final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { + final OptimizationContext optimizationContext, + final JdbcExecutor jdbcExecutor) { assert task.getNumOuputChannels() == 1; assert task.getOutputChannel(0) instanceof SqlQueryChannel; final SqlQueryChannel outputChannel = (SqlQueryChannel) task.getOutputChannel(0); - final OptimizationContext.OperatorContext operatorContext = optimizationContext .getOperatorContext(task.getOperator()); - return outputChannel.createInstance(jdbcExecutor, operatorContext, 0); } private static SqlQueryChannel.Instance instantiateOutboundChannel(final ExecutionTask task, - final OptimizationContext optimizationContext, final SqlQueryChannel.Instance predecessorChannelInstance, - final JdbcExecutor jdbcExecutor) { + final OptimizationContext optimizationContext, + final SqlQueryChannel.Instance predecessorChannelInstance, + final JdbcExecutor jdbcExecutor) { final SqlQueryChannel.Instance newInstance = JdbcExecutor.instantiateOutboundChannel(task, optimizationContext, jdbcExecutor); - newInstance.getLineage().addPredecessor(predecessorChannelInstance.getLineage()); - return newInstance; } @@ -421,8 +372,7 @@ public JdbcExecutor(final JdbcPlatformTemplate platform, final Job job) { @Override public void execute(final ExecutionStage stage, final OptimizationContext optimizationContext, - final ExecutionState executionState) { - // Check if this stage ends with a sink operator + final ExecutionState executionState) { final Collection termTasks = stage.getTerminalTasks(); assert termTasks.size() == 1 : "Invalid JDBC stage: multiple terminal tasks are not currently supported."; final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; @@ -430,8 +380,6 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi if (termTask.getOperator() instanceof JdbcTableSinkOperator) { JdbcExecutor.executeSinkStage(stage, optimizationContext, this); } else { - // If it is normal stage: compose SQL and store in channel for downstream - // consumption final Tuple2 pair = JdbcExecutor.createSqlQuery(stage, optimizationContext, this); final String query = pair.field0; diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java index 570897aed..e8457f508 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java @@ -19,6 +19,7 @@ package org.apache.wayang.jdbc.operators; import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.plan.wayangplan.ExecutionOperator; import org.apache.wayang.core.platform.ChannelDescriptor; import org.apache.wayang.jdbc.compiler.FunctionCompiler; @@ -42,6 +43,13 @@ public interface JdbcExecutionOperator extends ExecutionOperator { */ String createSqlClause(Connection connection, FunctionCompiler compiler); + /** + * Creates a SQL clause under the given configuration. + */ + default String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.createSqlClause(connection, compiler); + } + @Override JdbcPlatformTemplate getPlatform(); diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java index 6ef378422..1368dde66 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java @@ -66,11 +66,17 @@ public JdbcJoinOperator(JoinOperator that) { @Override public String createSqlClause(Connection connection, FunctionCompiler compiler) { + return this.createSqlClause(connection, compiler, null); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { final Tuple left = this.keyDescriptor0.getSqlImplementation(); final Tuple right = this.keyDescriptor1.getSqlImplementation(); - final String leftTableName = left.field0; + final String platformId = this.getPlatform().getPlatformId(); + final String leftTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, left.field0); final String leftKeys = left.field1; - final String rightTableName = right.field0; + final String rightTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, right.field0); final String rightKeys = right.field1; if (leftKeys.contains(",") && rightKeys.contains(",")) { diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java new file mode 100644 index 000000000..65a7ce2f4 --- /dev/null +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.jdbc.operators; + +import org.apache.logging.log4j.LogManager; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.commons.util.profiledb.model.measurement.TimeMeasurement; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.exception.WayangException; +import org.apache.wayang.core.optimizer.OptimizationContext; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Optional; + +/** + * JDBC implementation for {@link ParquetSource}s exposed by an engine as a SQL + * relation, such as a Hive/Iceberg table or BigQuery external table. + */ +public abstract class JdbcParquetSource extends ParquetSource implements JdbcSourceOperator { + + public JdbcParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public JdbcParquetSource(ParquetSource that) { + super(that); + } + + @Override + public String getSourceName() { + return this.getInputUrl(); + } + + @Override + public String getSourceName(Configuration configuration) { + return this.resolveSourceName(configuration); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler) { + return this.getSourceName(); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.resolveSourceName(connection, configuration); + } + + @Override + public void prepareSource(Connection connection, FunctionCompiler compiler, Configuration configuration) { + this.resolveSourceName(connection, configuration); + } + + @Override + public String getLoadProfileEstimatorConfigurationKey() { + return String.format("wayang.%s.parquetsource.load", this.getPlatform().getPlatformId()); + } + + @Override + public org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator getCardinalityEstimator(int outputIndex) { + assert outputIndex == 0; + return new org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator() { + @Override + public CardinalityEstimate estimate(OptimizationContext optimizationContext, + CardinalityEstimate... inputEstimates) { + final TimeMeasurement timeMeasurement = optimizationContext.getJob().getStopWatch().start( + "Optimization", "Cardinality&Load Estimation", "Push Estimation", "Estimate source cardinalities" + ); + + try (Connection connection = JdbcParquetSource.this.getPlatform() + .createDatabaseDescriptor(optimizationContext.getConfiguration()) + .createJdbcConnection()) { + final String sql = String.format("SELECT count(*) FROM %s", + JdbcParquetSource.this.resolveSourceName( + connection, + optimizationContext.getConfiguration() + )); + final ResultSet resultSet = connection.createStatement().executeQuery(sql); + if (!resultSet.next()) { + throw new SQLException("No query result for \"" + sql + "\"."); + } + long cardinality = resultSet.getLong(1); + return new CardinalityEstimate(cardinality, cardinality, 1d); + } catch (Exception e) { + LogManager.getLogger(this.getClass()).error( + "Could not estimate cardinality for {}.", JdbcParquetSource.this, e + ); + return new CardinalityEstimate(10, 10000000, 0.9); + } finally { + timeMeasurement.stop(); + } + } + }; + } + + @Override + public Optional createCardinalityEstimator( + int outputIndex, + Configuration configuration) { + return Optional.of(this.getCardinalityEstimator(outputIndex)); + } + + private String resolveSourceName(Configuration configuration) { + if (configuration == null) { + return this.getSourceName(); + } + + final String platformId = this.getPlatform().getPlatformId(); + return resolveSourceName(configuration, platformId, this.getInputUrl()); + } + + private String resolveSourceName(Connection connection, Configuration configuration) { + final String sourceName = this.resolveSourceName(configuration); + if (configuration == null) { + return sourceName; + } + + final String platformId = this.getPlatform().getPlatformId(); + if (isAutoCreateEnabled(configuration, platformId)) { + this.createExternalRelation(connection, configuration, platformId, sourceName); + } + + return sourceName; + } + + public static String resolveSourceName(Configuration configuration, String platformId, String inputUrl) { + if (configuration == null) { + return inputUrl; + } + + return findMappedRelation(configuration, platformId, inputUrl) + .orElseGet(() -> isAutoCreateEnabled(configuration, platformId) && isParquetLocation(inputUrl) + ? createGeneratedRelationName(configuration, platformId, inputUrl) + : inputUrl); + } + + private static boolean isParquetLocation(String inputUrl) { + return inputUrl.contains("://") + || inputUrl.startsWith("/") + || inputUrl.startsWith("\\") + || inputUrl.matches("^[A-Za-z]:[\\\\/].*") + || inputUrl.contains(".parquet"); + } + + private static Optional findMappedRelation(Configuration configuration, String platformId, String inputUrl) { + final String mappingKey = String.format("wayang.%s.parquetsource.mappings", platformId); + final Optional mapping = configuration.getOptionalStringProperty(mappingKey); + if (mapping.isEmpty()) { + return Optional.empty(); + } + + for (String entry : mapping.get().split(";")) { + final String trimmedEntry = entry.trim(); + if (trimmedEntry.isEmpty()) { + continue; + } + + final int separator = trimmedEntry.indexOf('='); + if (separator < 0) { + LogManager.getLogger(JdbcParquetSource.class).warn( + "Ignoring invalid Parquet source mapping entry '{}' for {}.", trimmedEntry, mappingKey + ); + continue; + } + + final String sourceUri = trimmedEntry.substring(0, separator).trim(); + final String relationName = trimmedEntry.substring(separator + 1).trim(); + if (sourceUri.equals(inputUrl) && !relationName.isEmpty()) { + return Optional.of(relationName); + } + } + + return Optional.empty(); + } + + private static boolean isAutoCreateEnabled(Configuration configuration, String platformId) { + return configuration.getBooleanProperty( + String.format("wayang.%s.parquetsource.auto-create", platformId), + false + ); + } + + private static String createGeneratedRelationName(Configuration configuration, String platformId, String inputUrl) { + final String prefix = configuration.getStringProperty( + String.format("wayang.%s.parquetsource.auto-create.relation-prefix", platformId), + "wayang_parquet_" + ); + return prefix + shortHash(inputUrl); + } + + private static String shortHash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(16); + for (int i = 0; i < 8; i++) { + sb.append(String.format("%02x", hash[i])); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new WayangException("Could not create stable Parquet relation name.", e); + } + } + + private void createExternalRelation(Connection connection, + Configuration configuration, + String platformId, + String relationName) { + final String templateKey = String.format("wayang.%s.parquetsource.auto-create.template", platformId); + final Optional optionalTemplate = configuration.getOptionalStringProperty(templateKey); + if (optionalTemplate.isEmpty()) { + throw new WayangException(String.format( + "Parquet auto-create is enabled for platform '%s', but '%s' is not configured.", + platformId, + templateKey + )); + } + + final String template = optionalTemplate.get(); + final String ddl = template + .replace("${relation}", relationName) + .replace("${uri}", this.escapeSqlString(this.getInputUrl())) + .replace("${columns}", this.createColumnDefinitions(templateKey, template)); + + try (Statement statement = connection.createStatement()) { + statement.execute(ddl); + } catch (SQLException e) { + throw new WayangException(String.format( + "Could not create Parquet SQL relation '%s' for '%s'.", + relationName, + this.getInputUrl() + ), e); + } + } + + private String escapeSqlString(String value) { + return value.replace("'", "''"); + } + + private String createColumnDefinitions(String templateKey, String template) { + if (!template.contains("${columns}")) { + return ""; + } + + if (this.getSchema() == null || this.getSchema().getFields().isEmpty()) { + throw new WayangException(String.format( + "Parquet source auto-create template '%s' uses ${columns}, but no Parquet schema is available. " + + "Create the source with ParquetSource.create(...) or configure a template that does not " + + "need explicit columns.", + templateKey + )); + } + + return this.getSchema().getFields().stream() + .map(field -> field.getName() + " " + this.toSqlType(field)) + .reduce((left, right) -> left + ", " + right) + .orElse(""); + } + + private String toSqlType(Type field) { + if (!field.isPrimitive()) { + return "VARCHAR"; + } + + final PrimitiveType primitiveType = field.asPrimitiveType(); + final LogicalTypeAnnotation logicalType = primitiveType.getLogicalTypeAnnotation(); + if (logicalType instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation + || logicalType instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation + || logicalType instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + return "VARCHAR"; + } + if (logicalType instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimal = + (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType; + return String.format("DECIMAL(%d,%d)", decimal.getPrecision(), decimal.getScale()); + } + if (logicalType instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + return "DATE"; + } + if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + return "TIMESTAMP"; + } + + switch (primitiveType.getPrimitiveTypeName()) { + case BOOLEAN: + return "BOOLEAN"; + case INT32: + return "INTEGER"; + case INT64: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE"; + case BINARY: + return "VARCHAR"; + case FIXED_LEN_BYTE_ARRAY: + return "VARBINARY"; + case INT96: + return "TIMESTAMP"; + default: + return "VARCHAR"; + } + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java new file mode 100644 index 000000000..b7934ac89 --- /dev/null +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.jdbc.operators; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; + +import java.sql.Connection; +import java.util.List; + +/** + * Marks JDBC operators that can start a SQL stage and provide a relation for a + * {@code FROM} clause. + */ +public interface JdbcSourceOperator extends JdbcExecutionOperator { + + /** + * Name or expression used to identify this source in SQL metadata, e.g., in + * join descriptors. + */ + String getSourceName(); + + /** + * Name or expression used for this source under the given configuration. + */ + default String getSourceName(Configuration configuration) { + return this.getSourceName(); + } + + /** + * Creates a SQL clause for this source under the given configuration. + */ + default String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.createSqlClause(connection, compiler); + } + + /** + * Prepares this source for SQL generation, e.g., by registering a temporary + * relation. Implementations can keep this as a no-op when no preparation is + * required. + */ + default void prepareSource(Connection connection, FunctionCompiler compiler, Configuration configuration) { + } + + @Override + default List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("JDBC source operators have no input channels."); + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java index 4d7096649..ba5b97a69 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java @@ -33,7 +33,7 @@ /** * PostgreSQL implementation for the {@link TableSource}. */ -public abstract class JdbcTableSource extends TableSource implements JdbcExecutionOperator { +public abstract class JdbcTableSource extends TableSource implements JdbcSourceOperator { /** * Creates a new instance. @@ -58,6 +58,11 @@ public String createSqlClause(Connection connection, FunctionCompiler compiler) return this.getTableName(); } + @Override + public String getSourceName() { + return this.getTableName(); + } + @Override public String getLoadProfileEstimatorConfigurationKey() { @@ -81,7 +86,7 @@ public CardinalityEstimate estimate(OptimizationContext optimizationContext, Car .createJdbcConnection()) { // Query the table cardinality. - // No trailing ';' — strict parsers (Trino, BigQuery) reject it in executeQuery. + // No trailing ';': strict parsers (Trino, BigQuery) reject it in executeQuery. final String sql = String.format("SELECT count(*) FROM %s", JdbcTableSource.this.getTableName()); final ResultSet resultSet = connection.createStatement().executeQuery(sql); if (!resultSet.next()) { diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index 0dfd8b698..be8bd0dd2 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -29,6 +29,7 @@ import org.apache.wayang.core.profiling.NoInstrumentationStrategy; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.operators.JdbcFilterOperator; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; import org.apache.wayang.jdbc.operators.JdbcTableSource; import org.apache.wayang.jdbc.operators.SqlToStreamOperator; @@ -38,10 +39,14 @@ import org.apache.wayang.jdbc.test.HsqldbTableSource; import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -50,6 +55,120 @@ */ class JdbcExecutorTest { + @Test + void testCreateSqlStringResolvesConfiguredParquetSourceMapping() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/orders=orders_external" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/orders", null, "id", "name"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + StringBuilder query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ); + + assertEquals("SELECT * FROM orders_external", query.toString()); + } + + @Test + void testCreateSqlStringAutoCreatesConfiguredParquetSourceRelation() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/auto-orders=auto_orders_external" + ); + configuration.setProperty("wayang.hsqldb.parquetsource.auto-create", "true"); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.template", + "CREATE TABLE IF NOT EXISTS ${relation} (id INTEGER)" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/auto-orders", null, "id"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + StringBuilder query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ); + + assertEquals("SELECT * FROM auto_orders_external", query.toString()); + try (Connection connection = HsqldbPlatform.getInstance() + .createDatabaseDescriptor(configuration) + .createJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM auto_orders_external")) { + resultSet.next(); + assertEquals(0, resultSet.getLong(1)); + } + } + + @Test + void testCreateSqlStringAutoCreatesGeneratedParquetSourceRelation() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.hsqldb.parquetsource.auto-create", "true"); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.relation-prefix", + "auto_parquet_" + ); + configuration.setProperty( + "wayang.hsqldb.parquetsource.auto-create.template", + "CREATE TABLE IF NOT EXISTS ${relation} (id INTEGER)" + ); + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + + HsqldbParquetSource source = new HsqldbParquetSource("file:///warehouse/generated-orders", null, "id"); + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + + String query = JdbcExecutor.createSqlString( + executor, + source, + Collections.emptyList(), + null, + null, + null, + null, + Collections.emptyList(), + configuration + ).toString(); + + assertTrue(query.startsWith("SELECT * FROM auto_parquet_")); + String relationName = query.substring("SELECT * FROM ".length()); + try (Connection connection = HsqldbPlatform.getInstance() + .createDatabaseDescriptor(configuration) + .createJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM " + relationName)) { + resultSet.next(); + assertEquals(0, resultSet.getLong(1)); + } + } + @Test void testExecuteWithPlainTableSource() throws SQLException { Configuration configuration = new Configuration(); @@ -81,7 +200,7 @@ void testExecuteWithPlainTableSource() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer;", + "SELECT * FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -130,7 +249,7 @@ void testExecuteWithFilter() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer WHERE age >= 18;", + "SELECT * FROM customer WHERE age >= 18", sqlQueryChannelInstance.getSqlQuery() ); } @@ -172,7 +291,7 @@ void testExecuteWithProjection() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer;", + "SELECT name, age FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -240,8 +359,20 @@ void testExecuteWithProjectionAndFilters() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL;", + "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL", sqlQueryChannelInstance.getSqlQuery() ); } + + private static class HsqldbParquetSource extends JdbcParquetSource { + + HsqldbParquetSource(String inputUrl, String[] projection, String... columnNames) { + super(inputUrl, projection, columnNames); + } + + @Override + public HsqldbPlatform getPlatform() { + return HsqldbPlatform.getInstance(); + } + } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java index e1723f320..5850fd2cf 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java @@ -106,6 +106,6 @@ void testWithHsqldb() throws SQLException { assertEquals(2, count); } - assertEquals("SELECT COUNT(*) FROM testA;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT COUNT(*) FROM testA", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index 1d2834ede..bb1c2863a 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -66,6 +66,74 @@ void initDB() throws SQLException { statement.execute("DROP TABLE IF EXISTS shipments"); } } + + @Test + void testParquetSourceJoinResolvesConfiguredSourceMappings() throws SQLException { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "file:///warehouse/testA=testA;file:///warehouse/testB=testB" + ); + + Job job = mock(Job.class); + when(job.getConfiguration()).thenReturn(configuration); + when(job.getCrossPlatformExecutor()).thenReturn(new CrossPlatformExecutor(job, new NoInstrumentationStrategy())); + SqlQueryChannel.Descriptor sqlChannelDescriptor = HsqldbPlatform.getInstance().getSqlQueryChannelDescriptor(); + + ExecutionStage sqlStage = mock(ExecutionStage.class); + + HsqldbParquetSource sourceA = new HsqldbParquetSource("file:///warehouse/testA", null, "a", "b"); + HsqldbParquetSource sourceB = new HsqldbParquetSource("file:///warehouse/testB", null, "a", "b"); + + ExecutionTask sourceATask = new ExecutionTask(sourceA); + sourceATask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, sourceA.getOutput(0))); + sourceATask.setStage(sqlStage); + + ExecutionTask sourceBTask = new ExecutionTask(sourceB); + sourceBTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, sourceB.getOutput(0))); + sourceBTask.setStage(sqlStage); + + final ExecutionOperator joinOperator = new HsqldbJoinOperator( + new TransformationDescriptor( + (record) -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation("file:///warehouse/testA", "a"), + new TransformationDescriptor( + (record) -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation("file:///warehouse/testB", "a") + ); + + ExecutionTask joinTask = new ExecutionTask(joinOperator); + sourceATask.getOutputChannel(0).addConsumer(joinTask, 0); + sourceBTask.getOutputChannel(0).addConsumer(joinTask, 1); + joinTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, joinOperator.getOutput(0))); + joinTask.setStage(sqlStage); + + when(sqlStage.getStartTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList(sourceBTask, sourceATask))); + when(sqlStage.getAllTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList(sourceBTask, sourceATask, joinTask))); + when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(joinTask)); + + ExecutionStage nextStage = mock(ExecutionStage.class); + + SqlToStreamOperator sqlToStreamOperator = new SqlToStreamOperator(HsqldbPlatform.getInstance()); + ExecutionTask sqlToStreamTask = new ExecutionTask(sqlToStreamOperator); + joinTask.getOutputChannel(0).addConsumer(sqlToStreamTask, 0); + sqlToStreamTask.setStage(nextStage); + + JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); + executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + + SqlQueryChannel.Instance sqlQueryChannelInstance = + (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); + + assertEquals( + "SELECT * FROM testA JOIN testB ON testB.a=testA.a", + sqlQueryChannelInstance.getSqlQuery() + ); + } @Test void testWithHsqldb() throws SQLException { Configuration configuration = new Configuration(); @@ -142,7 +210,7 @@ void testWithHsqldb() throws SQLException { System.out.println(); assertEquals( - "SELECT * FROM testA JOIN testB ON testB.a=testA.a;", + "SELECT * FROM testA JOIN testB ON testB.a=testA.a", sqlQueryChannelInstance.getSqlQuery() ); } @@ -220,7 +288,7 @@ void testMultiConditionJoinWithHsqldb() throws SQLException { String generatedSql = sqlQueryChannelInstance.getSqlQuery(); assertEquals( - "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id;", + "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id", generatedSql ); @@ -259,4 +327,16 @@ void testMultiConditionJoinWithHsqldb() throws SQLException { } } + private static class HsqldbParquetSource extends JdbcParquetSource { + + HsqldbParquetSource(String inputUrl, String[] projection, String... columnNames) { + super(inputUrl, projection, columnNames); + } + + @Override + public HsqldbPlatform getPlatform() { + return HsqldbPlatform.getInstance(); + } + } + } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java index f00f4020e..556224027 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java @@ -91,6 +91,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java index 118fb7efa..1dc2fe12f 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java @@ -86,6 +86,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT * FROM testA ORDER BY col0 DESC;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT * FROM testA ORDER BY col0 DESC", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java index 9c8e0f42b..c72ad8433 100644 --- a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/Mappings.java @@ -32,6 +32,7 @@ public class Mappings { new FilterMapping(), new GlobalReduceMapping(), new JoinMapping(), + new ParquetSourceMapping(), new ProjectionMapping(), new ReduceByMapping(), new SortMapping(), diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..de25e7bab --- /dev/null +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.apache.wayang.presto.platform.PrestoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link PrestoParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + PrestoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new PrestoParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java new file mode 100644 index 000000000..77c0c72e6 --- /dev/null +++ b/wayang-platforms/wayang-presto/src/main/java/org/apache/wayang/presto/operators/PrestoParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * Presto implementation for Parquet-backed SQL relations. + */ +public class PrestoParquetSource extends JdbcParquetSource implements PrestoExecutionOperator { + + public PrestoParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public PrestoParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties b/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties index f0e165964..bd534e8cd 100644 --- a/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties +++ b/wayang-platforms/wayang-presto/src/main/resources/wayang-presto-defaults.properties @@ -21,6 +21,18 @@ # wayang.presto.jdbc.password = wayang.presto.jdbc.driverName = com.facebook.presto.jdbc.PrestoDriver +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a Presto-visible SQL relation during execution. +# Format: parquet-uri=presto-relation;parquet-uri-2=presto-relation-2 +# wayang.presto.parquetsource.mappings = file:///data/orders=hive.profile.orders +# +# Optional auto-create support. Keep disabled unless the configured catalog +# supports the selected DDL syntax. +# wayang.presto.parquetsource.auto-create = false +# wayang.presto.parquetsource.auto-create.relation-prefix = hive.profile.wayang_parquet_ +# wayang.presto.parquetsource.auto-create.template = CREATE TABLE IF NOT EXISTS ${relation} (${columns}) WITH (format = 'PARQUET', external_location = '${uri}') + # Hardware profile used by LoadProfileToTimeConverter. # Presto runs as a distributed MPP cluster; model a representative coordinator node. wayang.presto.cpu.mhz = 2700 @@ -28,21 +40,19 @@ wayang.presto.cores = 4 wayang.presto.costs.fix = 0.0 wayang.presto.costs.per-ms = 1.0 -# ── Cost model ──────────────────────────────────────────────────────────────── +# Cost model. # -# Formula: cpu = α * rows + β +# Formula: cpu = alpha * rows + beta # -# Presto is a distributed MPP engine: low per-row cost (small α) because scans -# are parallelised across workers, but a noticeable fixed overhead (larger β) +# Presto is a distributed MPP engine: low per-row cost (small alpha) because scans +# are parallelised across workers, but a noticeable fixed overhead (larger beta) # from query planning and cluster coordination. # -# α = 10 — parallel scan; per-row cost is cheap -# β = 800k — cluster startup + query dispatch overhead +# alpha = 10 - parallel scan; per-row cost is cheap +# beta = 800k - cluster startup + query dispatch overhead # # These are initial estimates; tune after real benchmarks by fitting the # template formula on measured data and updating the 'load' key. -# ────────────────────────────────────────────────────────────────────────────── - wayang.presto.tablesource.load.template = {\ "type":"mathex", "in":0, "out":1,\ "cpu":"?*out0 + ?",\ diff --git a/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java new file mode 100644 index 000000000..e45d14bb8 --- /dev/null +++ b/wayang-platforms/wayang-presto/src/test/java/org/apache/wayang/presto/PrestoParquetSourceIT.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.presto; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link PrestoParquetSource}. + * + *

Set {@code -Dpresto.parquet.source=catalog.schema.table} or + * {@code PRESTO_PARQUET_SOURCE} to point at an existing Parquet-backed table. + */ +class PrestoParquetSourceIT { + + private static final String HOST = System.getenv().getOrDefault("PRESTO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("PRESTO_PORT", "8081")); + private static final String USER = System.getenv().getOrDefault("PRESTO_USER", "test"); + private static final String JDBC_URL = cfg("presto.jdbc.url", "PRESTO_JDBC_URL", + String.format("jdbc:presto://%s:%d/hive", HOST, PORT)); + private static final String SOURCE = cfg("presto.parquet.source", "PRESTO_PARQUET_SOURCE", ""); + private static final String SINK = cfg("presto.parquet.sink", "PRESTO_PARQUET_SINK", + defaultSinkName(SOURCE, "wayang_parquet_source_copy")); + + private static boolean available = false; + private static long sourceCount = -1; + + @BeforeAll + static void setUp() { + if (SOURCE.isEmpty()) { + return; + } + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + sourceCount = queryLong(statement, "SELECT count(*) FROM " + SOURCE); + statement.execute("DROP TABLE IF EXISTS " + SINK); + available = true; + } catch (Exception e) { + System.err.println("[PrestoParquetSourceIT] Presto Parquet source unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + } catch (Exception e) { + System.err.println("[PrestoParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsConfiguredParquetBackedRelationIntoPrestoSink() throws Exception { + Assumptions.assumeFalse(SOURCE.isEmpty(), "No Presto Parquet source configured"); + Assumptions.assumeTrue(available, "Presto Parquet source unavailable"); + + PrestoParquetSource source = new PrestoParquetSource(SOURCE, null); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK); + source.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(sourceCount, queryLong("SELECT count(*) FROM " + SINK)); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.presto.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.presto.jdbc.user", USER); + return new WayangContext(configuration).withPlugin(Presto.plugin()); + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) value = System.getenv(envVar); + return value == null || value.isEmpty() ? dflt : value; + } + + private static String defaultSinkName(String source, String tableName) { + int lastDot = source.lastIndexOf('.'); + if (lastDot < 0) { + return tableName; + } + return source.substring(0, lastDot + 1) + tableName; + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement()) { + return queryLong(statement, sql); + } + } + + private static long queryLong(Statement statement, String sql) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static Connection jdbc() throws Exception { + Properties properties = new Properties(); + properties.setProperty("user", USER); + return DriverManager.getConnection(JDBC_URL, properties); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java index 2912ed7ed..5f26b8405 100644 --- a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java @@ -32,6 +32,7 @@ public class Mappings { new FilterMapping(), new GlobalReduceMapping(), new JoinMapping(), + new ParquetSourceMapping(), new ProjectionMapping(), new ReduceByMapping(), new SortMapping(), diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..add76f800 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link TrinoParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new TrinoParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java new file mode 100644 index 000000000..ab7f277d2 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * Trino implementation for Parquet-backed SQL relations. + */ +public class TrinoParquetSource extends JdbcParquetSource implements TrinoExecutionOperator { + + public TrinoParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public TrinoParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties index fe0429b85..0e4d328b4 100644 --- a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties +++ b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties @@ -21,6 +21,18 @@ # wayang.trino.jdbc.password = wayang.trino.jdbc.driverName = io.trino.jdbc.TrinoDriver +# Optional Parquet source resolution. +# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a Trino-visible SQL relation during execution. +# Format: parquet-uri=trino-relation;parquet-uri-2=trino-relation-2 +# wayang.trino.parquetsource.mappings = file:///data/orders=iceberg.profile.orders +# +# Optional auto-create support. Keep disabled unless the configured catalog +# supports the selected DDL syntax. +# wayang.trino.parquetsource.auto-create = false +# wayang.trino.parquetsource.auto-create.relation-prefix = iceberg.profile.wayang_parquet_ +# wayang.trino.parquetsource.auto-create.template = CREATE TABLE IF NOT EXISTS ${relation} (${columns}) WITH (format = 'PARQUET', external_location = '${uri}') + # Hardware profile used by LoadProfileToTimeConverter. # Trino runs as a distributed cluster; model a representative coordinator node. wayang.trino.cpu.mhz = 2700 @@ -28,22 +40,20 @@ wayang.trino.cores = 4 wayang.trino.costs.fix = 0.0 wayang.trino.costs.per-ms = 1.0 -# ── Cost model ──────────────────────────────────────────────────────────────── +# Cost model. # -# Formula: cpu = α * rows + β +# Formula: cpu = alpha * rows + beta # -# Trino is a distributed MPP engine: very low per-row cost (small α) because -# scans are parallelised across workers, but noticeable fixed overhead (larger β) +# Trino is a distributed MPP engine: very low per-row cost (small alpha) because +# scans are parallelised across workers, but noticeable fixed overhead (larger beta) # from query planning and cluster coordination. # -# Compared to a single-node source (Postgres α=55, β=380k): -# α = 10 — parallel scan makes per-row cost ~5× cheaper -# β = 800k — cluster startup + query dispatch overhead +# Compared to a single-node source (Postgres alpha=55, beta=380k): +# alpha = 10 - parallel scan makes per-row cost about 5x cheaper +# beta = 800k - cluster startup + query dispatch overhead # # These are initial estimates; tune after real benchmarks by fitting the # template formula below on measured data and updating the 'load' key. -# ────────────────────────────────────────────────────────────────────────────── - wayang.trino.tablesource.load.template = {\ "type":"mathex", "in":0, "out":1,\ "cpu":"?*out0 + ?",\ diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java new file mode 100644 index 000000000..46455cf83 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoParquetSourceIT.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Minimal integration test for {@link TrinoParquetSource}. + */ +class TrinoParquetSourceIT { + + private static final String HOST = System.getenv().getOrDefault("TRINO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("TRINO_PORT", "8080")); + private static final String USER = System.getenv().getOrDefault("TRINO_USER", "admin"); + private static final String JDBC_URL = String.format("jdbc:trino://%s:%d", HOST, PORT); + + private static final String SCHEMA = "iceberg.wayang_parquet_it"; + private static final String SOURCE = SCHEMA + ".orders_parquet"; + private static final String SINK = SCHEMA + ".orders_parquet_copy"; + private static final String[] COLUMNS = {"order_id", "region", "amount"}; + + private static boolean available = false; + + @BeforeAll + static void setUp() { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + statement.execute("DROP TABLE IF EXISTS " + SINK); + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + statement.execute("CREATE TABLE " + SOURCE + " WITH (format = 'PARQUET') AS " + + "SELECT * FROM (VALUES " + + "(CAST(1 AS BIGINT), 'AMER', CAST(10.0 AS DOUBLE)), " + + "(CAST(2 AS BIGINT), 'EMEA', CAST(20.0 AS DOUBLE)), " + + "(CAST(3 AS BIGINT), 'APAC', CAST(30.0 AS DOUBLE))" + + ") AS t(order_id, region, amount)"); + available = true; + } catch (Exception e) { + System.err.println("[TrinoParquetSourceIT] Trino Parquet setup unavailable: " + e.getMessage()); + } + } + + @AfterAll + static void tearDown() { + if (!available) return; + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + SINK); + statement.execute("DROP TABLE IF EXISTS " + SOURCE); + } catch (Exception e) { + System.err.println("[TrinoParquetSourceIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + void readsParquetBackedRelationIntoTrinoSink() throws Exception { + Assumptions.assumeTrue(available, "Trino Parquet setup unavailable"); + + TrinoParquetSource source = new TrinoParquetSource(SOURCE, null, COLUMNS); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK, COLUMNS); + source.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK)); + assertEquals(60.0, queryDouble("SELECT sum(amount) FROM " + SINK), 0.01); + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.trino.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.trino.jdbc.user", USER); + configuration.setProperty("wayang.trino.jdbc.password", ""); + return new WayangContext(configuration).withPlugin(Trino.plugin()); + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static double queryDouble(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getDouble(1); + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(JDBC_URL, USER, ""); + } +} diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml index 47e4b7b13..69c1ef183 100644 --- a/wayang-tests-integration/pom.xml +++ b/wayang-tests-integration/pom.xml @@ -55,6 +55,24 @@ 1.1.2-SNAPSHOT test + + org.apache.wayang + wayang-trino + 1.1.2-SNAPSHOT + test + + + org.apache.wayang + wayang-presto + 1.1.2-SNAPSHOT + test + + + org.apache.wayang + wayang-bigquery + 1.1.2-SNAPSHOT + test + org.apache.wayang wayang-giraph diff --git a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java new file mode 100644 index 000000000..7bb68c832 --- /dev/null +++ b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetJoinProfilingIT.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.tests; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.BigQuery; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.presto.Presto; +import org.apache.wayang.presto.operators.PrestoProjectionOperator; +import org.apache.wayang.presto.platform.PrestoPlatform; +import org.apache.wayang.trino.Trino; +import org.apache.wayang.trino.operators.TrinoProjectionOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Properties; +import java.util.function.BiFunction; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Real-service smoke test for Parquet-backed SQL join profiling. + * + *

The test runs on a single target SQL platform at a time. Logical + * {@link ParquetSource}s are mapped to relations that the selected platform can + * read from Parquet. + */ +class SqlParquetJoinProfilingIT { + + private static final String TARGET_PLATFORM = cfg( + "wayang.profile.parquet.target", + "WAYANG_PROFILE_PARQUET_TARGET", + "trino" + ).toLowerCase(Locale.ROOT); + private static final String ORDERS_URI = cfg( + "wayang.profile.parquet.orders.uri", + "WAYANG_PROFILE_PARQUET_ORDERS_URI", + "" + ); + private static final String CUSTOMERS_URI = cfg( + "wayang.profile.parquet.customers.uri", + "WAYANG_PROFILE_PARQUET_CUSTOMERS_URI", + "" + ); + private static final long EXPECTED_ROWS = Long.parseLong(cfg( + "wayang.profile.parquet.expectedRows", + "WAYANG_PROFILE_PARQUET_EXPECTED_ROWS", + "-1" + )); + private static final String JOIN_FLATTEN_NAME = "SQL Parquet profile join flatten"; + private static final String[] ORDERS_COLUMNS = {"order_id", "customer_id", "region", "amount"}; + private static final String[] CUSTOMERS_COLUMNS = {"cust_id", "tier"}; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "cust_id", "tier" + }; + private static final Path OUTPUT_DIR = Paths.get(cfg( + "wayang.profile.parquet.outputDir", + "WAYANG_PROFILE_PARQUET_OUTPUT_DIR", + "target/cost-profiling/parquet-sql" + )); + private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); + private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); + private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); + + private String sinkRelation; + + @AfterEach + void tearDown() throws Exception { + if (this.sinkRelation != null && isCleanupEnabled()) { + dropRelation(TARGET_PLATFORM, this.sinkRelation); + } + } + + @Test + void runsParquetJoinProfilingOnTargetSqlPlatform() throws Exception { + Assumptions.assumeTrue(isEnabled(), "Set wayang.profile.parquet.enabled=true to run this test."); + Assumptions.assumeFalse(ORDERS_URI.isEmpty(), "Configure wayang.profile.parquet.orders.uri."); + Assumptions.assumeFalse(CUSTOMERS_URI.isEmpty(), "Configure wayang.profile.parquet.customers.uri."); + Assumptions.assumeTrue(isSupportedTarget(TARGET_PLATFORM), "Unsupported target platform: " + TARGET_PLATFORM); + + this.sinkRelation = relation(TARGET_PLATFORM, "sink"); + Assumptions.assumeFalse(this.sinkRelation.isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".sink.relation."); + Assumptions.assumeFalse(relation(TARGET_PLATFORM, "orders").isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".orders.relation."); + Assumptions.assumeFalse(relation(TARGET_PLATFORM, "customers").isEmpty(), + "Configure wayang.profile.parquet." + TARGET_PLATFORM + ".customers.relation."); + Assumptions.assumeTrue(isPlatformAvailable(TARGET_PLATFORM), + "Target platform is not reachable: " + TARGET_PLATFORM); + + Files.createDirectories(OUTPUT_DIR); + writeManifestHeaderIfNeeded(); + dropRelation(TARGET_PLATFORM, this.sinkRelation); + + Platform targetPlatform = targetPlatform(); + WayangPlan plan = createParquetJoinPlan(this.sinkRelation, targetPlatform); + long startedAt = System.currentTimeMillis(); + wayangContext(targetPlatform).execute("SQL-Parquet-Join-Profiling-" + TARGET_PLATFORM, plan); + long elapsedMillis = System.currentTimeMillis() - startedAt; + + long actualRows = queryLong(TARGET_PLATFORM, "SELECT count(*) FROM " + this.sinkRelation); + if (EXPECTED_ROWS >= 0) { + assertEquals(EXPECTED_ROWS, actualRows); + } + appendManifest(TARGET_PLATFORM, actualRows, elapsedMillis, this.sinkRelation); + } + + private static WayangPlan createParquetJoinPlan(String sinkRelation, Platform targetPlatform) { + ParquetSource orders = new ParquetSource(ORDERS_URI, null, ORDERS_COLUMNS); + ParquetSource customers = new ParquetSource(CUSTOMERS_URI, null, CUSTOMERS_COLUMNS); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(ORDERS_URI, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_URI, "cust_id") + ); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkRelation, JOIN_COLUMNS); + + orders.addTargetPlatform(targetPlatform); + customers.addTargetPlatform(targetPlatform); + join.addTargetPlatform(targetPlatform); + flatten.addTargetPlatform(targetPlatform); + sink.addTargetPlatform(targetPlatform); + + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + return new WayangPlan(sink); + } + + private static WayangContext wayangContext(Platform targetPlatform) { + Configuration configuration = new Configuration(); + configureJdbc(configuration, TARGET_PLATFORM); + configureParquetMappings(configuration, TARGET_PLATFORM); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.core.explain.enabled", "false"); + configuration.setProperty("wayang.core.log.executions", EXECUTIONS_PATH.toString().replace('\\', '/')); + configuration.setProperty("wayang.core.log.cardinalities", CARDINALITIES_PATH.toString().replace('\\', '/')); + configuration.getMappingProvider().addAllToWhitelist(Collections.singleton(new JoinFlattenMapping(targetPlatform))); + + WayangContext context = new WayangContext(configuration); + switch (TARGET_PLATFORM) { + case "trino": + return context.withPlugin(Trino.plugin()); + case "presto": + return context.withPlugin(Presto.plugin()); + case "bigquery": + return context.withPlugin(BigQuery.plugin()); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static void configureJdbc(Configuration configuration, String platform) { + switch (platform) { + case "trino": + configuration.setProperty("wayang.trino.jdbc.url", cfg( + "wayang.trino.jdbc.url", + "WAYANG_TRINO_JDBC_URL", + "jdbc:trino://localhost:8080" + )); + configuration.setProperty("wayang.trino.jdbc.user", cfg( + "wayang.trino.jdbc.user", + "WAYANG_TRINO_JDBC_USER", + "admin" + )); + configuration.setProperty("wayang.trino.jdbc.password", cfg( + "wayang.trino.jdbc.password", + "WAYANG_TRINO_JDBC_PASSWORD", + "" + )); + return; + case "presto": + configuration.setProperty("wayang.presto.jdbc.url", cfg( + "wayang.presto.jdbc.url", + "WAYANG_PRESTO_JDBC_URL", + "jdbc:presto://localhost:8081/hive" + )); + configuration.setProperty("wayang.presto.jdbc.user", cfg( + "wayang.presto.jdbc.user", + "WAYANG_PRESTO_JDBC_USER", + "test" + )); + return; + case "bigquery": + configuration.setProperty("wayang.bigquery.jdbc.url", bigQueryJdbcUrl()); + return; + default: + throw new IllegalArgumentException("Unsupported target platform: " + platform); + } + } + + private static void configureParquetMappings(Configuration configuration, String platform) { + String ordersRelation = relation(platform, "orders"); + String customersRelation = relation(platform, "customers"); + if (!ordersRelation.isEmpty() && !customersRelation.isEmpty()) { + configuration.setProperty( + String.format("wayang.%s.parquetsource.mappings", platform), + ORDERS_URI + "=" + ordersRelation + ";" + CUSTOMERS_URI + "=" + customersRelation + ); + } + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class) + ), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class) + ); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private static Platform targetPlatform() { + switch (TARGET_PLATFORM) { + case "trino": + return TrinoPlatform.getInstance(); + case "presto": + return PrestoPlatform.getInstance(); + case "bigquery": + return BigQueryPlatform.getInstance(); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static boolean isSupportedTarget(String platform) { + return "trino".equals(platform) || "presto".equals(platform) || "bigquery".equals(platform); + } + + private static boolean isPlatformAvailable(String platform) { + try (Connection connection = jdbc(platform); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT 1")) { + return resultSet.next(); + } catch (Exception e) { + return false; + } + } + + private static long queryLong(String platform, String sql) throws Exception { + try (Connection connection = jdbc(platform); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static void dropRelation(String platform, String relation) throws Exception { + try (Connection connection = jdbc(platform); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + relation); + } + } + + private static Connection jdbc(String platform) throws Exception { + switch (platform) { + case "trino": + return DriverManager.getConnection( + cfg("wayang.trino.jdbc.url", "WAYANG_TRINO_JDBC_URL", "jdbc:trino://localhost:8080"), + cfg("wayang.trino.jdbc.user", "WAYANG_TRINO_JDBC_USER", "admin"), + cfg("wayang.trino.jdbc.password", "WAYANG_TRINO_JDBC_PASSWORD", "") + ); + case "presto": + Properties prestoProperties = new Properties(); + prestoProperties.setProperty( + "user", + cfg("wayang.presto.jdbc.user", "WAYANG_PRESTO_JDBC_USER", "test") + ); + return DriverManager.getConnection( + cfg("wayang.presto.jdbc.url", "WAYANG_PRESTO_JDBC_URL", "jdbc:presto://localhost:8081/hive"), + prestoProperties + ); + case "bigquery": + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + return DriverManager.getConnection(bigQueryJdbcUrl()); + default: + throw new IllegalArgumentException("Unsupported platform: " + platform); + } + } + + private static String bigQueryJdbcUrl() { + String configuredUrl = cfg("wayang.bigquery.jdbc.url", "WAYANG_BIGQUERY_JDBC_URL", ""); + if (!configuredUrl.isEmpty()) { + return configuredUrl; + } + String projectId = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + String serviceAccount = cfg( + "bigquery.saEmail", + "BIGQUERY_SA_EMAIL", + "wayang-bq@" + projectId + ".iam.gserviceaccount.com" + ); + String keyPath = cfg( + "bigquery.keyPath", + "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json" + ); + String location = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + return String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + projectId, + serviceAccount, + keyPath, + location + ); + } + + private static String relation(String platform, String relationRole) { + return cfg( + String.format("wayang.profile.parquet.%s.%s.relation", platform, relationRole), + String.format( + "WAYANG_PROFILE_PARQUET_%s_%s_RELATION", + platform.toUpperCase(Locale.ROOT), + relationRole.toUpperCase(Locale.ROOT) + ), + "" + ); + } + + private static boolean isEnabled() { + return Boolean.parseBoolean(cfg( + "wayang.profile.parquet.enabled", + "WAYANG_PROFILE_PARQUET_ENABLED", + "false" + )); + } + + private static boolean isCleanupEnabled() { + return Boolean.parseBoolean(cfg( + "wayang.profile.parquet.cleanup", + "WAYANG_PROFILE_PARQUET_CLEANUP", + "true" + )); + } + + private static void writeManifestHeaderIfNeeded() throws Exception { + if (Files.exists(MANIFEST_PATH)) { + return; + } + try (BufferedWriter writer = Files.newBufferedWriter(MANIFEST_PATH, StandardCharsets.UTF_8)) { + writer.write("target_platform,orders_uri,customers_uri,actual_rows,expected_rows,elapsed_millis,sink_relation"); + writer.newLine(); + } + } + + private static void appendManifest(String platform, + long actualRows, + long elapsedMillis, + String sinkRelation) throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter( + MANIFEST_PATH, + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND)) { + writer.write(String.join(",", + platform, + ORDERS_URI, + CUSTOMERS_URI, + String.valueOf(actualRows), + String.valueOf(EXPECTED_ROWS), + String.valueOf(elapsedMillis), + sinkRelation)); + writer.newLine(); + } + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) { + value = System.getenv(envVar); + } + return value == null || value.isEmpty() ? dflt : value; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0), + right.getField(1) + ); + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + private final Platform targetPlatform; + + private JoinFlattenMapping(Platform targetPlatform) { + this.targetPlatform = targetPlatform; + } + + @Override + public Collection getTransformations() { + return Collections.singleton(createTransformation(this.targetPlatform, this::createProjectionOperator)); + } + + private org.apache.wayang.core.plan.wayangplan.Operator createProjectionOperator( + MapOperator matchedOperator, + Integer epoch) { + switch (TARGET_PLATFORM) { + case "trino": + return new TrinoProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + case "presto": + return new PrestoProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + case "bigquery": + return new BigQueryProjectionOperator((MapOperator) (MapOperator) createProjection()).at(epoch); + default: + throw new IllegalArgumentException("Unsupported target platform: " + TARGET_PLATFORM); + } + } + + private static PlanTransformation createTransformation( + Platform platform, + BiFunction factoryFunction) { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false + ).withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + factoryFunction::apply + ); + + return new PlanTransformation(SubplanPattern.createSingleton(pattern), factory, platform); + } + + private static MapOperator, Record> createProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + java.util.Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class) + ); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class) + ); + projection.setName(JOIN_FLATTEN_NAME); + return projection; + } + } +} diff --git a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java new file mode 100644 index 000000000..7fa42e40e --- /dev/null +++ b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/SqlParquetMultiPlatformIT.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.tests; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.bigquery.BigQuery; +import org.apache.wayang.bigquery.operators.BigQueryJoinOperator; +import org.apache.wayang.bigquery.operators.BigQueryParquetSource; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.plan.wayangplan.OperatorAlternative; +import org.apache.wayang.core.plan.wayangplan.PlanTraversal; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.util.WayangCollections; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; +import org.apache.wayang.presto.Presto; +import org.apache.wayang.presto.operators.PrestoJoinOperator; +import org.apache.wayang.presto.operators.PrestoParquetSource; +import org.apache.wayang.trino.Trino; +import org.apache.wayang.trino.operators.TrinoJoinOperator; +import org.apache.wayang.trino.operators.TrinoParquetSource; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Regression coverage for Parquet-backed SQL sources when several SQL platforms + * are registered in the same {@link WayangContext}. + */ +class SqlParquetMultiPlatformIT { + + private static final String ORDERS_PARQUET = "s3://wayang-profiler/orders.parquet"; + private static final String CUSTOMERS_PARQUET = "s3://wayang-profiler/customers.parquet"; + + @Test + void registersParquetJoinAlternativesForAllSqlPlatforms() { + WayangContext context = new WayangContext() + .withPlugin(Trino.plugin()) + .withPlugin(Presto.plugin()) + .withPlugin(BigQuery.plugin()); + Configuration configuration = context.getConfiguration(); + + WayangPlan plan = createParquetJoinPlan(); + plan.prepare(); + plan.applyTransformations(gatherTransformations(configuration)); + + Collection alternatives = PlanTraversal.upstream() + .traverse(plan.getSinks()) + .getTraversedNodesWith(operator -> operator instanceof OperatorAlternative) + .stream() + .map(operator -> (OperatorAlternative) operator) + .collect(Collectors.toList()); + + List sourceAlternatives = alternatives.stream() + .filter(alternative -> containsAlternative(alternative, ParquetSource.class)) + .collect(Collectors.toList()); + List joinAlternatives = alternatives.stream() + .filter(alternative -> containsAlternative(alternative, JoinOperator.class)) + .collect(Collectors.toList()); + + assertEquals(2, sourceAlternatives.size()); + sourceAlternatives.forEach(alternative -> { + assertTrue(containsAlternative(alternative, TrinoParquetSource.class)); + assertTrue(containsAlternative(alternative, PrestoParquetSource.class)); + assertTrue(containsAlternative(alternative, BigQueryParquetSource.class)); + }); + + assertEquals(1, joinAlternatives.size()); + OperatorAlternative joinAlternative = joinAlternatives.get(0); + assertTrue(containsAlternative(joinAlternative, TrinoJoinOperator.class)); + assertTrue(containsAlternative(joinAlternative, PrestoJoinOperator.class)); + assertTrue(containsAlternative(joinAlternative, BigQueryJoinOperator.class)); + } + + @Test + void resolvesSameParquetUriToPlatformSpecificRelations() { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.trino.parquetsource.mappings", + ORDERS_PARQUET + "=iceberg.profiler.orders" + ); + configuration.setProperty( + "wayang.presto.parquetsource.mappings", + ORDERS_PARQUET + "=hive.profiler.orders" + ); + configuration.setProperty( + "wayang.bigquery.parquetsource.mappings", + ORDERS_PARQUET + "=`project.profiler.orders_ext`" + ); + + assertEquals( + "iceberg.profiler.orders", + JdbcParquetSource.resolveSourceName(configuration, "trino", ORDERS_PARQUET) + ); + assertEquals( + "hive.profiler.orders", + JdbcParquetSource.resolveSourceName(configuration, "presto", ORDERS_PARQUET) + ); + assertEquals( + "`project.profiler.orders_ext`", + JdbcParquetSource.resolveSourceName(configuration, "bigquery", ORDERS_PARQUET) + ); + } + + private static WayangPlan createParquetJoinPlan() { + ParquetSource orders = new ParquetSource( + ORDERS_PARQUET, + null, + "order_id", + "customer_id", + "amount" + ); + ParquetSource customers = new ParquetSource( + CUSTOMERS_PARQUET, + null, + "customer_id", + "region" + ); + + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor( + record -> (Integer) record.getField(1), + Record.class, + Integer.class + ).withSqlImplementation(ORDERS_PARQUET, "customer_id"), + new TransformationDescriptor( + record -> (Integer) record.getField(0), + Record.class, + Integer.class + ).withSqlImplementation(CUSTOMERS_PARQUET, "customer_id") + ); + + @SuppressWarnings({"rawtypes", "unchecked"}) + DataSetType> joinOutputType = + (DataSetType) DataSetType.createDefaultUnchecked(Tuple2.class); + List> collector = new ArrayList<>(); + LocalCallbackSink> sink = + LocalCallbackSink.createCollectingSink(collector, joinOutputType); + + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, sink, 0); + + return new WayangPlan(sink); + } + + private static Collection gatherTransformations(Configuration configuration) { + Set platforms = WayangCollections.asSet(configuration.getPlatformProvider().provideAll()); + return configuration.getMappingProvider().provideAll().stream() + .flatMap((Mapping mapping) -> mapping.getTransformations().stream()) + .filter(transformation -> transformation.getTargetPlatforms().isEmpty() + || platforms.containsAll(transformation.getTargetPlatforms())) + .collect(Collectors.toList()); + } + + private static boolean containsAlternative(OperatorAlternative alternative, + Class operatorClass) { + return alternative.getAlternatives().stream() + .map(OperatorAlternative.Alternative::getContainedOperator) + .anyMatch(operatorClass::isInstance); + } +} From 30e211b7dbe426c2d377b3a8ba57f3dcfb46a21e Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Mon, 17 Aug 2026 02:44:44 +0800 Subject: [PATCH 2/4] Align spatial JDBC SQL expectations --- .../spatial/operators/jdbc/JdbcSpatialJoinOperatorTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wayang-plugins/wayang-spatial/src/test/java/org/apache/wayang/spatial/operators/jdbc/JdbcSpatialJoinOperatorTest.java b/wayang-plugins/wayang-spatial/src/test/java/org/apache/wayang/spatial/operators/jdbc/JdbcSpatialJoinOperatorTest.java index 313679f65..77d0272ed 100644 --- a/wayang-plugins/wayang-spatial/src/test/java/org/apache/wayang/spatial/operators/jdbc/JdbcSpatialJoinOperatorTest.java +++ b/wayang-plugins/wayang-spatial/src/test/java/org/apache/wayang/spatial/operators/jdbc/JdbcSpatialJoinOperatorTest.java @@ -78,7 +78,7 @@ void testSpatialJoinIntersectsGeneratesCorrectSql() throws SQLException { String sql = buildSpatialJoinSql(SpatialPredicate.INTERSECTS); assertEquals( - "SELECT * FROM testA JOIN testB ON ST_Intersects(testA.geom, testB.geom);", + "SELECT * FROM testA JOIN testB ON ST_Intersects(testA.geom, testB.geom)", sql ); } @@ -88,7 +88,7 @@ void testSpatialJoinContainsGeneratesCorrectSql() throws SQLException { String sql = buildSpatialJoinSql(SpatialPredicate.CONTAINS); assertEquals( - "SELECT * FROM testA JOIN testB ON ST_Contains(testA.geom, testB.geom);", + "SELECT * FROM testA JOIN testB ON ST_Contains(testA.geom, testB.geom)", sql ); } @@ -98,7 +98,7 @@ void testSpatialJoinWithinGeneratesCorrectSql() throws SQLException { String sql = buildSpatialJoinSql(SpatialPredicate.WITHIN); assertEquals( - "SELECT * FROM testA JOIN testB ON ST_Within(testA.geom, testB.geom);", + "SELECT * FROM testA JOIN testB ON ST_Within(testA.geom, testB.geom)", sql ); } From 4408821fe1fb3fd3bd9f2aa9ce081c19a64fcb93 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 27 Oct 2026 17:39:27 -0400 Subject: [PATCH 3/4] Restore JdbcExecutor documentation comments --- .../wayang/jdbc/execution/JdbcExecutor.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 1f2eeaac4..d5ba8f61a 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -149,16 +149,24 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, /** * Creates a query channel and the SQL statement. + * + * @param stage the JDBC execution stage + * @param context provides optimization information and the Wayang configuration + * @param jdbcExecutor the executor with the database connection + * @return a tuple containing the SQL statement and its output channel */ protected static Tuple2 createSqlQuery(final ExecutionStage stage, final OptimizationContext context, final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, context.getConfiguration()); + + // Verify that we can handle this instance. final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage, context.getConfiguration()); assert startTask.getOperator() instanceof JdbcSourceOperator : "Invalid JDBC stage: Start task has to be a JDBC source"; + // Extract the different types of ExecutionOperators from the stage. final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); SqlQueryChannel.Instance tipChannelInstance = JdbcExecutor.instantiateOutboundChannel(startTask, context, jdbcExecutor); @@ -172,20 +180,21 @@ protected static Tuple2 createSqlQuery(final E assert allTasks.size() <= 3; ExecutionTask nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(startTask, stage); while (nextTask != null) { + // Evaluate the nextTask. final ExecutionOperator operator = nextTask.getOperator(); if (operator instanceof FilterOperator || operator instanceof SpatialFilterOperator) { filterTasks.add((JdbcExecutionOperator) operator); } else if (operator instanceof JdbcProjectionOperator) { - assert projectionTask == null; + assert projectionTask == null; // Allow one projection operator per stage for now. projectionTask = (JdbcProjectionOperator) operator; } else if (operator instanceof final JdbcGlobalReduceOperator globalReduce) { - assert globalReduceTask == null; + assert globalReduceTask == null; // Allow one global reduce operator per stage for now. globalReduceTask = globalReduce; } else if (operator instanceof final JdbcReduceByOperator reduceBy) { - assert reduceByTask == null; + assert reduceByTask == null; // Allow one reduce-by operator per stage for now. reduceByTask = reduceBy; } else if (operator instanceof final JdbcSortOperator sort) { - assert sortTask == null; + assert sortTask == null; // Allow one sort operator per stage for now. sortTask = sort; } else if (operator instanceof JoinOperator || operator instanceof SpatialJoinOperator) { joinTasks.add((JdbcExecutionOperator) operator); @@ -193,11 +202,15 @@ protected static Tuple2 createSqlQuery(final E throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask)); } + // Move the tipChannelInstance. tipChannelInstance = JdbcExecutor.instantiateOutboundChannel(nextTask, context, tipChannelInstance, jdbcExecutor); + + // Go to the next nextTask. nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(nextTask, stage); } + // Create the SQL query. final StringBuilder query = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration()); return new Tuple2<>(query.toString(), tipChannelInstance); @@ -252,6 +265,12 @@ private static void prepareSourceTasks(final Collection startTasks, /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. + * Composes a SQL query from the stage's operators and executes it directly on + * the database connection, keeping all data within the database. + * + * @param stage the execution stage ending with a sink + * @param optimizationContext provides optimization information + * @param jdbcExecutor the executor with the database connection */ private static long executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { From c29946a1ff35a8c948b39caef1092e532b8feed1 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 28 Oct 2026 23:37:37 -0400 Subject: [PATCH 4/4] Decouple JDBC join source name resolution --- .../wayang/jdbc/execution/JdbcExecutor.java | 52 +++++++++++++++++-- .../jdbc/operators/JdbcJoinOperator.java | 15 +++--- .../jdbc/operators/JdbcJoinOperatorTest.java | 4 ++ 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 453f8a320..c5e19317f 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -57,6 +57,8 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -74,7 +76,7 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSortOperator sortTask, final Collection joinTasks) { return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, - reduceByTask, sortTask, joinTasks, null); + reduceByTask, sortTask, joinTasks, null, Map.of()); } public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, @@ -86,6 +88,24 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSortOperator sortTask, final Collection joinTasks, final Configuration configuration) { + final Map sourceNames = Map.of( + sourceOp.getSourceName(), + sourceOp.getSourceName(configuration) + ); + return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, + reduceByTask, sortTask, joinTasks, configuration, sourceNames); + } + + private static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, + final JdbcSourceOperator sourceOp, + final Collection filterTasks, + final JdbcProjectionOperator projectionTask, + final JdbcGlobalReduceOperator globalReduceTask, + final JdbcReduceByOperator reduceByTask, + final JdbcSortOperator sortTask, + final Collection joinTasks, + final Configuration configuration, + final Map sourceNames) { final String sourceName = sourceOp.createSqlClause( jdbcExecutor.connection, jdbcExecutor.functionCompiler, @@ -95,7 +115,13 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final Collection joins = joinTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) + .map(op -> op instanceof JdbcJoinOperator + ? ((JdbcJoinOperator) op).createSqlClause( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + sourceNames + ) + : op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final String selectClause; @@ -212,7 +238,8 @@ protected static Tuple2 createSqlQuery(final E // Create the SQL query. final StringBuilder query = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration()); + globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration(), + resolveSourceNames(startTasks, context.getConfiguration())); return new Tuple2<>(query.toString(), tipChannelInstance); } @@ -263,6 +290,22 @@ private static void prepareSourceTasks(final Collection startTasks, } } + private static Map resolveSourceNames(final Collection startTasks, + final Configuration configuration) { + final Map sourceNames = new HashMap<>(); + for (Object startTaskObject : startTasks) { + final ExecutionTask startTask = (ExecutionTask) startTaskObject; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + final JdbcSourceOperator sourceOperator = (JdbcSourceOperator) startTask.getOperator(); + sourceNames.put( + sourceOperator.getSourceName(), + sourceOperator.getSourceName(configuration) + ); + } + } + return sourceNames; + } + /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. * Composes a SQL query from the stage's operators and executes it directly on @@ -324,7 +367,8 @@ private static long executeSinkStage(final ExecutionStage stage, final Optimizat } final String selectSql = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration()).toString(); + globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration(), + resolveSourceNames(startTasks, optimizationContext.getConfiguration())).toString(); final String sinkClause = sinkOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); try (Statement stmt = jdbcExecutor.connection.createStatement()) { diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java index 1368dde66..9f7109233 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java @@ -20,15 +20,16 @@ import org.apache.wayang.core.util.Tuple; import org.apache.wayang.basic.data.Record; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.types.DataSetType; import org.apache.wayang.basic.operators.JoinOperator; import org.apache.wayang.core.function.TransformationDescriptor; import org.apache.wayang.jdbc.compiler.FunctionCompiler; -import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator; import org.apache.wayang.core.optimizer.costs.LoadProfileEstimators; import java.sql.Connection; +import java.util.Map; import java.util.Optional; /** @@ -66,17 +67,17 @@ public JdbcJoinOperator(JoinOperator that) { @Override public String createSqlClause(Connection connection, FunctionCompiler compiler) { - return this.createSqlClause(connection, compiler, null); + return this.createSqlClause(connection, compiler, Map.of()); } - @Override - public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + public String createSqlClause(Connection connection, + FunctionCompiler compiler, + Map sourceNames) { final Tuple left = this.keyDescriptor0.getSqlImplementation(); final Tuple right = this.keyDescriptor1.getSqlImplementation(); - final String platformId = this.getPlatform().getPlatformId(); - final String leftTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, left.field0); + final String leftTableName = sourceNames.getOrDefault(left.field0, left.field0); final String leftKeys = left.field1; - final String rightTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, right.field0); + final String rightTableName = sourceNames.getOrDefault(right.field0, right.field0); final String rightKeys = right.field1; if (leftKeys.contains(",") && rightKeys.contains(",")) { diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index bb1c2863a..cea2c11aa 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -137,6 +137,10 @@ void testParquetSourceJoinResolvesConfiguredSourceMappings() throws SQLException @Test void testWithHsqldb() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.hsqldb.parquetsource.mappings", + "testA=wrongParquetRelationA;testB=wrongParquetRelationB" + ); Job job = mock(Job.class); when(job.getConfiguration()).thenReturn(configuration);