From 7538421beaa2a8c5ad114129fd23525738e09c8c Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Tue, 10 Nov 2026 23:40:32 +0530 Subject: [PATCH 1/6] Add wayang-clickhouse platform skeleton with filter and projection support. Introduces the ClickHouse JDBC module following the Trino/Postgres plugin pattern so filter/projection SQL can be pushed down, as the first slice of #871. --- wayang-platforms/pom.xml | 1 + wayang-platforms/wayang-clickhouse/pom.xml | 70 ++++++++++++++ .../apache/wayang/clickhouse/ClickHouse.java | 61 ++++++++++++ .../channels/ChannelConversions.java | 54 +++++++++++ .../clickhouse/mapping/FilterMapping.java | 62 +++++++++++++ .../wayang/clickhouse/mapping/Mappings.java | 36 ++++++++ .../clickhouse/mapping/ProjectionMapping.java | 66 +++++++++++++ .../ClickHouseExecutionOperator.java | 31 +++++++ .../operators/ClickHouseFilterOperator.java | 51 ++++++++++ .../ClickHouseProjectionOperator.java | 48 ++++++++++ .../operators/ClickHouseTableSource.java | 57 ++++++++++++ .../platform/ClickHousePlatform.java | 53 +++++++++++ .../plugin/ClickHouseConversionsPlugin.java | 58 ++++++++++++ .../clickhouse/plugin/ClickHousePlugin.java | 58 ++++++++++++ .../wayang-clickhouse-defaults.properties | 92 +++++++++++++++++++ .../operators/ClickHousePluginTest.java | 56 +++++++++++ .../ClickHouseSqlGenerationTest.java | 62 +++++++++++++ .../platform/ClickHousePlatformTest.java | 44 +++++++++ 18 files changed, 960 insertions(+) create mode 100644 wayang-platforms/wayang-clickhouse/pom.xml create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/ClickHouse.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/channels/ChannelConversions.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/FilterMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ProjectionMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseExecutionOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseFilterOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseProjectionOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSource.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/platform/ClickHousePlatform.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHouseConversionsPlugin.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHousePlugin.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/platform/ClickHousePlatformTest.java diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml index 6a56735c1..c5791e4fd 100644 --- a/wayang-platforms/pom.xml +++ b/wayang-platforms/pom.xml @@ -46,6 +46,7 @@ wayang-bigquery wayang-presto wayang-trino + wayang-clickhouse wayang-tensorflow wayang-semantic diff --git a/wayang-platforms/wayang-clickhouse/pom.xml b/wayang-platforms/wayang-clickhouse/pom.xml new file mode 100644 index 000000000..85c1240a2 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/pom.xml @@ -0,0 +1,70 @@ + + + + 4.0.0 + + + wayang-platforms + org.apache.wayang + 1.1.2-SNAPSHOT + + + wayang-clickhouse + + Wayang Platform ClickHouse + + Wayang implementation of the operators to be working with the platform "ClickHouse" + + + + org.apache.wayang.platform.clickhouse + 0.9.0 + + + + + com.clickhouse + clickhouse-jdbc + ${clickhouse.version} + all + + + org.apache.wayang + wayang-basic + ${project.version} + + + org.apache.wayang + wayang-jdbc-template + ${project.version} + + + org.apache.wayang + wayang-spark + ${project.version} + + + org.junit.jupiter + junit-jupiter + test + + + + diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/ClickHouse.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/ClickHouse.java new file mode 100644 index 000000000..21f99a6f2 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/ClickHouse.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse; + +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.clickhouse.plugin.ClickHouseConversionsPlugin; +import org.apache.wayang.clickhouse.plugin.ClickHousePlugin; + +/** + * Register for relevant components of this module. + */ +public class ClickHouse { + + private static final ClickHousePlugin PLUGIN = new ClickHousePlugin(); + + private static final ClickHouseConversionsPlugin CONVERSIONS_PLUGIN = new ClickHouseConversionsPlugin(); + + /** + * Retrieve the {@link ClickHousePlugin}. + * + * @return the {@link ClickHousePlugin} + */ + public static ClickHousePlugin plugin() { + return PLUGIN; + } + + /** + * Retrieve the {@link ClickHouseConversionsPlugin}. + * + * @return the {@link ClickHouseConversionsPlugin} + */ + public static ClickHouseConversionsPlugin conversionPlugin() { + return CONVERSIONS_PLUGIN; + } + + /** + * Retrieve the {@link ClickHousePlatform}. + * + * @return the {@link ClickHousePlatform} + */ + public static ClickHousePlatform platform() { + return ClickHousePlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/channels/ChannelConversions.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/channels/ChannelConversions.java new file mode 100644 index 000000000..7ffc055ec --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/channels/ChannelConversions.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.channels; + +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.optimizer.channels.DefaultChannelConversion; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.jdbc.operators.SqlToRddOperator; +import org.apache.wayang.jdbc.operators.SqlToStreamOperator; +import org.apache.wayang.spark.channels.RddChannel; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link ChannelConversion}s supported for this platform. + */ +public class ChannelConversions { + + public static final ChannelConversion SQL_TO_STREAM_CONVERSION = new DefaultChannelConversion( + ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor(), + StreamChannel.DESCRIPTOR, + () -> new SqlToStreamOperator(ClickHousePlatform.getInstance()) + ); + + public static final ChannelConversion SQL_TO_UNCACHED_RDD_CONVERSION = new DefaultChannelConversion( + ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor(), + RddChannel.UNCACHED_DESCRIPTOR, + () -> new SqlToRddOperator(ClickHousePlatform.getInstance()) + ); + + public static final Collection ALL = Arrays.asList( + SQL_TO_STREAM_CONVERSION, + SQL_TO_UNCACHED_RDD_CONVERSION + ); + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/FilterMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/FilterMapping.java new file mode 100644 index 000000000..84648376f --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/FilterMapping.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseFilterOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link FilterOperator} to {@link ClickHouseFilterOperator}. + */ +@SuppressWarnings("unchecked") +public class FilterMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "filter", new FilterOperator<>(null, DataSetType.createDefault(Record.class)), false + ).withAdditionalTest(op -> op.getPredicateDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new ClickHouseFilterOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java new file mode 100644 index 000000000..942dd56ac --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.core.mapping.Mapping; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link Mapping}s supported for this platform. + */ +public class Mappings { + + public static final Collection ALL = Arrays.asList( + new FilterMapping(), + new ProjectionMapping() + ); + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ProjectionMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ProjectionMapping.java new file mode 100644 index 000000000..9e8b8e0c8 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ProjectionMapping.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseProjectionOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link MapOperator} to {@link ClickHouseProjectionOperator}. + */ +public class ProjectionMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "projection", + new MapOperator<>( + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getFunctionDescriptor() instanceof ProjectionDescriptor) + .withAdditionalTest(op -> op.getNumInputs() == 1); // No broadcasts. + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new ClickHouseProjectionOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseExecutionOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseExecutionOperator.java new file mode 100644 index 000000000..44c4fd834 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseExecutionOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; + +public interface ClickHouseExecutionOperator extends JdbcExecutionOperator { + + @Override + default ClickHousePlatform getPlatform() { + return ClickHousePlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseFilterOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseFilterOperator.java new file mode 100644 index 000000000..8a7654b40 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseFilterOperator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.operators.JdbcFilterOperator; + +/** + * ClickHouse implementation of the {@link FilterOperator}. + */ +public class ClickHouseFilterOperator extends JdbcFilterOperator implements ClickHouseExecutionOperator { + + /** + * Creates a new instance. + */ + public ClickHouseFilterOperator(PredicateDescriptor predicateDescriptor) { + super(predicateDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public ClickHouseFilterOperator(FilterOperator that) { + super(that); + } + + @Override + protected ClickHouseFilterOperator createCopy() { + return new ClickHouseFilterOperator(this); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseProjectionOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseProjectionOperator.java new file mode 100644 index 000000000..d9bb03b63 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseProjectionOperator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; + +/** + * ClickHouse implementation of the projection {@link MapOperator}. + */ +public class ClickHouseProjectionOperator extends JdbcProjectionOperator implements ClickHouseExecutionOperator { + + public ClickHouseProjectionOperator(String... fieldNames) { + super(fieldNames); + } + + public ClickHouseProjectionOperator(ProjectionDescriptor functionDescriptor) { + super(functionDescriptor); + } + + public ClickHouseProjectionOperator(MapOperator that) { + super(that); + } + + @Override + protected ClickHouseProjectionOperator createCopy() { + return new ClickHouseProjectionOperator(this); + } + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSource.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSource.java new file mode 100644 index 000000000..426d690eb --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSource.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.operators.JdbcTableSource; + +import java.util.List; + +/** + * ClickHouse implementation for the {@link TableSource}. + * + *

Table names may be unqualified ({@code orders}) or database-qualified + * ({@code analytics.orders}). + */ +public class ClickHouseTableSource extends JdbcTableSource implements ClickHouseExecutionOperator { + + /** + * Creates a new instance. + * + * @see TableSource#TableSource(String, String...) + */ + public ClickHouseTableSource(String tableName, String... columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public ClickHouseTableSource(JdbcTableSource that) { + super(that); + } + + @Override + public List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("This operator has no input channels."); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/platform/ClickHousePlatform.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/platform/ClickHousePlatform.java new file mode 100644 index 000000000..c64a416ad --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/platform/ClickHousePlatform.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.platform; + +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; + +/** + * {@link Platform} implementation for ClickHouse. + */ +public class ClickHousePlatform extends JdbcPlatformTemplate { + + private static final String PLATFORM_NAME = "ClickHouse"; + + private static final String CONFIG_NAME = "clickhouse"; + + private static ClickHousePlatform instance = null; + + public static ClickHousePlatform getInstance() { + if (instance == null) { + instance = new ClickHousePlatform(); + } + return instance; + } + + protected ClickHousePlatform() { + super(PLATFORM_NAME, CONFIG_NAME); + } + + @Override + public String getJdbcDriverClassName() { + // Prefer the non-deprecated Driver; ClickHouseDriver remains only as a + // compatibility alias in clickhouse-jdbc 0.9.x. + return com.clickhouse.jdbc.Driver.class.getName(); + } + +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHouseConversionsPlugin.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHouseConversionsPlugin.java new file mode 100644 index 000000000..a2a44c4c1 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHouseConversionsPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.plugin; + +import org.apache.wayang.clickhouse.channels.ChannelConversions; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * This {@link Plugin} enables channel conversions for the {@link ClickHousePlatform}. + */ +public class ClickHouseConversionsPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(ClickHousePlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Collections.emptyList(); + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHousePlugin.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHousePlugin.java new file mode 100644 index 000000000..081d89421 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/plugin/ClickHousePlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.plugin; + +import org.apache.wayang.clickhouse.channels.ChannelConversions; +import org.apache.wayang.clickhouse.mapping.Mappings; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link ClickHousePlatform}. + */ +public class ClickHousePlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(ClickHousePlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Mappings.ALL; + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties new file mode 100644 index 000000000..68d0109d2 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Connection (override per deployment in wayang.properties) +# wayang.clickhouse.jdbc.url = jdbc:clickhouse://localhost:8123/default +# wayang.clickhouse.jdbc.user = default +# wayang.clickhouse.jdbc.password = + +# Cost function parameters +# NB: Not measured. Placeholder values so the optimizer can schedule ClickHouse +# operators; calibrate with ClickHouseCostPilotIT in a later PR. +wayang.clickhouse.cpu.mhz = 2700 +wayang.clickhouse.cores = 4 +wayang.clickhouse.costs.fix = 0.0 +wayang.clickhouse.costs.per-ms = 1.0 + +wayang.clickhouse.tablesource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.tablesource.load = {\ + "in":0, "out":1,\ + "cpu":"${55*out0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.filter.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.filter.load = {\ + "in":1, "out":1,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.projection.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.projection.load = {\ + "in":1, "out":1,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.sqltostream.load.query.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0 + ?"\ +} +wayang.clickhouse.sqltostream.load.query = {\ + "in":1, "out":1,\ + "cpu":"${50*out0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} +wayang.clickhouse.sqltostream.load.output.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0"\ +} +wayang.clickhouse.sqltostream.load.output = {\ + "in":1, "out":1,\ + "cpu":"${50*out0}",\ + "ram":"0",\ + "p":0.9\ +} diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java new file mode 100644 index 000000000..b666fb2bd --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.clickhouse.ClickHouse; +import org.apache.wayang.clickhouse.mapping.FilterMapping; +import org.apache.wayang.clickhouse.mapping.ProjectionMapping; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.java.platform.JavaPlatform; +import org.junit.jupiter.api.Test; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Smoke tests for ClickHouse plugin registration. + */ +class ClickHousePluginTest { + + @Test + void pluginExposesPlatformsMappingsAndConversions() { + assertTrue(ClickHouse.plugin().getRequiredPlatforms().contains(ClickHousePlatform.getInstance())); + assertTrue(ClickHouse.plugin().getRequiredPlatforms().contains(JavaPlatform.getInstance())); + assertEquals(2, ClickHouse.plugin().getChannelConversions().size()); + + Collection mappings = ClickHouse.plugin().getMappings(); + assertEquals(2, mappings.size()); + assertTrue(mappings.stream().anyMatch(FilterMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(ProjectionMapping.class::isInstance)); + } + + @Test + void conversionsPluginHasNoMappings() { + assertTrue(ClickHouse.conversionPlugin().getMappings().isEmpty()); + assertEquals(2, ClickHouse.conversionPlugin().getChannelConversions().size()); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java new file mode 100644 index 000000000..9a9b5abc4 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * SQL-generation unit tests for PR-1 ClickHouse operators. These do not require + * a live ClickHouse server. + */ +class ClickHouseSqlGenerationTest { + + private final FunctionCompiler compiler = new FunctionCompiler(); + + @Test + void tableSourceEmitsTableName() { + ClickHouseTableSource source = new ClickHouseTableSource("analytics.orders", "order_id", "region"); + assertEquals("analytics.orders", source.createSqlClause(null, compiler)); + assertSame(org.apache.wayang.clickhouse.platform.ClickHousePlatform.getInstance(), source.getPlatform()); + } + + @Test + void filterEmitsSqlPredicate() { + ClickHouseFilterOperator filter = new ClickHouseFilterOperator( + new PredicateDescriptor<>( + (PredicateDescriptor.SerializablePredicate) record -> { + throw new UnsupportedOperationException(); + }, + Record.class + ).withSqlImplementation("region = 'AMER'") + ); + assertEquals("region = 'AMER'", filter.createSqlClause(null, compiler)); + } + + @Test + void projectionEmitsFieldList() { + ClickHouseProjectionOperator projection = new ClickHouseProjectionOperator("order_id", "amount"); + assertEquals("order_id, amount", projection.createSqlClause(null, compiler)); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/platform/ClickHousePlatformTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/platform/ClickHousePlatformTest.java new file mode 100644 index 000000000..c4153b870 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/platform/ClickHousePlatformTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.platform; + +import org.apache.wayang.clickhouse.ClickHouse; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Unit tests for {@link ClickHousePlatform}. + */ +class ClickHousePlatformTest { + + @Test + void platformIsSingleton() { + assertSame(ClickHousePlatform.getInstance(), ClickHouse.platform()); + assertSame(ClickHousePlatform.getInstance(), ClickHousePlatform.getInstance()); + } + + @Test + void platformIdAndDriver() { + ClickHousePlatform platform = ClickHousePlatform.getInstance(); + assertEquals("clickhouse", platform.getPlatformId()); + assertEquals("com.clickhouse.jdbc.Driver", platform.getJdbcDriverClassName()); + } +} From af54e414279f37392266d985ad34dce2821f0095 Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Fri, 13 Nov 2026 20:30:50 +0530 Subject: [PATCH 2/6] Complete ClickHouse platform: remaining operators, sink ENGINE, IT, and docs. Adds join/reduce/sort/table-sink mappings, ClickHouseOperatorsIT, README, and a wayang-applications demo so #871 is covered in this PR. --- wayang-applications/README.md | 1 + wayang-applications/clickhouse.md | 93 +++ wayang-applications/pom.xml | 5 + .../wayang/applications/ClickHouseDemo.java | 159 +++++ wayang-docs/src/main/resources/index.md | 2 + wayang-platforms/wayang-clickhouse/README.md | 49 ++ wayang-platforms/wayang-clickhouse/pom.xml | 6 + .../mapping/GlobalReduceMapping.java | 62 ++ .../clickhouse/mapping/JoinMapping.java | 74 +++ .../wayang/clickhouse/mapping/Mappings.java | 7 +- .../clickhouse/mapping/ReduceByMapping.java | 65 +++ .../clickhouse/mapping/SortMapping.java | 64 +++ .../clickhouse/mapping/TableSinkMapping.java | 60 ++ .../ClickHouseGlobalReduceOperator.java | 43 ++ .../operators/ClickHouseJoinOperator.java | 45 ++ .../operators/ClickHouseReduceByOperator.java | 45 ++ .../operators/ClickHouseSortOperator.java | 43 ++ .../ClickHouseTableSinkOperator.java | 88 +++ .../wayang-clickhouse-defaults.properties | 76 ++- .../clickhouse/ClickHouseOperatorsIT.java | 542 ++++++++++++++++++ .../operators/ClickHousePluginTest.java | 12 +- .../ClickHouseSqlGenerationTest.java | 85 ++- 22 files changed, 1621 insertions(+), 5 deletions(-) create mode 100644 wayang-applications/clickhouse.md create mode 100644 wayang-applications/src/main/java/org/apache/wayang/applications/ClickHouseDemo.java create mode 100644 wayang-platforms/wayang-clickhouse/README.md create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/GlobalReduceMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/JoinMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ReduceByMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/SortMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/TableSinkMapping.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseGlobalReduceOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseJoinOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseReduceByOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseSortOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java diff --git a/wayang-applications/README.md b/wayang-applications/README.md index e99789083..d9565b9f1 100644 --- a/wayang-applications/README.md +++ b/wayang-applications/README.md @@ -79,6 +79,7 @@ particular application are listed here. ## SQL Platform Examples - [BigQuery](bigquery.md) +- [ClickHouse](clickhouse.md) - [DuckDB](duckdb.md) - [Presto](presto.md) - [Trino](trino.md) diff --git a/wayang-applications/clickhouse.md b/wayang-applications/clickhouse.md new file mode 100644 index 000000000..df01ff886 --- /dev/null +++ b/wayang-applications/clickhouse.md @@ -0,0 +1,93 @@ + + +# ClickHouse Example + +This example runs filter and projection plans against an existing ClickHouse +table. Wayang pushes `region = 'AMER'` and the selected columns into SQL +executed by ClickHouse. + +## Prerequisites + +- Java 17 +- A ClickHouse deployment reachable from the machine running Wayang +- A ClickHouse user with `SELECT` permission on the example table + +Wayang does not start ClickHouse. + +## Prepare the table + +Using the ClickHouse client, replace `default` with a database available in +your deployment, then run: + +```sql +CREATE DATABASE IF NOT EXISTS default; + +CREATE TABLE IF NOT EXISTS default.orders ( + order_id Int64, + region String, + product String, + amount Float64, + order_date String +) ENGINE = MergeTree ORDER BY order_id; + +INSERT INTO default.orders VALUES + (1, 'AMER', 'book', 25.50, '2026-01-10'), + (2, 'EMEA', 'desk', 300.00, '2026-01-11'), + (3, 'AMER', 'chair', 85.25, '2026-01-12'); +``` + +## Configure Wayang + +Create a properties file such as `/tmp/clickhouse-example.properties`: + +```properties +wayang.clickhouse.jdbc.url = jdbc:clickhouse://clickhouse.example.com:8123/default +wayang.clickhouse.jdbc.user = default +wayang.clickhouse.jdbc.password = +wayang.clickhouse.demo.table = default.orders +``` + +Change the endpoint, credentials, database, and table for your deployment. + +## Build and run + +Build the application and its dependencies from the repository root: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-applications -am \ + -DskipTests -Dpython.worker.tests.skip=true install +``` + +Run the example: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-applications exec:java \ + -Dexec.mainClass=org.apache.wayang.applications.ClickHouseDemo \ + "-Dexec.args=file:///tmp/clickhouse-example.properties" +``` + +Both sections should return the two `AMER` rows. The projection section returns +only `region`, `product`, and `amount`. + +## Troubleshooting + +- **Connection refused:** verify the host, HTTP port (default `8123`), and + network access from Wayang. +- **Authentication failure:** check the user and password, or JDBC URL options + required by the deployment. +- **Table not found:** use a fully qualified `database.table` name. diff --git a/wayang-applications/pom.xml b/wayang-applications/pom.xml index 7a4884376..4ad23b496 100644 --- a/wayang-applications/pom.xml +++ b/wayang-applications/pom.xml @@ -75,6 +75,11 @@ wayang-trino ${project.version} + + org.apache.wayang + wayang-clickhouse + ${project.version} + org.apache.wayang wayang-core diff --git a/wayang-applications/src/main/java/org/apache/wayang/applications/ClickHouseDemo.java b/wayang-applications/src/main/java/org/apache/wayang/applications/ClickHouseDemo.java new file mode 100644 index 000000000..8c5643020 --- /dev/null +++ b/wayang-applications/src/main/java/org/apache/wayang/applications/ClickHouseDemo.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.applications; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.clickhouse.ClickHouse; +import org.apache.wayang.clickhouse.operators.ClickHouseTableSource; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.java.Java; + +import java.util.ArrayList; +import java.util.List; + +/** + * Configurable example for the Wayang ClickHouse connector. + * + *

See {@code wayang-applications/clickhouse.md} for configuration and usage. + */ +public class ClickHouseDemo { + + private static Configuration configuration; + private static String sourceTable; + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + throw new IllegalArgumentException("Usage: ClickHouseDemo "); + } + configuration = new Configuration(args[0]); + configuration.getStringProperty("wayang.clickhouse.jdbc.url"); + sourceTable = configuration.getStringProperty( + "wayang.clickhouse.demo.table", "default.orders" + ); + filterDemo(); + projectionDemo(); + } + + static void filterDemo() throws Exception { + System.out.println("ClickHouse filter pushdown"); + System.out.println(); + System.out.println(" Operator: FilterOperator -> ClickHouseFilterOperator"); + System.out.println(" SQL sent: SELECT * FROM " + sourceTable); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + + WayangContext wayang = buildWayang(); + + List results = new ArrayList<>(); + ClickHouseTableSource source = new ClickHouseTableSource( + sourceTable, "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayang.execute("ClickHouse-Filter-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang:"); + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + "order_id", "region", "product", "amount", "order_date"); + System.out.println(" " + repeat('-', 54)); + for (Record r : results) { + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + r.getField(0), r.getField(1), r.getField(2), r.getField(3), r.getField(4)); + } + System.out.println(); + System.out.printf(" %d AMER rows returned.%n", results.size()); + System.out.println(); + } + + static void projectionDemo() throws Exception { + System.out.println("ClickHouse projection pushdown"); + System.out.println(); + System.out.println(" Operators: FilterOperator -> ClickHouseFilterOperator"); + System.out.println(" MapOperator -> ClickHouseProjectionOperator"); + System.out.println(" SQL sent: SELECT region, product, amount"); + System.out.println(" FROM " + sourceTable); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + + WayangContext wayang = buildWayang(); + + List results = new ArrayList<>(); + ClickHouseTableSource source = new ClickHouseTableSource( + sourceTable, "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "product", "amount", "order_date"), + "region", "product", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayang.execute("ClickHouse-Projection-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang (projected columns only):"); + System.out.printf(" %-6s %-10s %10s%n", "region", "product", "amount"); + System.out.println(" " + repeat('-', 30)); + for (Record r : results) { + System.out.printf(" %-6s %-10s %10s%n", r.getField(0), r.getField(1), r.getField(2)); + } + System.out.println(); + System.out.printf(" %d AMER rows returned with 3 columns.%n", results.size()); + System.out.println(); + } + + private static WayangContext buildWayang() { + return new WayangContext(configuration) + .withPlugin(Java.basicPlugin()) + .withPlugin(ClickHouse.plugin()); + } + + private static String repeat(char c, int n) { + StringBuilder sb = new StringBuilder(n); + for (int i = 0; i < n; i++) { + sb.append(c); + } + return sb.toString(); + } +} diff --git a/wayang-docs/src/main/resources/index.md b/wayang-docs/src/main/resources/index.md index c1fba3fc9..1c2b2a8c3 100644 --- a/wayang-docs/src/main/resources/index.md +++ b/wayang-docs/src/main/resources/index.md @@ -49,6 +49,7 @@ Wayang has built in support for the following frameworks: - Trino - Presto - Google BigQuery +- ClickHouse - TensorFlow (Deep Learning) - Apache Giraph v1.2.0 @@ -95,6 +96,7 @@ Add the platform adapters you wish to target: - `wayang-postgres` / `wayang-sqlite3` / `wayang-generic-jdbc`: Relational database platforms - `wayang-trino` / `wayang-presto`: Distributed SQL query engines - `wayang-bigquery`: Google BigQuery cloud data warehouse +- `wayang-clickhouse`: ClickHouse OLAP database - `wayang-tensorflow`: TensorFlow deep learning platform - `wayang-giraph`: Graph processing platform diff --git a/wayang-platforms/wayang-clickhouse/README.md b/wayang-platforms/wayang-clickhouse/README.md new file mode 100644 index 000000000..8f1ee8fc5 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/README.md @@ -0,0 +1,49 @@ + + +# Wayang Platform ClickHouse + +This module connects Wayang to a user-managed ClickHouse deployment through JDBC. +Configure the endpoint and credentials in a Wayang properties file: + +```properties +wayang.clickhouse.jdbc.url = jdbc:clickhouse://clickhouse.example.com:8123/default +wayang.clickhouse.jdbc.user = default +wayang.clickhouse.jdbc.password = +# Optional ENGINE clause for CREATE TABLE ... AS SELECT (overwrite sinks) +wayang.clickhouse.tablesink.engine = MergeTree ORDER BY tuple() +``` + +The runnable filter/projection example and its table requirements are documented +in [`wayang-applications/clickhouse.md`](../../wayang-applications/clickhouse.md). + +## Notes + +- Table sinks in `overwrite` mode emit + `CREATE TABLE ENGINE = AS SELECT ...` because ClickHouse requires + an engine. Override the engine via `wayang.clickhouse.tablesink.engine` or the + same key on the `TableSink` `Properties`. +- ClickHouse identifiers are case-sensitive. Prefer `database.table` names. +- Joins build the hash table from the right input; nullable join behaviour + follows the server's `join_use_nulls` setting (default `0`). + +## Integration tests + +`ClickHouseOperatorsIT` is intended for connector development. It creates and +removes its own fixture tables in the `wayang_it` database. If ClickHouse is +unreachable the class is skipped (not failed). + +```bash +CLICKHOUSE_HOST=localhost CLICKHOUSE_PORT=8123 CLICKHOUSE_USER=default \ +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-clickhouse -am \ + -Dtest=ClickHouseOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false test +``` + +These environment variables configure the integration test only. Applications +use the `wayang.clickhouse.jdbc.*` properties described above. diff --git a/wayang-platforms/wayang-clickhouse/pom.xml b/wayang-platforms/wayang-clickhouse/pom.xml index 85c1240a2..6054339af 100644 --- a/wayang-platforms/wayang-clickhouse/pom.xml +++ b/wayang-platforms/wayang-clickhouse/pom.xml @@ -60,6 +60,12 @@ wayang-spark ${project.version} + + org.apache.wayang + wayang-api-scala-java + ${project.version} + test + org.junit.jupiter junit-jupiter diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/GlobalReduceMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/GlobalReduceMapping.java new file mode 100644 index 000000000..5a31ee15a --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/GlobalReduceMapping.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseGlobalReduceOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link GlobalReduceOperator} to {@link ClickHouseGlobalReduceOperator}. + */ +@SuppressWarnings("unchecked") +public class GlobalReduceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduce", new GlobalReduceOperator(null, DataSetType.createDefault(Record.class)), false) + .withAdditionalTest(op -> op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new ClickHouseGlobalReduceOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/JoinMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/JoinMapping.java new file mode 100644 index 000000000..13db6cedb --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/JoinMapping.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseJoinOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link JoinOperator} to {@link ClickHouseJoinOperator}. + */ +@SuppressWarnings("unchecked") +public class JoinMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "join", + new JoinOperator( + null, + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ), + false + ) + .withAdditionalTest(op -> op.getKeyDescriptor0() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor1() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor0().getSqlImplementation() != null) + .withAdditionalTest(op -> op.getKeyDescriptor1().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new ClickHouseJoinOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java index 942dd56ac..13baf06a6 100644 --- a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/Mappings.java @@ -30,7 +30,12 @@ public class Mappings { public static final Collection ALL = Arrays.asList( new FilterMapping(), - new ProjectionMapping() + new GlobalReduceMapping(), + new JoinMapping(), + new ProjectionMapping(), + new ReduceByMapping(), + new SortMapping(), + new TableSinkMapping() ); } diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ReduceByMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ReduceByMapping.java new file mode 100644 index 000000000..4bd700ea9 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/ReduceByMapping.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseReduceByOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ReduceByOperator} to {@link ClickHouseReduceByOperator}. + */ +@SuppressWarnings("unchecked") +public class ReduceByMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduceBy", + new ReduceByOperator(null, null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null + && op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new ClickHouseReduceByOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/SortMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/SortMapping.java new file mode 100644 index 000000000..112e7291c --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/SortMapping.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseSortOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link SortOperator} to {@link ClickHouseSortOperator}. + */ +@SuppressWarnings("unchecked") +public class SortMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "sort", + new SortOperator(null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new ClickHouseSortOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/TableSinkMapping.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/TableSinkMapping.java new file mode 100644 index 000000000..2e7d1e180 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/mapping/TableSinkMapping.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.clickhouse.operators.ClickHouseTableSinkOperator; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link TableSink} to {@link ClickHouseTableSinkOperator}. + */ +@SuppressWarnings("unchecked") +public class TableSinkMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + ClickHousePlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern<>( + "sink", new TableSink<>(null, null, null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new ClickHouseTableSinkOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseGlobalReduceOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseGlobalReduceOperator.java new file mode 100644 index 000000000..edad0a0c6 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseGlobalReduceOperator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.jdbc.operators.JdbcGlobalReduceOperator; + +/** + * ClickHouse implementation of the {@link GlobalReduceOperator}. + */ +public class ClickHouseGlobalReduceOperator extends JdbcGlobalReduceOperator implements ClickHouseExecutionOperator { + + public ClickHouseGlobalReduceOperator(ReduceDescriptor reduceDescriptor) { + super(reduceDescriptor); + } + + public ClickHouseGlobalReduceOperator(GlobalReduceOperator that) { + super(that); + } + + @Override + protected ClickHouseGlobalReduceOperator createCopy() { + return new ClickHouseGlobalReduceOperator(this); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseJoinOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseJoinOperator.java new file mode 100644 index 000000000..7a11c9bca --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseJoinOperator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcJoinOperator; + +/** + * ClickHouse implementation of the {@link JoinOperator}. + */ +public class ClickHouseJoinOperator extends JdbcJoinOperator implements ClickHouseExecutionOperator { + + public ClickHouseJoinOperator( + TransformationDescriptor keyDescriptor0, + TransformationDescriptor keyDescriptor1) { + super(keyDescriptor0, keyDescriptor1); + } + + public ClickHouseJoinOperator(JoinOperator that) { + super(that); + } + + @Override + protected ClickHouseJoinOperator createCopy() { + return new ClickHouseJoinOperator(this); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseReduceByOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseReduceByOperator.java new file mode 100644 index 000000000..131a21b92 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseReduceByOperator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; + +/** + * ClickHouse implementation of the {@link ReduceByOperator}. + */ +public class ClickHouseReduceByOperator extends JdbcReduceByOperator implements ClickHouseExecutionOperator { + + public ClickHouseReduceByOperator(TransformationDescriptor keyDescriptor, + ReduceDescriptor reduceDescriptor) { + super(keyDescriptor, reduceDescriptor); + } + + public ClickHouseReduceByOperator(ReduceByOperator that) { + super(that); + } + + @Override + protected ClickHouseReduceByOperator createCopy() { + return new ClickHouseReduceByOperator(this); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseSortOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseSortOperator.java new file mode 100644 index 000000000..1eb7be38a --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseSortOperator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcSortOperator; + +/** + * ClickHouse implementation of the {@link SortOperator}. + */ +public class ClickHouseSortOperator extends JdbcSortOperator implements ClickHouseExecutionOperator { + + public ClickHouseSortOperator(TransformationDescriptor keyDescriptor) { + super(keyDescriptor); + } + + public ClickHouseSortOperator(SortOperator that) { + super(that); + } + + @Override + protected ClickHouseSortOperator createCopy() { + return new ClickHouseSortOperator(this); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java new file mode 100644 index 000000000..d3f28e4fc --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; + +import java.sql.Connection; + +/** + * ClickHouse implementation of the {@link JdbcTableSinkOperator}. + * + *

ClickHouse requires an {@code ENGINE} clause for {@code CREATE TABLE ... AS + * SELECT}. The default is {@value #DEFAULT_ENGINE}; override via the + * {@code engine} constructor argument or the + * {@code wayang.clickhouse.tablesink.engine} property when constructing the sink. + */ +public class ClickHouseTableSinkOperator extends JdbcTableSinkOperator implements ClickHouseExecutionOperator { + + public static final String DEFAULT_ENGINE = "MergeTree ORDER BY tuple()"; + + public static final String ENGINE_PROPERTY = "wayang.clickhouse.tablesink.engine"; + + private final String engine; + + public ClickHouseTableSinkOperator(String tableName, String[] columnNames) { + this(tableName, columnNames, DEFAULT_ENGINE); + } + + public ClickHouseTableSinkOperator(String tableName, String[] columnNames, String engine) { + super(tableName, columnNames); + this.engine = normalizeEngine(engine); + } + + public ClickHouseTableSinkOperator(TableSink that) { + super(that); + this.engine = engineFrom(that); + } + + public String getEngine() { + return this.engine; + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler) { + String mode = this.getMode(); + String targetTable = this.getQuotedTableName(connection); + if ("overwrite".equals(mode)) { + return "CREATE TABLE " + targetTable + " ENGINE = " + this.engine + " AS"; + } + return "INSERT INTO " + targetTable; + } + + private static String normalizeEngine(String engine) { + if (engine == null || engine.trim().isEmpty()) { + return DEFAULT_ENGINE; + } + return engine.trim(); + } + + private static String engineFrom(TableSink that) { + if (that.getProperties() != null) { + String configured = that.getProperties().getProperty(ENGINE_PROPERTY); + if (configured != null && !configured.trim().isEmpty()) { + return configured.trim(); + } + } + return DEFAULT_ENGINE; + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties index 68d0109d2..5befbf0fb 100644 --- a/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties +++ b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties @@ -20,9 +20,13 @@ # wayang.clickhouse.jdbc.user = default # wayang.clickhouse.jdbc.password = +# Table sink ENGINE used by CREATE TABLE ... AS SELECT (overwrite mode). +# May also be set on a TableSink Properties object under the same key. +wayang.clickhouse.tablesink.engine = MergeTree ORDER BY tuple() + # Cost function parameters # NB: Not measured. Placeholder values so the optimizer can schedule ClickHouse -# operators; calibrate with ClickHouseCostPilotIT in a later PR. +# operators; calibrate later with a ClickHouseCostPilotIT. wayang.clickhouse.cpu.mhz = 2700 wayang.clickhouse.cores = 4 wayang.clickhouse.costs.fix = 0.0 @@ -70,6 +74,76 @@ wayang.clickhouse.projection.load = {\ "p":0.9\ } +wayang.clickhouse.join.load.template = {\ + "type":"mathex", "in":2, "out":1,\ + "cpu":"?*(in0 + in1) + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.join.load = {\ + "in":2, "out":1,\ + "cpu":"${55*(in0 + in1) + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.globalreduce.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.globalreduce.load = {\ + "in":1, "out":1,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.reduceby.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.reduceby.load = {\ + "in":1, "out":1,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.sort.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.sort.load = {\ + "in":1, "out":1,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.clickhouse.tablesink.load.template = {\ + "type":"mathex", "in":1, "out":0,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +# NB: Not measured. +wayang.clickhouse.tablesink.load = {\ + "in":1, "out":0,\ + "cpu":"${55*in0 + 380000}",\ + "ram":"0",\ + "p":0.9\ +} + wayang.clickhouse.sqltostream.load.query.template = {\ "type":"mathex", "in":1, "out":1,\ "cpu":"?*out0 + ?"\ diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java new file mode 100644 index 000000000..af1197f74 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java @@ -0,0 +1,542 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse; + +import org.apache.wayang.api.DataQuantaBuilder; +import org.apache.wayang.api.JavaPlanBuilder; +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.clickhouse.operators.ClickHouseProjectionOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseTableSource; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end integration tests for the ClickHouse platform against a live + * ClickHouse server. The class is skipped (not failed) when the server is + * unreachable. + * + *

Environment variables: {@code CLICKHOUSE_HOST} (default {@code localhost}), + * {@code CLICKHOUSE_PORT} (default {@code 8123}), {@code CLICKHOUSE_USER} + * (default {@code default}), {@code CLICKHOUSE_PASSWORD} (default empty). + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class ClickHouseOperatorsIT { + + private static final String HOST = System.getenv().getOrDefault("CLICKHOUSE_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("CLICKHOUSE_PORT", "8123")); + private static final String USER = System.getenv().getOrDefault("CLICKHOUSE_USER", "default"); + private static final String PASSWORD = System.getenv().getOrDefault("CLICKHOUSE_PASSWORD", ""); + private static final String JDBC_URL = String.format("jdbc:clickhouse://%s:%d", HOST, PORT); + + private static final String DATABASE = "wayang_it"; + private static final String ORDERS = DATABASE + ".orders"; + private static final String CUSTOMERS = DATABASE + ".customers"; + private static final String SINK_TABLE_NAME = "operator_result"; + private static final String SINK_TABLE = DATABASE + "." + SINK_TABLE_NAME; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "cust_id", "name", "tier" + }; + private static final String JOIN_FLATTEN_NAME = "ClickHouse test-only join flatten"; + + private static boolean clickHouseAvailable = false; + + @BeforeAll + static void setUp() { + try (Connection c = jdbc()) { + Statement st = c.createStatement(); + st.execute("CREATE DATABASE IF NOT EXISTS " + DATABASE); + + st.execute("DROP TABLE IF EXISTS " + ORDERS); + st.execute("CREATE TABLE " + ORDERS + " (" + + "order_id Int64, customer_id Int64, region String, amount Float64" + + ") ENGINE = MergeTree ORDER BY order_id"); + st.execute("INSERT INTO " + ORDERS + " VALUES " + + "(1, 100, 'AMER', 2200.0)," + + "(2, 101, 'EMEA', 800.5)," + + "(3, 100, 'AMER', 680.5)," + + "(4, 102, 'APAC', 1500.0)," + + "(5, 101, 'EMEA', 1100.0)," + + "(6, 100, 'AMER', 950.25)"); + // Scale to 120000 rows so the optimizer prefers SQL pushdown. + st.execute("INSERT INTO " + ORDERS + + " SELECT order_id + (n * 10), customer_id, region, amount" + + " FROM " + ORDERS + " ARRAY JOIN range(1, 10000) AS n"); + st.execute("INSERT INTO " + ORDERS + + " SELECT order_id + 600000, customer_id, region, amount" + + " FROM " + ORDERS); + + st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); + st.execute("CREATE TABLE " + CUSTOMERS + " (" + + "cust_id Int64, name String, tier String" + + ") ENGINE = MergeTree ORDER BY cust_id"); + st.execute("INSERT INTO " + CUSTOMERS + " VALUES " + + "(100, 'Acme', 'GOLD')," + + "(101, 'Globex', 'SILVER')," + + "(102, 'Initech','BRONZE')"); + + clickHouseAvailable = true; + System.out.println("[ClickHouseOperatorsIT] Connected at " + JDBC_URL + + " — fixtures created."); + } catch (Exception e) { + System.err.println("[ClickHouseOperatorsIT] ClickHouse not available (" + + e.getMessage() + ") — skipping."); + } + } + + @AfterAll + static void tearDown() { + if (!clickHouseAvailable) { + return; + } + try (Connection c = jdbc()) { + Statement st = c.createStatement(); + st.execute("DROP TABLE IF EXISTS " + ORDERS); + st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); + st.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + } catch (Exception e) { + System.err.println("[ClickHouseOperatorsIT] cleanup failed: " + e.getMessage()); + } + } + + @Test + @Order(1) + void tableSource() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "TableSource should return all orders"); + } + + @Test + @Order(2) + void filter() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "60000 AMER orders expected"); + assertEquals(0, queryLong("SELECT countIf(region != 'AMER') FROM " + SINK_TABLE), + "all rows AMER"); + } + + @Test + @Order(3) + void projection() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "customer_id", "region", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "60000 AMER rows expected"); + assertEquals(2, queryLong( + "SELECT count(*) FROM system.columns " + + "WHERE database = '" + DATABASE + "' AND table = '" + SINK_TABLE_NAME + "'"), + "projection keeps only 2 columns"); + } + + @Test + @Order(4) + void join() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource orders = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + ClickHouseTableSource customers = new ClickHouseTableSource( + CUSTOMERS, "cust_id", "name", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation(ORDERS, "customer_id"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation(CUSTOMERS, "cust_id")); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = tableSink(JOIN_COLUMNS); + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join should yield one row per order"); + assertEquals(0, queryLong("SELECT countIf(customer_id != cust_id) FROM " + SINK_TABLE), + "joined customer IDs should match"); + } + + @Test + @Order(5) + void globalReduce() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("total_amount"); + src.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertSingleDoubleResult(144_625_000.0, "global reduce must collapse to a single row"); + } + + @Test + @Order(6) + void reduceBy() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(2)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "total_amount"); + src.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + Map sums = readRegionSums(); + assertEquals(3, sums.size(), "one row per region expected"); + assertEquals(76_615_000.0, sums.get("AMER"), 0.01); + assertEquals(38_010_000.0, sums.get("EMEA"), 0.01); + assertEquals(30_000_000.0, sums.get("APAC"), 0.01); + } + + @Test + @Order(7) + void sort() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(3)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "sort must not change the cardinality"); + assertEquals(680.5, queryDouble("SELECT min(amount) FROM " + SINK_TABLE), 0.001); + assertEquals(2200.0, queryDouble("SELECT max(amount) FROM " + SINK_TABLE), 0.001); + } + + @Test + @Order(8) + void tableSink() throws Exception { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + + ClickHouseTableSource src = new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", SINK_TABLE, + "order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + try (Connection c = jdbc()) { + ResultSet rs = c.createStatement().executeQuery( + "SELECT count(*), countIf(region != 'AMER') FROM " + SINK_TABLE); + rs.next(); + assertEquals(60000, rs.getLong(1), "sink table must hold all AMER orders"); + assertEquals(0, rs.getLong(2), "sink table must hold only AMER orders"); + } + } + + @Test + @Order(9) + void javaPlanBuilderReadTableFilterProjection() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + + new JavaPlanBuilder( + wayangContext(), "ClickHouse JavaPlanBuilder readTable integration test") + .readTable(new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable(SINK_TABLE, "overwrite", new String[]{"order_id", "amount"}, new Properties()); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "60000 projected AMER orders expected"); + } + + @Test + @Order(10) + void javaPlanBuilderReadTableJoin() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + + JavaPlanBuilder plan = new JavaPlanBuilder( + wayangContext(), "ClickHouse JavaPlanBuilder join integration test"); + DataQuantaBuilder orders = plan.readTable(new ClickHouseTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")); + DataQuantaBuilder customers = plan.readTable(new ClickHouseTableSource( + CUSTOMERS, "cust_id", "name", "tier")); + + orders + .join( + record -> new Record(record.getField(1)), + customers, + record -> new Record(record.getField(0))) + .withSqlUdfs(ORDERS, "customer_id", CUSTOMERS, "cust_id") + .map(new JoinFlattenFunction()) + .withName(JOIN_FLATTEN_NAME) + .writeTable(SINK_TABLE, "overwrite", JOIN_COLUMNS, new Properties()); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join should yield one row per order"); + assertEquals(0, queryLong("SELECT countIf(customer_id != cust_id) FROM " + SINK_TABLE), + "joined customer IDs should match"); + } + + private WayangContext wayangContext() { + Configuration config = new Configuration(); + config.setProperty("wayang.clickhouse.jdbc.url", JDBC_URL + "/" + DATABASE); + config.setProperty("wayang.clickhouse.jdbc.user", USER); + config.setProperty("wayang.clickhouse.jdbc.password", PASSWORD); + config.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(config) + .withPlugin(ClickHouse.plugin()); + } + + private TableSink tableSink(String... columnNames) { + return new TableSink<>(new Properties(), "overwrite", SINK_TABLE, columnNames); + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0), + right.getField(1), + right.getField(2)); + } + + private long queryLong(String sql) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery(sql)) { + rs.next(); + return rs.getLong(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private double queryDouble(String sql) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery(sql)) { + rs.next(); + return rs.getDouble(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private void assertSingleDoubleResult(double expected, String message) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + assertTrue(rs.next(), message); + assertEquals(expected, rs.getDouble(1), 0.01, message); + assertFalse(rs.next(), message); + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private Map readRegionSums() { + Map sums = new HashMap<>(); + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + while (rs.next()) { + sums.put(rs.getString(1), rs.getDouble(2)); + } + return sums; + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createClickHouseProjection().at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + ClickHousePlatform.getInstance())); + } + + private static ClickHouseProjectionOperator createClickHouseProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(JOIN_FLATTEN_NAME); + return new ClickHouseProjectionOperator((MapOperator) (MapOperator) projection); + } + } + + private static Connection jdbc() throws Exception { + Properties p = new Properties(); + p.setProperty("user", USER); + if (PASSWORD != null && !PASSWORD.isEmpty()) { + p.setProperty("password", PASSWORD); + } + return DriverManager.getConnection(JDBC_URL, p); + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java index b666fb2bd..1cd5db66f 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHousePluginTest.java @@ -20,7 +20,12 @@ import org.apache.wayang.clickhouse.ClickHouse; import org.apache.wayang.clickhouse.mapping.FilterMapping; +import org.apache.wayang.clickhouse.mapping.GlobalReduceMapping; +import org.apache.wayang.clickhouse.mapping.JoinMapping; import org.apache.wayang.clickhouse.mapping.ProjectionMapping; +import org.apache.wayang.clickhouse.mapping.ReduceByMapping; +import org.apache.wayang.clickhouse.mapping.SortMapping; +import org.apache.wayang.clickhouse.mapping.TableSinkMapping; import org.apache.wayang.clickhouse.platform.ClickHousePlatform; import org.apache.wayang.core.mapping.Mapping; import org.apache.wayang.java.platform.JavaPlatform; @@ -43,9 +48,14 @@ void pluginExposesPlatformsMappingsAndConversions() { assertEquals(2, ClickHouse.plugin().getChannelConversions().size()); Collection mappings = ClickHouse.plugin().getMappings(); - assertEquals(2, mappings.size()); + assertEquals(7, mappings.size()); assertTrue(mappings.stream().anyMatch(FilterMapping.class::isInstance)); assertTrue(mappings.stream().anyMatch(ProjectionMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(JoinMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(GlobalReduceMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(ReduceByMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(SortMapping.class::isInstance)); + assertTrue(mappings.stream().anyMatch(TableSinkMapping.class::isInstance)); } @Test diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java index 9a9b5abc4..6032ad088 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseSqlGenerationTest.java @@ -19,16 +19,21 @@ package org.apache.wayang.clickhouse.operators; import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; import org.apache.wayang.jdbc.compiler.FunctionCompiler; import org.junit.jupiter.api.Test; +import java.util.Properties; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; /** - * SQL-generation unit tests for PR-1 ClickHouse operators. These do not require - * a live ClickHouse server. + * SQL-generation unit tests for ClickHouse operators. These do not require a + * live ClickHouse server. */ class ClickHouseSqlGenerationTest { @@ -59,4 +64,80 @@ void projectionEmitsFieldList() { ClickHouseProjectionOperator projection = new ClickHouseProjectionOperator("order_id", "amount"); assertEquals("order_id, amount", projection.createSqlClause(null, compiler)); } + + @Test + void joinEmitsJoinClause() { + ClickHouseJoinOperator join = new ClickHouseJoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation("orders", "customer_id"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation("customers", "cust_id") + ); + assertEquals("JOIN customers ON customers.cust_id=orders.customer_id", + join.createSqlClause(null, compiler)); + } + + @Test + void globalReduceEmitsAggregate() { + ClickHouseGlobalReduceOperator reduce = new ClickHouseGlobalReduceOperator( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount") + ); + assertEquals("SUM(amount) AS total_amount", reduce.createSqlClause(null, compiler)); + } + + @Test + void reduceByEmitsKeyAndAggregate() { + ClickHouseReduceByOperator reduceBy = new ClickHouseReduceByOperator( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount") + ); + assertEquals("region,SUM(amount) AS total_amount", reduceBy.createSqlClause(null, compiler)); + } + + @Test + void sortEmitsOrderBy() { + ClickHouseSortOperator sort = new ClickHouseSortOperator( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC") + ); + assertEquals(" ORDER BY amount ASC", sort.createSqlClause(null, compiler)); + } + + @Test + void tableSinkOverwriteIncludesEngine() { + ClickHouseTableSinkOperator sink = new ClickHouseTableSinkOperator( + "wayang_it.result", new String[]{"order_id", "amount"}); + sink.setMode("overwrite"); + assertEquals( + "CREATE TABLE \"wayang_it\".\"result\" ENGINE = MergeTree ORDER BY tuple() AS", + sink.createSqlClause(null, compiler)); + } + + @Test + void tableSinkInsertOmitsEngine() { + ClickHouseTableSinkOperator sink = new ClickHouseTableSinkOperator( + "wayang_it.result", new String[]{"order_id", "amount"}); + sink.setMode("append"); + assertEquals("INSERT INTO \"wayang_it\".\"result\"", sink.createSqlClause(null, compiler)); + } + + @Test + void tableSinkReadsEngineFromProperties() { + Properties props = new Properties(); + props.setProperty(ClickHouseTableSinkOperator.ENGINE_PROPERTY, + "ReplacingMergeTree ORDER BY order_id"); + TableSink logical = new TableSink<>( + props, "overwrite", "wayang_it.result", "order_id", "amount"); + ClickHouseTableSinkOperator sink = new ClickHouseTableSinkOperator(logical); + assertEquals( + "CREATE TABLE \"wayang_it\".\"result\" ENGINE = ReplacingMergeTree ORDER BY order_id AS", + sink.createSqlClause(null, compiler)); + } } From 63a0b000720a676dec28f0ba3090f3417ce897d1 Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Fri, 13 Nov 2026 20:35:53 +0530 Subject: [PATCH 3/6] Fix ClickHouse sink copy/ENGINE docs and harden the IT fixtures. Preserve custom ENGINE across createCopy, clarify that the engine key is a TableSink property, document ClickHouseOperatorsIT, and scale fixtures with CROSS JOIN numbers(). --- INTEGRATION-TESTS.md | 49 +++++++++++++------ wayang-applications/clickhouse.md | 4 ++ wayang-platforms/wayang-clickhouse/README.md | 13 +++-- .../ClickHouseTableSinkOperator.java | 18 +++++-- .../wayang-clickhouse-defaults.properties | 7 +-- .../clickhouse/ClickHouseOperatorsIT.java | 5 +- .../ClickHouseTableSinkCopyTest.java | 48 ++++++++++++++++++ 7 files changed, 115 insertions(+), 29 deletions(-) create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkCopyTest.java diff --git a/INTEGRATION-TESTS.md b/INTEGRATION-TESTS.md index 2567d7c54..b1c0575c4 100644 --- a/INTEGRATION-TESTS.md +++ b/INTEGRATION-TESTS.md @@ -16,16 +16,16 @@ limitations under the License. --> -# Integration Tests – Presto +# Integration Tests – SQL Platforms -End-to-end tests that drive the Presto platform operators (TableSource, Filter, -Projection, Join) through the Wayang API against a live PrestoDB cluster. +End-to-end tests that drive JDBC platform operators through the Wayang API +against a user-managed database. Each suite **skips** (it does not fail) when +the server is unreachable. | Test | Module | Needs | |------|--------|-------| -| `AllOperatorsIT` | `wayang-presto` | a local Presto (Docker) | - -The test **skips** (it does not fail) when Presto is unreachable. +| `PrestoOperatorsIT` | `wayang-presto` | a Presto deployment | +| `ClickHouseOperatorsIT` | `wayang-clickhouse` | a ClickHouse deployment | --- @@ -74,17 +74,36 @@ Connection settings are supplied through `PRESTO_HOST`, `PRESTO_PORT`, and --- +## ClickHouse + +The test creates and seeds its own `wayang_it` database/tables in a user-managed +ClickHouse deployment, then drops the fixtures afterward. + +```bash +CLICKHOUSE_HOST=localhost CLICKHOUSE_PORT=8123 CLICKHOUSE_USER=default \ +JAVA_HOME=/path/to/jdk-17 \ +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-clickhouse -am \ + -Dtest=ClickHouseOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false \ + -Drat.skip=true -Dlicense.skip=true -Dmaven.javadoc.skip=true test +``` + +Connection settings are supplied through `CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, +`CLICKHOUSE_USER`, and optional `CLICKHOUSE_PASSWORD`. Applications use +`wayang.clickhouse.jdbc.*` properties instead. + ## Notes - **Pushdown is cost-gated.** On tiny tables Wayang's optimizer prefers a full scan + Java-side filter/projection, so pushdown only appears once a table is - large enough (hence the test scales to 120k rows). Each test asserts both correct - results and that the expected SQL reached Presto (`system.runtime.queries`). + large enough (hence the test scales to 120k rows). Presto tests also assert + that the expected SQL reached the server (`system.runtime.queries`). - **Join.** A JDBC join is verified through the operator's SQL-clause contract - executed on Presto, not the high-level `WayangContext` API — the logical - `JoinOperator` emits `Tuple2`, which cannot connect to a `Record` - sink before the SQL pushdown flattens it. -- **Trailing semicolons.** Presto's SQL parser rejects a trailing `;` in - `executeQuery`, so this branch also carries the jdbc-template change that stops - emitting one (shared with the other JDBC platforms; Postgres/SQLite tolerate its - absence). + executed on the platform, not the high-level `WayangContext` API alone — the + logical `JoinOperator` emits `Tuple2`, which cannot connect to a + `Record` sink before the SQL pushdown flattens it. +- **Trailing semicolons.** Strict SQL parsers (Presto, Trino, BigQuery, + ClickHouse JDBC in some modes) reject a trailing `;` in `executeQuery`, so + `wayang-jdbc-template` does not emit one. +- **ClickHouse table sinks.** Overwrite mode emits + `CREATE TABLE ... ENGINE = MergeTree ORDER BY tuple() AS SELECT ...`. diff --git a/wayang-applications/clickhouse.md b/wayang-applications/clickhouse.md index df01ff886..51e84aa1c 100644 --- a/wayang-applications/clickhouse.md +++ b/wayang-applications/clickhouse.md @@ -64,6 +64,10 @@ wayang.clickhouse.demo.table = default.orders Change the endpoint, credentials, database, and table for your deployment. +To override the table-sink ENGINE for `writeTable(..., "overwrite", ...)`, put +`wayang.clickhouse.tablesink.engine` on the `Properties` argument (default is +`MergeTree ORDER BY tuple()`). + ## Build and run Build the application and its dependencies from the repository root: diff --git a/wayang-platforms/wayang-clickhouse/README.md b/wayang-platforms/wayang-clickhouse/README.md index 8f1ee8fc5..6a387e308 100644 --- a/wayang-platforms/wayang-clickhouse/README.md +++ b/wayang-platforms/wayang-clickhouse/README.md @@ -15,8 +15,6 @@ Configure the endpoint and credentials in a Wayang properties file: wayang.clickhouse.jdbc.url = jdbc:clickhouse://clickhouse.example.com:8123/default wayang.clickhouse.jdbc.user = default wayang.clickhouse.jdbc.password = -# Optional ENGINE clause for CREATE TABLE ... AS SELECT (overwrite sinks) -wayang.clickhouse.tablesink.engine = MergeTree ORDER BY tuple() ``` The runnable filter/projection example and its table requirements are documented @@ -25,12 +23,17 @@ in [`wayang-applications/clickhouse.md`](../../wayang-applications/clickhouse.md ## Notes - Table sinks in `overwrite` mode emit - `CREATE TABLE ENGINE = AS SELECT ...` because ClickHouse requires - an engine. Override the engine via `wayang.clickhouse.tablesink.engine` or the - same key on the `TableSink` `Properties`. + `CREATE TABLE ENGINE = MergeTree ORDER BY tuple() AS SELECT ...` because + ClickHouse requires an engine. Override per sink by putting + `wayang.clickhouse.tablesink.engine` on the `TableSink` / `writeTable` + `Properties` (not the JDBC connection properties above). - ClickHouse identifiers are case-sensitive. Prefer `database.table` names. - Joins build the hash table from the right input; nullable join behaviour follows the server's `join_use_nulls` setting (default `0`). +- JDBC types such as `UInt64`, `Date`/`DateTime`, `Nullable(T)`, and + `LowCardinality(String)` are returned via `ResultSet.getObject` into + Wayang `Record`s as the driver provides them. + ## Integration tests diff --git a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java index d3f28e4fc..72e096c76 100644 --- a/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java +++ b/wayang-platforms/wayang-clickhouse/src/main/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkOperator.java @@ -29,9 +29,10 @@ * ClickHouse implementation of the {@link JdbcTableSinkOperator}. * *

ClickHouse requires an {@code ENGINE} clause for {@code CREATE TABLE ... AS - * SELECT}. The default is {@value #DEFAULT_ENGINE}; override via the - * {@code engine} constructor argument or the - * {@code wayang.clickhouse.tablesink.engine} property when constructing the sink. + * SELECT}. The default is {@value #DEFAULT_ENGINE}. Override via the + * {@code engine} constructor argument, or by setting {@link #ENGINE_PROPERTY} + * on the logical {@link TableSink}'s {@link java.util.Properties} (for example + * the props passed to {@code writeTable(...)}). */ public class ClickHouseTableSinkOperator extends JdbcTableSinkOperator implements ClickHouseExecutionOperator { @@ -52,13 +53,22 @@ public ClickHouseTableSinkOperator(String tableName, String[] columnNames, Strin public ClickHouseTableSinkOperator(TableSink that) { super(that); - this.engine = engineFrom(that); + if (that instanceof ClickHouseTableSinkOperator) { + this.engine = ((ClickHouseTableSinkOperator) that).engine; + } else { + this.engine = engineFrom(that); + } } public String getEngine() { return this.engine; } + @Override + protected ClickHouseTableSinkOperator createCopy() { + return new ClickHouseTableSinkOperator(this); + } + @Override public String createSqlClause(Connection connection, FunctionCompiler compiler) { String mode = this.getMode(); diff --git a/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties index 5befbf0fb..961da5f7c 100644 --- a/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties +++ b/wayang-platforms/wayang-clickhouse/src/main/resources/wayang-clickhouse-defaults.properties @@ -20,9 +20,10 @@ # wayang.clickhouse.jdbc.user = default # wayang.clickhouse.jdbc.password = -# Table sink ENGINE used by CREATE TABLE ... AS SELECT (overwrite mode). -# May also be set on a TableSink Properties object under the same key. -wayang.clickhouse.tablesink.engine = MergeTree ORDER BY tuple() +# Documented default ENGINE for overwrite table sinks. The operator reads this +# key from the TableSink/writeTable Properties (see ClickHouseTableSinkOperator), +# not from the JDBC connection Configuration. +# wayang.clickhouse.tablesink.engine = MergeTree ORDER BY tuple() # Cost function parameters # NB: Not measured. Placeholder values so the optimizer can schedule ClickHouse diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java index af1197f74..48420aaaf 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java @@ -118,9 +118,10 @@ static void setUp() { + "(5, 101, 'EMEA', 1100.0)," + "(6, 100, 'AMER', 950.25)"); // Scale to 120000 rows so the optimizer prefers SQL pushdown. + // numbers(9999) yields 0..9998; offset by 1 to mirror Trino's sequence(1,9999). st.execute("INSERT INTO " + ORDERS - + " SELECT order_id + (n * 10), customer_id, region, amount" - + " FROM " + ORDERS + " ARRAY JOIN range(1, 10000) AS n"); + + " SELECT order_id + ((number + 1) * 10), customer_id, region, amount" + + " FROM " + ORDERS + " CROSS JOIN numbers(9999)"); st.execute("INSERT INTO " + ORDERS + " SELECT order_id + 600000, customer_id, region, amount" + " FROM " + ORDERS); diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkCopyTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkCopyTest.java new file mode 100644 index 000000000..5baa4d344 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseTableSinkCopyTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; + +/** + * Ensures {@link ClickHouseTableSinkOperator#createCopy()} preserves the ENGINE. + */ +class ClickHouseTableSinkCopyTest { + + @Test + void createCopyPreservesCustomEngine() { + ClickHouseTableSinkOperator original = new ClickHouseTableSinkOperator( + "wayang_it.result", + new String[]{"order_id"}, + "ReplacingMergeTree ORDER BY order_id"); + original.setMode("overwrite"); + + ClickHouseTableSinkOperator copy = original.createCopy(); + + assertNotSame(original, copy); + assertEquals("ReplacingMergeTree ORDER BY order_id", copy.getEngine()); + assertEquals("overwrite", copy.getMode()); + assertEquals( + "CREATE TABLE \"wayang_it\".\"result\" ENGINE = ReplacingMergeTree ORDER BY order_id AS", + copy.createSqlClause(null, null)); + } +} From 01c48a3e88a8561e1abcdf49c23734a328975d72 Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Fri, 13 Nov 2026 20:50:08 +0530 Subject: [PATCH 4/6] Fix ClickHouse driver ANTLR clash and make ClickHouseOperatorsIT run. Bump clickhouse-jdbc to 0.11.1, whose shaded jar relocates ANTLR, and exclude its transitive dependencies. The 0.9.0 jar shipped an unrelocated ANTLR 4.13 that shadowed Wayang's 4.9.3 runtime and broke the mathex parser. Always pass the password property (the driver requires it even when empty) and alias the numbers() table function in the fixture SQL, so the IT actually runs against a passwordless ClickHouse instead of silently skipping. --- wayang-platforms/wayang-clickhouse/pom.xml | 10 +++++++++- .../wayang/clickhouse/ClickHouseOperatorsIT.java | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/wayang-platforms/wayang-clickhouse/pom.xml b/wayang-platforms/wayang-clickhouse/pom.xml index 6054339af..fbe00684a 100644 --- a/wayang-platforms/wayang-clickhouse/pom.xml +++ b/wayang-platforms/wayang-clickhouse/pom.xml @@ -35,7 +35,7 @@ org.apache.wayang.platform.clickhouse - 0.9.0 + 0.11.1 @@ -44,6 +44,14 @@ clickhouse-jdbc ${clickhouse.version} all + + + + * + * + + org.apache.wayang diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java index 48420aaaf..21c44d790 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java @@ -121,7 +121,7 @@ static void setUp() { // numbers(9999) yields 0..9998; offset by 1 to mirror Trino's sequence(1,9999). st.execute("INSERT INTO " + ORDERS + " SELECT order_id + ((number + 1) * 10), customer_id, region, amount" - + " FROM " + ORDERS + " CROSS JOIN numbers(9999)"); + + " FROM " + ORDERS + " CROSS JOIN numbers(9999) AS n"); st.execute("INSERT INTO " + ORDERS + " SELECT order_id + 600000, customer_id, region, amount" + " FROM " + ORDERS); @@ -535,9 +535,9 @@ private static ClickHouseProjectionOperator createClickHouseProjection() { private static Connection jdbc() throws Exception { Properties p = new Properties(); p.setProperty("user", USER); - if (PASSWORD != null && !PASSWORD.isEmpty()) { - p.setProperty("password", PASSWORD); - } + // clickhouse-jdbc 0.9.x refuses to connect unless a password property is + // present, even an empty one for the passwordless default user. + p.setProperty("password", PASSWORD); return DriverManager.getConnection(JDBC_URL, p); } } From 31af1203249f64aa6983c8109b0058548ef25b02 Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Fri, 13 Nov 2026 20:52:24 +0530 Subject: [PATCH 5/6] Skip ClickHouseOperatorsIT only when the server is unreachable. Probe the connection with SELECT 1 first and skip the class only if that fails. Fixture statements now run outside the catch, so a broken fixture fails the run instead of being reported as a skip. --- .../clickhouse/ClickHouseOperatorsIT.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java index 21c44d790..2de8caa67 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java @@ -101,9 +101,19 @@ class ClickHouseOperatorsIT { private static boolean clickHouseAvailable = false; @BeforeAll - static void setUp() { - try (Connection c = jdbc()) { - Statement st = c.createStatement(); + static void setUp() throws Exception { + // Only an unreachable server skips the class. Once connected, a failing + // fixture statement is a real error and must fail the run. + try (Connection c = jdbc(); Statement st = c.createStatement()) { + st.execute("SELECT 1"); + } catch (Exception e) { + System.err.println("[ClickHouseOperatorsIT] ClickHouse not available (" + + e.getMessage() + ") — skipping."); + return; + } + clickHouseAvailable = true; + + try (Connection c = jdbc(); Statement st = c.createStatement()) { st.execute("CREATE DATABASE IF NOT EXISTS " + DATABASE); st.execute("DROP TABLE IF EXISTS " + ORDERS); @@ -135,12 +145,8 @@ static void setUp() { + "(101, 'Globex', 'SILVER')," + "(102, 'Initech','BRONZE')"); - clickHouseAvailable = true; System.out.println("[ClickHouseOperatorsIT] Connected at " + JDBC_URL + " — fixtures created."); - } catch (Exception e) { - System.err.println("[ClickHouseOperatorsIT] ClickHouse not available (" - + e.getMessage() + ") — skipping."); } } From 4e323df03c83a4eeeeeceabf1d20a6952239699a Mon Sep 17 00:00:00 2001 From: ekas_7 Date: Fri, 13 Nov 2026 20:55:57 +0530 Subject: [PATCH 6/6] Add ClickHouse mapping, operator and type-mapping tests. ClickHouseMappingsTest applies every mapping to a logical plan and checks the replacement operator, including the cases that must not be mapped. ClickHouseOperatorsTest covers copies, channels, the sink ENGINE fallbacks and that the defaults file provides a cost model for every operator. ClickHouseOperatorsIT now reads UInt64, Date, DateTime, Nullable and LowCardinality columns into Records; the README documents the Java types. --- wayang-platforms/wayang-clickhouse/README.md | 15 +- .../clickhouse/ClickHouseOperatorsIT.java | 52 +++ .../mapping/ClickHouseMappingsTest.java | 297 ++++++++++++++++++ .../operators/ClickHouseOperatorsTest.java | 238 ++++++++++++++ 4 files changed, 599 insertions(+), 3 deletions(-) create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/mapping/ClickHouseMappingsTest.java create mode 100644 wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseOperatorsTest.java diff --git a/wayang-platforms/wayang-clickhouse/README.md b/wayang-platforms/wayang-clickhouse/README.md index 6a387e308..63e6c126e 100644 --- a/wayang-platforms/wayang-clickhouse/README.md +++ b/wayang-platforms/wayang-clickhouse/README.md @@ -30,9 +30,18 @@ in [`wayang-applications/clickhouse.md`](../../wayang-applications/clickhouse.md - ClickHouse identifiers are case-sensitive. Prefer `database.table` names. - Joins build the hash table from the right input; nullable join behaviour follows the server's `join_use_nulls` setting (default `0`). -- JDBC types such as `UInt64`, `Date`/`DateTime`, `Nullable(T)`, and - `LowCardinality(String)` are returned via `ResultSet.getObject` into - Wayang `Record`s as the driver provides them. +- Column values reach Wayang `Record`s via `ResultSet.getObject`, as the driver + provides them. With `clickhouse-jdbc` 0.11.1 (covered by + `ClickHouseOperatorsIT`): + + | ClickHouse type | Java type in the `Record` | + |---|---| + | `Int32` | `Integer` | + | `UInt64` | `java.math.BigInteger` | + | `Date` | `java.sql.Date` | + | `DateTime` | `java.sql.Timestamp` | + | `Nullable(String)` | `String` or `null` | + | `LowCardinality(String)` | `String` | ## Integration tests diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java index 2de8caa67..0a35dcfd8 100644 --- a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/ClickHouseOperatorsIT.java @@ -26,6 +26,7 @@ import org.apache.wayang.basic.operators.FilterOperator; import org.apache.wayang.basic.operators.GlobalReduceOperator; import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; import org.apache.wayang.basic.operators.MapOperator; import org.apache.wayang.basic.operators.ReduceByOperator; import org.apache.wayang.basic.operators.SortOperator; @@ -48,6 +49,7 @@ import org.apache.wayang.core.plan.wayangplan.WayangPlan; import org.apache.wayang.core.types.DataSetType; import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.java.Java; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; @@ -56,18 +58,23 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +import java.math.BigInteger; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -93,6 +100,7 @@ class ClickHouseOperatorsIT { private static final String CUSTOMERS = DATABASE + ".customers"; private static final String SINK_TABLE_NAME = "operator_result"; private static final String SINK_TABLE = DATABASE + "." + SINK_TABLE_NAME; + private static final String TYPES = DATABASE + ".types"; private static final String[] JOIN_COLUMNS = { "order_id", "customer_id", "region", "amount", "cust_id", "name", "tier" }; @@ -145,6 +153,15 @@ static void setUp() throws Exception { + "(101, 'Globex', 'SILVER')," + "(102, 'Initech','BRONZE')"); + st.execute("DROP TABLE IF EXISTS " + TYPES); + st.execute("CREATE TABLE " + TYPES + " (" + + "id Int32, big UInt64, day Date, ts DateTime('UTC')," + + " note Nullable(String), tag LowCardinality(String)" + + ") ENGINE = MergeTree ORDER BY id"); + st.execute("INSERT INTO " + TYPES + " VALUES " + + "(1, 18446744073709551615, '2026-01-10', '2026-01-10 12:30:00', NULL, 'gold')," + + "(2, 7, '2026-01-11', '2026-01-11 00:00:00', 'hello', 'silver')"); + System.out.println("[ClickHouseOperatorsIT] Connected at " + JDBC_URL + " — fixtures created."); } @@ -160,6 +177,7 @@ static void tearDown() { st.execute("DROP TABLE IF EXISTS " + ORDERS); st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); st.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + st.execute("DROP TABLE IF EXISTS " + TYPES); } catch (Exception e) { System.err.println("[ClickHouseOperatorsIT] cleanup failed: " + e.getMessage()); } @@ -406,6 +424,40 @@ record -> new Record(record.getField(0))) "joined customer IDs should match"); } + @Test + @Order(11) + void clickHouseTypesReachJavaRecords() { + Assumptions.assumeTrue(clickHouseAvailable, "ClickHouse not reachable"); + + List records = new ArrayList<>(); + ClickHouseTableSource src = new ClickHouseTableSource( + TYPES, "id", "big", "day", "ts", "note", "tag"); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(records, Record.class); + src.connectTo(0, sink, 0); + + wayangContext().withPlugin(Java.basicPlugin()).execute(new WayangPlan(sink)); + + assertEquals(2, records.size(), "both rows expected"); + records.sort((a, b) -> Integer.compare((Integer) a.getField(0), (Integer) b.getField(0))); + Record first = records.get(0); + Record second = records.get(1); + + // Int32 -> Integer + assertEquals(1, first.getField(0)); + // UInt64 -> BigInteger, so values above Long.MAX_VALUE survive. + assertEquals(new BigInteger("18446744073709551615"), first.getField(1)); + assertEquals(BigInteger.valueOf(7), second.getField(1)); + // Date -> java.sql.Date, DateTime -> java.sql.Timestamp + assertInstanceOf(java.sql.Date.class, first.getField(2)); + assertInstanceOf(java.sql.Timestamp.class, first.getField(3)); + // Nullable(String) -> null or String + assertNull(first.getField(4)); + assertEquals("hello", second.getField(4)); + // LowCardinality(String) -> String + assertEquals("gold", first.getField(5)); + assertEquals("silver", second.getField(5)); + } + private WayangContext wayangContext() { Configuration config = new Configuration(); config.setProperty("wayang.clickhouse.jdbc.url", JDBC_URL + "/" + DATABASE); diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/mapping/ClickHouseMappingsTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/mapping/ClickHouseMappingsTest.java new file mode 100644 index 000000000..906c80495 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/mapping/ClickHouseMappingsTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.clickhouse.operators.ClickHouseFilterOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseGlobalReduceOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseJoinOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseProjectionOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseReduceByOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseSortOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseTableSinkOperator; +import org.apache.wayang.clickhouse.operators.ClickHouseTableSource; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Applies each ClickHouse {@link Mapping} to a small logical plan and checks + * that the matching operator is replaced by its ClickHouse counterpart. No + * ClickHouse server is needed. + */ +class ClickHouseMappingsTest { + + private final FunctionCompiler compiler = new FunctionCompiler(); + + @Test + void filterWithSqlImplementationIsMapped() { + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>((Record r) -> true, Record.class) + .withSqlImplementation("region = 'AMER'")); + + Operator mapped = this.applyToUnary(new FilterMapping(), filter); + + ClickHouseFilterOperator clickHouseFilter = assertInstanceOf(ClickHouseFilterOperator.class, mapped); + assertEquals("region = 'AMER'", clickHouseFilter.createSqlClause(null, this.compiler)); + assertSame(ClickHousePlatform.getInstance(), clickHouseFilter.getPlatform()); + } + + @Test + void filterWithoutSqlImplementationIsNotMapped() { + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>((Record r) -> true, Record.class)); + + Operator mapped = this.applyToUnary(new FilterMapping(), filter); + + assertSame(filter, mapped); + } + + @Test + void projectionIsMapped() { + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "amount"), "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new ProjectionMapping(), projection); + + ClickHouseProjectionOperator clickHouseProjection = + assertInstanceOf(ClickHouseProjectionOperator.class, mapped); + assertEquals("region, amount", clickHouseProjection.createSqlClause(null, this.compiler)); + } + + @Test + void mapWithoutProjectionDescriptorIsNotMapped() { + MapOperator map = new MapOperator<>( + new TransformationDescriptor<>((Record r) -> r, Record.class, Record.class), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new ProjectionMapping(), map); + + assertSame(map, mapped); + } + + @Test + void globalReduceWithSqlImplementationIsMapped() { + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new GlobalReduceMapping(), reduce); + + ClickHouseGlobalReduceOperator clickHouseReduce = + assertInstanceOf(ClickHouseGlobalReduceOperator.class, mapped); + assertEquals("SUM(amount) AS total_amount", clickHouseReduce.createSqlClause(null, this.compiler)); + } + + @Test + void globalReduceWithoutSqlImplementationIsNotMapped() { + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new GlobalReduceMapping(), reduce); + + assertSame(reduce, mapped); + } + + @Test + void reduceByWithSqlImplementationIsMapped() { + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>((Record r) -> r, Record.class, Record.class) + .withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new ReduceByMapping(), reduceBy); + + ClickHouseReduceByOperator clickHouseReduceBy = + assertInstanceOf(ClickHouseReduceByOperator.class, mapped); + assertEquals("region,SUM(amount) AS total_amount", + clickHouseReduceBy.createSqlClause(null, this.compiler)); + } + + @Test + void reduceByWithoutSqlAggregateIsNotMapped() { + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>((Record r) -> r, Record.class, Record.class) + .withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new ReduceByMapping(), reduceBy); + + assertSame(reduceBy, mapped); + } + + @Test + void sortWithSqlImplementationIsMapped() { + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>((Record r) -> r, Record.class, Record.class) + .withSqlImplementation("amount", "DESC"), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new SortMapping(), sort); + + ClickHouseSortOperator clickHouseSort = assertInstanceOf(ClickHouseSortOperator.class, mapped); + assertEquals(" ORDER BY amount DESC", clickHouseSort.createSqlClause(null, this.compiler)); + } + + @Test + void sortWithoutSqlImplementationIsNotMapped() { + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>((Record r) -> r, Record.class, Record.class), + DataSetType.createDefault(Record.class)); + + Operator mapped = this.applyToUnary(new SortMapping(), sort); + + assertSame(sort, mapped); + } + + @Test + void joinWithSqlImplementationIsMapped() { + JoinOperator join = this.createJoin(true); + + Operator mapped = this.applyToJoin(join); + + ClickHouseJoinOperator clickHouseJoin = assertInstanceOf(ClickHouseJoinOperator.class, mapped); + assertEquals("JOIN customers ON customers.cust_id=orders.customer_id", + clickHouseJoin.createSqlClause(null, this.compiler)); + } + + @Test + void joinWithoutSqlImplementationIsNotMapped() { + JoinOperator join = this.createJoin(false); + + Operator mapped = this.applyToJoin(join); + + assertSame(join, mapped); + } + + @Test + void tableSinkIsMappedAndKeepsEngineProperty() { + Properties properties = new Properties(); + properties.setProperty(ClickHouseTableSinkOperator.ENGINE_PROPERTY, "Log"); + TableSink sink = new TableSink<>(properties, "overwrite", "wayang_it.result", "order_id"); + new ClickHouseTableSource("orders", "order_id").connectTo(0, sink, 0); + WayangPlan plan = new WayangPlan(sink); + + this.apply(new TableSinkMapping(), plan); + + ClickHouseTableSinkOperator clickHouseSink = assertInstanceOf( + ClickHouseTableSinkOperator.class, plan.getSinks().iterator().next()); + assertEquals("wayang_it.result", clickHouseSink.getTableName()); + assertEquals("CREATE TABLE \"wayang_it\".\"result\" ENGINE = Log AS", + clickHouseSink.createSqlClause(null, this.compiler)); + } + + @Test + void everyMappingTargetsTheClickHousePlatform() { + assertFalse(Mappings.ALL.isEmpty()); + for (Mapping mapping : Mappings.ALL) { + for (PlanTransformation transformation : mapping.getTransformations()) { + assertEquals(1, transformation.getTargetPlatforms().size()); + assertSame(ClickHousePlatform.getInstance(), + transformation.getTargetPlatforms().iterator().next()); + } + } + } + + /** + * Builds {@code source -> operator -> sink}, applies the mapping and + * returns the operator that now feeds the sink. + */ + private Operator applyToUnary(Mapping mapping, Operator operator) { + ClickHouseTableSource source = new ClickHouseTableSource("orders", "order_id", "region", "amount"); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(new ArrayList<>(), Record.class); + source.connectTo(0, operator, 0); + operator.connectTo(0, sink, 0); + WayangPlan plan = new WayangPlan(sink); + + this.apply(mapping, plan); + + return plan.getSinks().iterator().next().getEffectiveOccupant(0).getOwner(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Operator applyToJoin(JoinOperator join) { + ClickHouseTableSource orders = new ClickHouseTableSource("orders", "order_id", "customer_id"); + ClickHouseTableSource customers = new ClickHouseTableSource("customers", "cust_id", "name"); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink( + new ArrayList<>(), DataSetType.createDefaultUnchecked(Tuple2.class)); + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, sink, 0); + WayangPlan plan = new WayangPlan(sink); + + this.apply(new JoinMapping(), plan); + + return plan.getSinks().iterator().next().getEffectiveOccupant(0).getOwner(); + } + + private JoinOperator createJoin(boolean withSql) { + TransformationDescriptor leftKey = new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class); + TransformationDescriptor rightKey = new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class); + if (withSql) { + leftKey = leftKey.withSqlImplementation("orders", "customer_id"); + rightKey = rightKey.withSqlImplementation("customers", "cust_id"); + } + return new JoinOperator<>(leftKey, rightKey); + } + + private void apply(Mapping mapping, WayangPlan plan) { + for (PlanTransformation transformation : mapping.getTransformations()) { + transformation.thatReplaces().transform(plan, Operator.FIRST_EPOCH + 1); + } + } +} diff --git a/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseOperatorsTest.java b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseOperatorsTest.java new file mode 100644 index 000000000..90a5cae62 --- /dev/null +++ b/wayang-platforms/wayang-clickhouse/src/test/java/org/apache/wayang/clickhouse/operators/ClickHouseOperatorsTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.clickhouse.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.clickhouse.ClickHouse; +import org.apache.wayang.clickhouse.channels.ChannelConversions; +import org.apache.wayang.clickhouse.platform.ClickHousePlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.ExecutionOperator; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; +import org.apache.wayang.spark.channels.RddChannel; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for the ClickHouse execution operators beyond SQL generation: + * copies, channels, cost-model configuration and the sink ENGINE fallbacks. + * No ClickHouse server is needed. + */ +class ClickHouseOperatorsTest { + + private final FunctionCompiler compiler = new FunctionCompiler(); + + private static TransformationDescriptor key(String... sql) { + TransformationDescriptor descriptor = new TransformationDescriptor<>( + (Record r) -> r, Record.class, Record.class); + return sql.length == 2 ? descriptor.withSqlImplementation(sql[0], sql[1]) : descriptor; + } + + private static ReduceDescriptor sum() { + return new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"); + } + + /** One instance of every operator that reads from and writes to the SQL channel. */ + private static List queryOperators() { + return Arrays.asList( + new ClickHouseFilterOperator( + new PredicateDescriptor<>((Record r) -> true, Record.class) + .withSqlImplementation("region = 'AMER'")), + new ClickHouseProjectionOperator("region", "amount"), + new ClickHouseJoinOperator<>(key("orders", "customer_id"), key("customers", "cust_id")), + new ClickHouseGlobalReduceOperator(sum()), + new ClickHouseReduceByOperator(key("region", "region"), sum()), + new ClickHouseSortOperator(key("amount", "ASC"))); + } + + @Test + void copiesKeepTypeAndSqlClause() { + for (JdbcExecutionOperator operator : queryOperators()) { + ExecutionOperator copy = operator.copy(); + + assertNotSame(operator, copy); + assertSame(operator.getClass(), copy.getClass()); + assertEquals( + operator.createSqlClause(null, this.compiler), + ((JdbcExecutionOperator) copy).createSqlClause(null, this.compiler)); + } + } + + @Test + void queryOperatorsUseTheClickHouseSqlChannel() { + ChannelDescriptor sqlChannel = ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor(); + for (JdbcExecutionOperator operator : queryOperators()) { + assertSame(ClickHousePlatform.getInstance(), operator.getPlatform()); + assertEquals(Collections.singletonList(sqlChannel), operator.getSupportedInputChannels(0)); + assertEquals(Collections.singletonList(sqlChannel), operator.getSupportedOutputChannels(0)); + } + } + + @Test + void tableSourceHasNoInputChannels() { + ClickHouseTableSource source = new ClickHouseTableSource("analytics.orders", "order_id"); + + assertThrows(UnsupportedOperationException.class, () -> source.getSupportedInputChannels(0)); + assertEquals( + Collections.singletonList(ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor()), + source.getSupportedOutputChannels(0)); + } + + @Test + void tableSourceCopyConstructorKeepsTable() { + ClickHouseTableSource source = new ClickHouseTableSource("analytics.orders", "order_id", "region"); + + ClickHouseTableSource copy = new ClickHouseTableSource(source); + + assertEquals("analytics.orders", copy.getTableName()); + assertEquals("analytics.orders", copy.createSqlClause(null, this.compiler)); + } + + @Test + void projectionAcceptsDescriptor() { + ClickHouseProjectionOperator projection = new ClickHouseProjectionOperator( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "amount"), "order_id", "amount")); + + assertEquals("order_id, amount", projection.createSqlClause(null, this.compiler)); + } + + @Test + void tableSinkHasNoOutputChannels() { + ClickHouseTableSinkOperator sink = new ClickHouseTableSinkOperator( + "wayang_it.result", new String[]{"order_id"}); + + assertEquals( + Collections.singletonList(ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor()), + sink.getSupportedInputChannels(0)); + assertThrows(UnsupportedOperationException.class, () -> sink.getSupportedOutputChannels(0)); + } + + @Test + void tableSinkFallsBackToDefaultEngine() { + String[] columns = {"order_id"}; + + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator("t", columns).getEngine()); + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator("t", columns, null).getEngine()); + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator("t", columns, " ").getEngine()); + assertEquals("Log", new ClickHouseTableSinkOperator("t", columns, " Log ").getEngine()); + } + + @Test + void tableSinkEngineFromLogicalSinkProperties() { + // No properties at all. + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator( + new TableSink(null, "overwrite", "t", "order_id")).getEngine()); + + // Properties without the engine key. + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator( + new TableSink(new Properties(), "overwrite", "t", "order_id")).getEngine()); + + // Blank value is ignored, surrounding whitespace is trimmed. + Properties blank = new Properties(); + blank.setProperty(ClickHouseTableSinkOperator.ENGINE_PROPERTY, " "); + assertEquals(ClickHouseTableSinkOperator.DEFAULT_ENGINE, + new ClickHouseTableSinkOperator( + new TableSink(blank, "overwrite", "t", "order_id")).getEngine()); + + Properties padded = new Properties(); + padded.setProperty(ClickHouseTableSinkOperator.ENGINE_PROPERTY, " Memory "); + assertEquals("Memory", + new ClickHouseTableSinkOperator( + new TableSink(padded, "overwrite", "t", "order_id")).getEngine()); + } + + @Test + void tableSinkRejectsUnsafeTableName() { + assertThrows(IllegalArgumentException.class, + () -> new ClickHouseTableSinkOperator("result; DROP TABLE orders", new String[]{"order_id"})); + } + + @Test + void defaultsProvideACostModelForEveryOperator() { + Configuration configuration = new Configuration(); + // Touch the platform so its defaults file is registered. + assertNotNull(ClickHouse.platform()); + + List operators = new java.util.ArrayList<>(queryOperators()); + operators.add(new ClickHouseTableSource("orders", "order_id")); + operators.add(new ClickHouseTableSinkOperator("result", new String[]{"order_id"})); + + for (JdbcExecutionOperator operator : operators) { + String key = operator.getLoadProfileEstimatorConfigurationKey(); + assertTrue(key.startsWith("wayang.clickhouse."), key); + assertTrue(configuration.getOptionalStringProperty(key).isPresent(), + "missing default for " + key); + assertTrue(operator.createLoadProfileEstimator(configuration).isPresent(), + "no load profile estimator for " + key); + } + } + + @Test + void channelConversionsLeadToJavaAndSpark() { + assertEquals(2, ChannelConversions.ALL.size()); + ChannelDescriptor sqlChannel = ClickHousePlatform.getInstance().getSqlQueryChannelDescriptor(); + for (ChannelConversion conversion : ChannelConversions.ALL) { + assertEquals(sqlChannel, conversion.getSourceChannelDescriptor()); + } + assertEquals(StreamChannel.DESCRIPTOR, + ChannelConversions.SQL_TO_STREAM_CONVERSION.getTargetChannelDescriptor()); + assertEquals(RddChannel.UNCACHED_DESCRIPTOR, + ChannelConversions.SQL_TO_UNCACHED_RDD_CONVERSION.getTargetChannelDescriptor()); + } + + @Test + void conversionsPluginRequiresClickHouseAndJava() { + assertTrue(ClickHouse.conversionPlugin().getRequiredPlatforms() + .contains(ClickHousePlatform.getInstance())); + assertTrue(ClickHouse.conversionPlugin().getRequiredPlatforms() + .contains(JavaPlatform.getInstance())); + // setProperties is a no-op for both plugins; it must not throw. + ClickHouse.conversionPlugin().setProperties(new Configuration()); + ClickHouse.plugin().setProperties(new Configuration()); + } +}