Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,28 @@ public CardinalityEstimate estimate(OptimizationContext optimizationContext, Car
CardinalityEstimate cardinalityEstimate = optimizationContext.queryJobCache(jobCacheKey, CardinalityEstimate.class);
if (cardinalityEstimate != null) return cardinalityEstimate;

if (ParquetSource.this.metadata == null) {
ParquetSource.this.logger.warn("Could not inspect metadata of {}... deliver fallback estimate.",
ParquetSource.this.inputUrl);
timeMeasurement.stop();
return this.FALLBACK_ESTIMATE;
}

// Otherwise calculate the cardinality.
// First, inspect the size of the file and its line sizes.
OptionalLong fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl);
OptionalLong fileSize;
try {
fileSize = FileSystems.getFileSize(ParquetSource.this.inputUrl);
} catch (Exception e) {
ParquetSource.this.logger.warn(
"Could not determine size of {} ({}: {})... deliver fallback estimate.",
ParquetSource.this.inputUrl,
e.getClass().getSimpleName(),
e.getMessage()
);
timeMeasurement.stop();
return this.FALLBACK_ESTIMATE;
}
if (fileSize.isEmpty()) {
ParquetSource.this.logger.warn("Could not determine size of {}... deliver fallback estimate.",
ParquetSource.this.inputUrl);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.wayang.basic.operators;

import org.apache.wayang.commons.util.profiledb.instrumentation.StopWatch;
import org.apache.wayang.commons.util.profiledb.model.Experiment;
import org.apache.wayang.commons.util.profiledb.model.Subject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.Job;
import org.apache.wayang.core.optimizer.DefaultOptimizationContext;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

/**
* Test suite for {@link ParquetSource}.
*/
class ParquetSourceTest {

@TempDir
Path tempDir;

@Test
void cardinalityEstimationFallsBackWhenMetadataIsMissing() throws Exception {
Path parquetPath = this.tempDir.resolve("orders.parquet");
Files.write(parquetPath, new byte[]{1, 2, 3, 4});

Job job = mock(Job.class);
DefaultOptimizationContext optimizationContext = mock(DefaultOptimizationContext.class);
when(job.getOptimizationContext()).thenReturn(optimizationContext);
when(optimizationContext.getJob()).thenReturn(job);
when(job.getStopWatch()).thenReturn(new StopWatch(new Experiment("mock", new Subject("mock", "mock"))));
when(optimizationContext.getConfiguration()).thenReturn(new Configuration());

ParquetSource source = new ParquetSource(parquetPath.toUri().toString(), null, "order_id");
Optional<CardinalityEstimator> estimator = source.createCardinalityEstimator(
0,
optimizationContext.getConfiguration()
);

assertTrue(estimator.isPresent());
CardinalityEstimate estimate = estimator.get().estimate(optimizationContext);
assertEquals(1000L, estimate.getLowerEstimate());
assertEquals(100000000L, estimate.getUpperEstimate());
}
}
2 changes: 1 addition & 1 deletion wayang-platforms/wayang-bigquery/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
</properties>

<dependencies>
<!-- BigQuery JDBC driver — shaded uber jar includes all transitive deps.
<!-- BigQuery JDBC driver; shaded uber jar includes all transitive deps.
Do NOT add google-cloud-bigquery alongside this; the shaded classes
conflict with the non-shaded ones. -->
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class Mappings {
new FilterMapping(),
new GlobalReduceMapping(),
new JoinMapping(),
new ParquetSourceMapping(),
new ProjectionMapping(),
new ReduceByMapping(),
new SortMapping(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.wayang.bigquery.mapping;

import org.apache.wayang.basic.operators.ParquetSource;
import org.apache.wayang.bigquery.operators.BigQueryParquetSource;
import org.apache.wayang.bigquery.platform.BigQueryPlatform;
import org.apache.wayang.core.mapping.Mapping;
import org.apache.wayang.core.mapping.OperatorPattern;
import org.apache.wayang.core.mapping.PlanTransformation;
import org.apache.wayang.core.mapping.ReplacementSubplanFactory;
import org.apache.wayang.core.mapping.SubplanPattern;

import java.util.Collection;
import java.util.Collections;

/**
* Mapping from {@link ParquetSource} to {@link BigQueryParquetSource}.
*/
public class ParquetSourceMapping implements Mapping {

@Override
public Collection<PlanTransformation> getTransformations() {
return Collections.singleton(new PlanTransformation(
this.createSubplanPattern(),
this.createReplacementSubplanFactory(),
BigQueryPlatform.getInstance()
));
}

private SubplanPattern createSubplanPattern() {
final OperatorPattern operatorPattern = new OperatorPattern(
"source", new ParquetSource((String) null, (String[]) null), false
);
return SubplanPattern.createSingleton(operatorPattern);
}

private ReplacementSubplanFactory createReplacementSubplanFactory() {
return new ReplacementSubplanFactory.OfSingleOperators<ParquetSource>(
(matchedOperator, epoch) -> new BigQueryParquetSource(matchedOperator).at(epoch)
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.wayang.bigquery.operators;

import org.apache.wayang.basic.operators.ParquetSource;
import org.apache.wayang.jdbc.operators.JdbcParquetSource;

/**
* BigQuery implementation for Parquet-backed external tables.
*/
public class BigQueryParquetSource extends JdbcParquetSource implements BigQueryExecutionOperator {

public BigQueryParquetSource(String sourceName, String[] projection, String... columnNames) {
super(sourceName, projection, columnNames);
}

public BigQueryParquetSource(ParquetSource that) {
super(that);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@
# JDBC driver (loaded via reflection; no compile-time dependency needed)
wayang.bigquery.jdbc.driverName = com.google.cloud.bigquery.jdbc.BigQueryDriver

# Optional Parquet source resolution.
# Use this to keep ParquetSource inputUrl as a canonical Parquet URI while
# resolving it to a BigQuery-visible external table during execution.
# Format: parquet-uri=bigquery-relation;parquet-uri-2=bigquery-relation-2
# wayang.bigquery.parquetsource.mappings = gs://bucket/orders/*.parquet=`project.dataset.orders`
#
# Optional auto-create support. Keep disabled unless the configured dataset is
# available and the URI is readable by BigQuery.
# wayang.bigquery.parquetsource.auto-create = false
# wayang.bigquery.parquetsource.auto-create.relation-prefix = project.dataset.wayang_parquet_
# wayang.bigquery.parquetsource.auto-create.template = CREATE EXTERNAL TABLE IF NOT EXISTS ${relation} OPTIONS (format = 'PARQUET', uris = ['${uri}'])

# Connection URL and credentials are deployment-specific.
# Set these in your wayang.properties or programmatically via Configuration.
#
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.wayang.bigquery;

import org.apache.wayang.basic.data.Record;
import org.apache.wayang.basic.operators.TableSink;
import org.apache.wayang.bigquery.operators.BigQueryParquetSource;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.WayangContext;
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Properties;

import static org.junit.jupiter.api.Assertions.assertEquals;

/**
* Minimal integration test for {@link BigQueryParquetSource}.
*
* <p>Set {@code -Dbigquery.parquet.source=`project.dataset.table`} or
* {@code BIGQUERY_PARQUET_SOURCE} for an existing external table. Alternatively,
* set {@code -Dbigquery.parquet.uri=gs://bucket/path/*.parquet} to let the test
* create a temporary BigQuery external table.
*/
class BigQueryParquetSourceIT {

private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project");
private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL",
"wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com");
private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH",
System.getProperty("user.home") + "/wayang-bq-key.json");
private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US");
private static final String DATASET = cfg("bigquery.dataset", "BIGQUERY_DATASET", "wayang_it");
private static final String PARQUET_URI = cfg("bigquery.parquet.uri", "BIGQUERY_PARQUET_URI", "");
private static final String CONFIGURED_SOURCE = cfg("bigquery.parquet.source", "BIGQUERY_PARQUET_SOURCE", "");
private static final String SOURCE = CONFIGURED_SOURCE.isEmpty() ? tableName("parquet_source") : CONFIGURED_SOURCE;
private static final String SINK = cfg("bigquery.parquet.sink", "BIGQUERY_PARQUET_SINK",
tableName("parquet_source_copy"));

private static final String JDBC_URL = String.format(
"jdbc:bigquery://https://www.googleapis.com/bigquery/v2;"
+ "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s",
PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION);

private static boolean available = false;
private static long sourceCount = -1;

@BeforeAll
static void setUp() {
if (CONFIGURED_SOURCE.isEmpty() && PARQUET_URI.isEmpty()) {
return;
}
try {
Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver");
try (Connection connection = DriverManager.getConnection(JDBC_URL);
Statement statement = connection.createStatement()) {
statement.execute("CREATE SCHEMA IF NOT EXISTS `" + PROJECT_ID + "." + DATASET + "` "
+ "OPTIONS(location='" + LOCATION + "')");
statement.execute("DROP TABLE IF EXISTS " + SINK);
if (!PARQUET_URI.isEmpty()) {
statement.execute("DROP TABLE IF EXISTS " + SOURCE);
statement.execute("CREATE EXTERNAL TABLE " + SOURCE
+ " OPTIONS (format = 'PARQUET', uris = ['" + PARQUET_URI + "'])");
}
sourceCount = queryLong(statement, "SELECT count(*) FROM " + SOURCE);
available = true;
}
} catch (Exception e) {
System.err.println("[BigQueryParquetSourceIT] BigQuery Parquet source unavailable: " + e.getMessage());
}
}

@AfterAll
static void tearDown() {
if (!available) return;
try (Connection connection = DriverManager.getConnection(JDBC_URL);
Statement statement = connection.createStatement()) {
statement.execute("DROP TABLE IF EXISTS " + SINK);
if (!PARQUET_URI.isEmpty()) {
statement.execute("DROP TABLE IF EXISTS " + SOURCE);
}
} catch (Exception e) {
System.err.println("[BigQueryParquetSourceIT] cleanup failed: " + e.getMessage());
}
}

@Test
void readsConfiguredParquetExternalTableIntoBigQuerySink() throws Exception {
Assumptions.assumeTrue(!CONFIGURED_SOURCE.isEmpty() || !PARQUET_URI.isEmpty(),
"No BigQuery Parquet source or URI configured");
Assumptions.assumeTrue(available, "BigQuery Parquet source unavailable");

BigQueryParquetSource source = new BigQueryParquetSource(SOURCE, null);
TableSink<Record> sink = new TableSink<>(new Properties(), "overwrite", SINK);
source.connectTo(0, sink, 0);

wayangContext().execute("BQ-ParquetSource", new WayangPlan(sink));

assertEquals(sourceCount, queryLong("SELECT count(*) FROM " + SINK));
}

private WayangContext wayangContext() {
Configuration configuration = new Configuration();
configuration.setProperty("wayang.bigquery.jdbc.url", JDBC_URL);
return new WayangContext(configuration).withPlugin(BigQuery.plugin());
}

private static String cfg(String sysProp, String envVar, String dflt) {
String value = System.getProperty(sysProp);
if (value == null || value.isEmpty()) value = System.getenv(envVar);
return value == null || value.isEmpty() ? dflt : value;
}

private static String tableName(String table) {
return "`" + PROJECT_ID + "." + DATASET + "." + table + "`";
}

private static long queryLong(String sql) throws Exception {
try (Connection connection = DriverManager.getConnection(JDBC_URL);
Statement statement = connection.createStatement()) {
return queryLong(statement, sql);
}
}

private static long queryLong(Statement statement, String sql) throws Exception {
try (ResultSet resultSet = statement.executeQuery(sql)) {
resultSet.next();
return resultSet.getLong(1);
}
}
}
Loading
Loading