Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.wayang.core.plan.wayangplan.UnaryToUnaryOperator;
import org.apache.wayang.core.platform.ChannelDescriptor;
import org.apache.wayang.core.platform.ChannelInstance;
import org.apache.wayang.core.platform.CrossPlatformExecutor;
import org.apache.wayang.core.platform.ExecutionResourceTemplate;
import org.apache.wayang.core.platform.lineage.ExecutionLineageNode;
import org.apache.wayang.core.types.DataSetType;
import org.apache.wayang.core.util.JsonSerializable;
Expand Down Expand Up @@ -97,30 +99,52 @@ public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> eval
final StreamChannel.Instance output = (StreamChannel.Instance) outputs[0];

JdbcPlatformTemplate producerPlatform = (JdbcPlatformTemplate) input.getChannel().getProducer().getPlatform();
final String sqlQuery = input.getSqlQuery();
final Connection connection = producerPlatform
.createDatabaseDescriptor(executor.getConfiguration())
.createJdbcConnection();

Iterator<Record> resultSetIterator = new ResultSetIterator(connection, input.getSqlQuery());
Spliterator<Record> resultSetSpliterator = Spliterators.spliteratorUnknownSize(resultSetIterator, 0);
Stream<Record> resultSetStream = StreamSupport.stream(resultSetSpliterator, false);
ResultSetIterator resultSetIterator = new ResultSetIterator(connection, sqlQuery);
boolean success = false;
try {
Spliterator<Record> resultSetSpliterator = Spliterators.spliteratorUnknownSize(resultSetIterator, 0);
Stream<Record> resultSetStream = StreamSupport.stream(resultSetSpliterator, false)
.onClose(resultSetIterator::close);

output.accept(resultSetStream);
// Terminal operations do not close streams, and StreamChannel disposal only drops its reference.
// Register cleanup for job shutdown: live channels can keep the JavaExecutor referenced after
// downstream failures, so executor disposal alone is insufficient. Closing on channel disposal
// could instead interrupt a lazy successor that still needs the JDBC resources.
CrossPlatformExecutor crossPlatformExecutor = executor.getCrossPlatformExecutor();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need to call the executor within an operator? this seems odd

@Kaileshwar16 Kaileshwar16 Nov 11, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand the concern about coupling the operator directly to CrossPlatformExecutor, I added this so JDBC resources get cleaned up when the job ends, even if downstream processing stops early. Stream operations don’t automatically close the stream, and disposing the channel only clears its reference.

I also tried registering cleanup with JavaExecutor alone, but live channels can prevent it from being disposed after a downstream failure. Both new regression cases failed with that approach. I will add tests and have the comment more precise. is it justifed to add it? do prefer anyother way to solve it

if (crossPlatformExecutor != null) {
crossPlatformExecutor.registerGlobal(new ExecutionResourceTemplate(null) {
@Override
protected void doDispose() {
resultSetIterator.close();
}
});
}
output.accept(resultSetStream);

ExecutionLineageNode queryLineageNode = new ExecutionLineageNode(operatorContext);
queryLineageNode.add(LoadProfileEstimators.createFromSpecification(
String.format("wayang.%s.sqltostream.load.query", this.jdbcPlatform.getPlatformId()),
executor.getConfiguration()
));
queryLineageNode.addPredecessor(input.getLineage());
ExecutionLineageNode outputLineageNode = new ExecutionLineageNode(operatorContext);
outputLineageNode.add(LoadProfileEstimators.createFromSpecification(
String.format("wayang.%s.sqltostream.load.output", this.jdbcPlatform.getPlatformId()),
executor.getConfiguration()
));
output.getLineage().addPredecessor(outputLineageNode);

return queryLineageNode.collectAndMark();
ExecutionLineageNode queryLineageNode = new ExecutionLineageNode(operatorContext);
queryLineageNode.add(LoadProfileEstimators.createFromSpecification(
String.format("wayang.%s.sqltostream.load.query", this.jdbcPlatform.getPlatformId()),
executor.getConfiguration()
));
queryLineageNode.addPredecessor(input.getLineage());
ExecutionLineageNode outputLineageNode = new ExecutionLineageNode(operatorContext);
outputLineageNode.add(LoadProfileEstimators.createFromSpecification(
String.format("wayang.%s.sqltostream.load.output", this.jdbcPlatform.getPlatformId()),
executor.getConfiguration()
));
output.getLineage().addPredecessor(outputLineageNode);

Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> result = queryLineageNode.collectAndMark();
success = true;
return result;
} finally {
if (!success) resultSetIterator.close();
}
}

@Override
Expand All @@ -146,6 +170,10 @@ public Collection<String> getLoadProfileEstimatorConfigurationKeys() {
*/
public static class ResultSetIterator implements Iterator<Record>, AutoCloseable {

private Connection connection;

private Statement statement;

/**
* Keeps around the {@link ResultSet} of the SQL query.
*/
Expand All @@ -159,26 +187,33 @@ public static class ResultSetIterator implements Iterator<Record>, AutoCloseable
/**
* Creates a new instance.
*
* @param connection the JDBC connection on which to execute a SQL query
* @param connection the JDBC connection on which to execute a SQL query; ownership is transferred
* to this iterator, including when construction fails
* @param sqlQuery the SQL query
*/
ResultSetIterator(Connection connection, String sqlQuery) {
this.connection = connection;
boolean success = false;
try {
//connection.setAutoCommit(false);
Statement st = connection.createStatement();
this.statement = st;
//st.setFetchSize(100000000);
this.resultSet = st.executeQuery(sqlQuery);
this.moveToNext();
success = true;
} catch (SQLException e) {
this.close();
throw new WayangException("Could not execute SQL.", e);
} finally {
if (!success) this.close();
}
this.moveToNext();
}

/**
* Moves this instance to the next {@link Record}.
*/
private void moveToNext() {
boolean success = false;
try {
if (this.resultSet == null || !this.resultSet.next()) {
this.next = null;
Expand All @@ -191,10 +226,11 @@ private void moveToNext() {
}
this.next = new Record(values);
}
success = true;
} catch (SQLException e) {
this.next = null;
this.close();
throw new WayangException("Exception while iterating the result set.", e);
} finally {
if (!success) this.close();
}
}

Expand All @@ -212,13 +248,17 @@ public Record next() {

@Override
public void close() {
if (this.resultSet != null) {
AutoCloseable[] resources = {this.resultSet, this.statement, this.connection};
this.resultSet = null;
this.statement = null;
this.connection = null;
this.next = null;
for (AutoCloseable resource : resources) {
if (resource == null) continue;
try {
this.resultSet.close();
} catch (Throwable t) {
LogManager.getLogger(this.getClass()).error("Could not close result set.", t);
} finally {
this.resultSet = null;
resource.close();
} catch (Exception e) {
LogManager.getLogger(this.getClass()).error("Could not close JDBC resource.", e);
}
}
}
Expand Down
Loading