From 9b5453f9bb69bb850609e81193a67216c7202990 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Mon, 2 Nov 2026 11:58:20 +0530 Subject: [PATCH 1/3] Add PyWayang load_collection using literal JSON input --- python/README.md | 75 ++++++++++ python/src/pywy/core/serializer.py | 5 +- python/src/pywy/dataquanta.py | 17 ++- python/src/pywy/operators/__init__.py | 3 +- python/src/pywy/operators/source.py | 35 +++++ python/src/pywy/tests/test_load_collection.py | 112 +++++++++++++++ .../tests/test_load_collection_integration.py | 82 +++++++++++ .../main/scala/builder/JsonPlanBuilder.scala | 8 +- .../input/InputCollectionFromJson.scala | 36 ++++- .../wayang/api/json/InputCollectionTest.scala | 135 ++++++++++++++++++ 10 files changed, 499 insertions(+), 9 deletions(-) create mode 100644 python/src/pywy/tests/test_load_collection.py create mode 100644 python/src/pywy/tests/test_load_collection_integration.py create mode 100644 wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala diff --git a/python/README.md b/python/README.md index eafa2e7ee..9c3388519 100644 --- a/python/README.md +++ b/python/README.md @@ -119,6 +119,51 @@ configuration.set_property("wayang.api.python.port", "8080") The same values can be supplied through environment variables: `WAYANG_API_URL`, or `WAYANG_API_HOST` and `WAYANG_API_PORT`. +## In-memory collections + +`load_collection()` snapshots a Python list or tuple and returns a `DataQuanta`: + +```python +from pywy.dataquanta import WayangContext +from pywy.platforms.java import JavaPlugin + +ctx = WayangContext().register({JavaPlugin}) +ctx.load_collection([1, 2, 3]).store_textfile("file:///tmp/numbers.txt") +``` + +Supported values are built-in strings, booleans, signed 64-bit integers, finite +floats, and nested lists or tuples. Tuples become lists. Empty and mixed +collections are supported. Input order and nested structure are preserved by +the source; subsequent operators/platforms determine result ordering. `None`, +mappings, sets, generators, custom objects, cyclic containers, out-of-range +integers, and non-finite floats are rejected before submission. + +The current Python worker passes records to UDFs as strings, including records +from collection sources. Convert numeric inputs explicitly: + +```python +(ctx.load_collection([1, 2, 3]) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile("file:///tmp/doubled.txt")) +``` + +This example uses the existing fusion of adjacent Python transformations into +one worker call. The standalone Python filter and primitive-typed map-to-text +paths have existing limitations (empty filter output and incompatible sink +types, respectively), also reproducible with text-file sources. Collection +input does not change those paths. + +Nested values remain structured on the JVM, but reach Python UDFs and text sinks +in Java list notation, such as `[1, 2]`. This is not a JSON round-trip format. +Text sinks use the existing JVM string representation, one record per line; +embedded newlines in strings produce additional lines. File paths refer to the +backend's filesystem. Collection data is held in memory and sent in one JSON +request, so this source is intended for small local datasets. + +This API requires a backend supporting literal `inputCollection.data.values`. +The backend continues to accept existing `inputCollection.data.udf` plans. + ## Docker The Python API can be run with the Wayang REST API as a separate container. @@ -148,3 +193,33 @@ docker network rm wayang-python ### Testing python code You can run the python tests by using pytest, the requirements for the tests are listed in `python/src/pywy/requirements.txt`. To run the tests navigate to the base wayang folder, e.g. `/var/www/html` and run `pytest -s python/src/pywy` if you need to pass a specific configuration for your use case you can also add a config flag `pytest -s --config=pathToYourConfig python/src/pywy/` + +The collection unit tests run without a backend: + +```bash +PYTHONPATH=python/src pytest -q python/src/pywy/tests/test_load_collection.py +``` + +For the collection integration tests, build and install the JSON API and its +dependencies with JDK 17 (`./mvnw install -pl :wayang-api-json -am -DskipTests`). +Start `org.apache.wayang.api.json.Main` as described in the JSON API README, or use: + +```bash +./mvnw -pl :wayang-api-json exec:java \ + -Dexec.mainClass=org.apache.wayang.api.json.Main -Dexec.args=8080 \ + -Dwayang.configuration=file:///absolute/path/to/wayang.properties +``` + +The properties file must set `wayang.api.python.path` to the Python executable, +`wayang.api.python.worker` to the absolute path of +`python/src/pywy/execution/worker.py`, and `wayang.api.python.env.path` to the +absolute path of `python/src`. Run from the repository root against a backend +on the same machine (the tests read its output files): + +```bash +PYTHONPATH=python/src \ +PYWAYANG_INTEGRATION_URL=http://localhost:8080/wayang-api-json/submit-plan/json \ +pytest -q python/src/pywy/tests/test_load_collection_integration.py +``` + +Without `PYWAYANG_INTEGRATION_URL`, these integration tests explicitly skip. diff --git a/python/src/pywy/core/serializer.py b/python/src/pywy/core/serializer.py index 7e727c4bf..c2305a7f6 100644 --- a/python/src/pywy/core/serializer.py +++ b/python/src/pywy/core/serializer.py @@ -60,7 +60,10 @@ def serialize(self, operator): if hasattr(operator, "output_type") and operator.output_type is not None: json_operator["data"]["outputType"] = ndim_from_type(operator.output_type).to_json() - if operator.json_name == "filter": + if operator.json_name == "inputCollection": + json_operator["data"]["values"] = operator.values + + elif operator.json_name == "filter": json_operator["data"]["udf"] = base64.b64encode(cloudpickle.dumps(operator.use_predicate)).decode('utf-8') elif operator.json_name == "reduceBy": diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py index dd8490ba8..4b98b4807 100644 --- a/python/src/pywy/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import Set, List, Optional, cast +from typing import Set, List, Optional, Tuple, Union, cast from pywy.core.core import Plugin, PywyPlan from pywy.operators.base import PO_T @@ -56,6 +56,21 @@ def unregister(self, *plugins: Plugin): def textfile(self, file_path: str) -> "DataQuanta[str]": return DataQuanta(self, TextFileSource(file_path)) + def load_collection(self, collection: Union[List[T], Tuple[T, ...]]) -> "DataQuanta[T]": + """Create a source from a snapshot of an ordered list or tuple. + + Values may be strings, booleans, signed 64-bit integers, finite floats, + or nested lists/tuples (tuples become lists). Empty collections are valid. + Other values, including None, mappings, sets, and cyclic containers, + raise TypeError or ValueError before a plan is submitted. + + As with other sources, the current Python worker passes records to UDFs + as strings. Numeric UDFs must convert their input explicitly with int() + or float(). See the README for a fused transformation example and + existing worker limitations. + """ + return DataQuanta(self, CollectionSource(collection)) + def parquet( self, file_path: str, projection: Optional[List[str]] = None, column_names: Optional[List[str]] = None ) -> "DataQuanta[Record]": diff --git a/python/src/pywy/operators/__init__.py b/python/src/pywy/operators/__init__.py index b4740fd3c..7a349cd03 100644 --- a/python/src/pywy/operators/__init__.py +++ b/python/src/pywy/operators/__init__.py @@ -19,7 +19,7 @@ from pywy.operators.binary import BinaryToUnaryOperator, JoinOperator, DLTrainingOperator, PredictOperator, \ CartesianOperator from pywy.operators.sink import TextFileSink, SinkOperator -from pywy.operators.source import TextFileSource, ParquetSource, SourceUnaryOperator +from pywy.operators.source import CollectionSource, TextFileSource, ParquetSource, SourceUnaryOperator from pywy.operators.unary import UnaryToUnaryOperator, FilterOperator, MapOperator, FlatmapOperator, \ ReduceByKeyOperator, SortOperator @@ -29,6 +29,7 @@ BinaryToUnaryOperator, TextFileSink, TextFileSource, + CollectionSource, ParquetSource, FilterOperator, SinkOperator, diff --git a/python/src/pywy/operators/source.py b/python/src/pywy/operators/source.py index d55caba44..87330c48b 100644 --- a/python/src/pywy/operators/source.py +++ b/python/src/pywy/operators/source.py @@ -15,6 +15,7 @@ # limitations under the License. # +import math from typing import List, Optional from pywy.operators.base import PywyOperator @@ -44,6 +45,40 @@ def __repr__(self): return super().__repr__() +class CollectionSource(SourceUnaryOperator): + def __init__(self, collection): + # Empty and heterogeneous collections use the backend's generic type. + super().__init__('Collection', output_type=None) + self.json_name = "inputCollection" + if type(collection) not in (list, tuple): + raise TypeError("load_collection expects a list or tuple") + self.values = self._snapshot(collection, set(), "collection") + + @staticmethod + def _snapshot(value, ancestors, path): + if type(value) in (str, bool): + return value + if type(value) is int: + if not -(2 ** 63) <= value < 2 ** 63: + raise ValueError(f"{path}: integers must fit in signed 64 bits") + return value + if type(value) is float: + if not math.isfinite(value): + raise ValueError(f"{path}: floats must be finite") + return value + if type(value) in (list, tuple): + if id(value) in ancestors: + raise ValueError(f"{path}: cyclic collections are not supported") + ancestors.add(id(value)) + result = [ + CollectionSource._snapshot(item, ancestors, f"{path}[{index}]") + for index, item in enumerate(value) + ] + ancestors.remove(id(value)) + return result + raise TypeError(f"{path}: unsupported collection value of type {type(value).__name__}") + + class TextFileSource(SourceUnaryOperator): path: str json_name: str diff --git a/python/src/pywy/tests/test_load_collection.py b/python/src/pywy/tests/test_load_collection.py new file mode 100644 index 000000000..757119b67 --- /dev/null +++ b/python/src/pywy/tests/test_load_collection.py @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import json +from unittest.mock import patch + +import cloudpickle +import pytest + +from pywy.dataquanta import DataQuanta, WayangContext +from pywy.operators import CollectionSource, SourceUnaryOperator +from pywy.platforms.java import JavaPlugin + + +@pytest.mark.parametrize("values", [ + [1, -2, 0, -(2 ** 63), 2 ** 63 - 1], + [1.5, -0.0, 1e-300, 1e300], + ['quotes " and \\', "line\nfeed\ttab", "雪 😀", "", '${System.exit(1)}'], + [], + [True, False], + [[1, 2], [], [3, ["nested", False]]], + [1, "two", 3.5, True], +]) +def test_collection_plan(values): + context = WayangContext().register({JavaPlugin}) + data = context.load_collection(values) + assert isinstance(data, DataQuanta) + assert data.context is context + assert isinstance(data.operator, SourceUnaryOperator) + assert data.operator.inputs == 0 + assert data.operator.outputs == 1 + + with patch("pywy.core.core.requests.post") as post: + data.store_textfile("file:///tmp/collection.txt") + + plan = json.loads(json.dumps(post.call_args.kwargs["json"], allow_nan=False)) + assert plan["context"]["origin"] == "python" + assert plan["context"]["platforms"] == ["java"] + source, sink = plan["operators"] + assert source == { + "id": 1, "operatorName": "inputCollection", "cat": "input", + "input": [], "output": [2], "data": {"values": values}, + } + assert sink["operatorName"] == "textFileOutput" + assert sink["input"] == [1] + assert sink["data"]["filename"] == "file:///tmp/collection.txt" + + +def test_snapshot_and_tuples(): + nested = [1, (2, 3)] + values = [nested, nested] + source = CollectionSource(values) + nested.append(4) + values.clear() + assert source.values == [[1, [2, 3]], [1, [2, 3]]] + assert CollectionSource((1, 2)).values == [1, 2] + + +@pytest.mark.parametrize("values", ["abc", b"abc", {1, 2}, {"a": 1}, None, iter([1, 2])]) +def test_reject_unordered_or_non_collection_inputs(values): + with pytest.raises(TypeError, match="expects a list or tuple"): + WayangContext().load_collection(values) + + +@pytest.mark.parametrize("value", [None, {}, {1}, b"bytes", complex(1, 2), object()]) +def test_reject_unsupported_values(value): + with pytest.raises(TypeError, match=r"collection\[0\]\[0\]: unsupported"): + WayangContext().load_collection([[value]]) + + +@pytest.mark.parametrize("value", [-(2 ** 63) - 1, 2 ** 63, float("nan"), float("inf"), -float("inf")]) +def test_reject_out_of_range_values(value): + with pytest.raises(ValueError, match=r"collection\[0\]"): + WayangContext().load_collection([value]) + + +def test_reject_cycles(): + values = [] + values.append(values) + with pytest.raises(ValueError, match="cyclic"): + WayangContext().load_collection(values) + + +def test_collection_transformation(): + with patch("pywy.core.core.requests.post") as post: + (WayangContext().register({JavaPlugin}).load_collection([3, 1, 2]) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile("file:///tmp/doubled.txt")) + source, transform, sink = post.call_args.kwargs["json"]["operators"] + assert source["output"] == [transform["id"]] + assert transform["input"] == [source["id"]] + assert transform["output"] == [sink["id"]] + assert sink["input"] == [transform["id"]] + assert transform["operatorName"] == "mapPartitions" + udf = cloudpickle.loads(base64.b64decode(transform["data"]["udf"])) + assert list(udf(iter(["3", "1", "2"]))) == [6, 4] diff --git a/python/src/pywy/tests/test_load_collection_integration.py b/python/src/pywy/tests/test_load_collection_integration.py new file mode 100644 index 000000000..17f14f51e --- /dev/null +++ b/python/src/pywy/tests/test_load_collection_integration.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Run against a local backend using PYWAYANG_INTEGRATION_URL (see README).""" + +import os + +import pytest +import requests + +from pywy.configuration import Configuration +from pywy.dataquanta import WayangContext +from pywy.platforms.java import JavaPlugin + + +@pytest.fixture +def collection_context(monkeypatch): + url = os.environ.get("PYWAYANG_INTEGRATION_URL") + if not url: + pytest.skip("Set PYWAYANG_INTEGRATION_URL to run against a local Wayang backend") + configuration = Configuration() + configuration.set_property("wayang.api.python.url", url) + post = requests.post + + def checked_post(*args, **kwargs): + response = post(*args, **kwargs, timeout=120) + assert response.status_code == 200, response.text + assert response.text == "Success" + return response + + # Execute real HTTP requests and check errors that PywyPlan currently ignores. + monkeypatch.setattr("pywy.core.core.requests.post", checked_post) + return WayangContext(configuration).register({JavaPlugin}) + + +@pytest.mark.parametrize("values, expected", [ + ([1, -2, 0, -(2 ** 63), 2 ** 63 - 1], "1\n-2\n0\n-9223372036854775808\n9223372036854775807\n"), + ([1.5, -0.25, -0.0, 1.23456789012345], "1.5\n-0.25\n-0.0\n1.23456789012345\n"), + (['quotes " and \\', "雪 😀", "line\nfeed\ttab", "", '${System.exit(1)}'], + 'quotes " and \\\n雪 😀\nline\nfeed\ttab\n\n${System.exit(1)}\n'), + ([], ""), + ([True, False], "true\nfalse\n"), + ([[1, 2], [], [3, ["nested", False]]], "[1, 2]\n[]\n[3, [nested, false]]\n"), +]) +def test_collection_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "collection.txt" + collection_context.load_collection(values).store_textfile(output.as_uri()) + assert output.read_text(encoding="utf-8") == expected + + +@pytest.mark.parametrize("values, expected", [([1, 2, 3], ["3", "5", "7"]), ([], [])]) +def test_collection_maps_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "doubled.txt" + (collection_context.load_collection(values) + .map(lambda value: int(value) * 2, str, int) + .map(lambda value: value + 1, int, int) + .store_textfile(output.as_uri())) + assert output.read_text(encoding="utf-8").splitlines() == expected + + +@pytest.mark.parametrize("values, expected", [([3, 1, 2], ["6", "4"]), ([], [])]) +def test_collection_filter_map_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "filtered.txt" + (collection_context.load_collection(values) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile(output.as_uri())) + assert output.read_text(encoding="utf-8").splitlines() == expected diff --git a/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala b/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala index d9d5a2a62..ec9dc3530 100644 --- a/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala +++ b/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala @@ -18,7 +18,7 @@ package org.apache.wayang.api.json.builder import org.apache.wayang.api.json.operatorfromjson.OperatorFromJson.ExecutionPlatforms -import org.apache.wayang.api.json.parserutil.{SerializableIterable, SerializableLambda, SerializableLambda2} +import org.apache.wayang.api.json.parserutil.{SerializableLambda, SerializableLambda2} import org.apache.wayang.api.json.operatorfromjson.{ComposedOperatorFromJson, OperatorFromJson} import org.apache.wayang.api.json.operatorfromjson.binary.{CartesianOperatorFromJson, CoGroupOperatorFromJson, IntersectOperatorFromJson, JoinOperatorFromJson, PredictOperatorFromJson, DLTrainingOperatorFromJson, UnionOperatorFromJson} import org.apache.wayang.api.json.operatorfromjson.other.KMeansFromJson @@ -213,11 +213,11 @@ class JsonPlanBuilder() { } private def visit(operator: InputCollectionFromJson, planBuilder: PlanBuilder): DataQuanta[Any] = { - val iterable = SerializableIterable.create(operator.data.udf) + val iterable = operator.data.collection if (!ExecutionPlatforms.All.contains(operator.executionPlatform)) - planBuilder.loadCollection(iterable.get) + planBuilder.loadCollection(iterable) else - planBuilder.loadCollection(iterable.get).withTargetPlatforms(getExecutionPlatform(operator.executionPlatform)) + planBuilder.loadCollection(iterable).withTargetPlatforms(getExecutionPlatform(operator.executionPlatform)) } private def visit(operator: TableInputFromJson, planBuilder: PlanBuilder): DataQuanta[Any] = { diff --git a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala index 9d514643d..8522b229b 100644 --- a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala +++ b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala @@ -18,7 +18,11 @@ package org.apache.wayang.api.json.operatorfromjson.input import com.fasterxml.jackson.annotation.JsonTypeName +import com.fasterxml.jackson.databind.JsonNode import org.apache.wayang.api.json.operatorfromjson.OperatorFromJson +import org.apache.wayang.api.json.parserutil.SerializableIterable + +import scala.collection.JavaConverters._ @JsonTypeName(OperatorFromJson.OperatorNames.InputCollection) case class InputCollectionFromJson(override val id: Long, @@ -32,7 +36,35 @@ case class InputCollectionFromJson(override val id: Long, } object InputCollectionFromJson { - case class Data(udf: String) -} + case class Data(udf: String = null, values: JsonNode = null) { + def collection: Iterable[Any] = { + require((udf != null) != (values != null), + "inputCollection requires exactly one of data.udf or data.values") + if (values == null) return SerializableIterable.create(udf).get + require(values.isArray, "inputCollection data.values must be an array") + values.elements().asScala.map(readValue).toVector + } + } + + // Use serializable JVM values, not Jackson nodes or generated Scala code. + private def readValue(value: JsonNode): Any = { + if (value.isTextual) value.textValue() + else if (value.isBoolean) value.booleanValue() + else if (value.isIntegralNumber) { + require(value.canConvertToLong, "inputCollection integers must fit in signed 64 bits") + value.longValue() + } else if (value.isFloatingPointNumber) { + val number = value.doubleValue() + require(!number.isNaN && !number.isInfinity, "inputCollection floats must be finite") + number + } else if (value.isArray) { + val result = new java.util.ArrayList[Any]() + value.elements().asScala.foreach(item => result.add(readValue(item))) + result + } else { + throw new IllegalArgumentException(s"Unsupported inputCollection value: $value") + } + } +} diff --git a/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala new file mode 100644 index 000000000..c129a085d --- /dev/null +++ b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.wayang.api.json + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.wayang.api.json.builder.JsonPlanBuilder +import org.apache.wayang.api.json.operatorfromjson.input.InputCollectionFromJson +import org.apache.wayang.api.json.parserutil.{ParsePlanFromJson, SerializableIterable} +import org.apache.wayang.core.api.Configuration +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} + +class InputCollectionTest { + @TempDir + var directory: Path = _ + + private val mapper = new ObjectMapper() + + private def values(json: String): Iterable[Any] = + InputCollectionFromJson.Data(values = mapper.readTree(json)).collection + + private def roundTrip[T](value: T): T = { + val bytes = new ByteArrayOutputStream() + val output = new ObjectOutputStream(bytes) + try output.writeObject(value) finally output.close() + val input = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray)) + try input.readObject().asInstanceOf[T] finally input.close() + } + + @Test + def literalTypesAndSerialization(): Unit = { + val result = values("""[1,-9223372036854775808,9223372036854775807,1.25,true,"雪",[2,[],"nested"]]""").toVector + assertEquals(Vector(1L, Long.MinValue, Long.MaxValue, 1.25, true, "雪"), result.take(6)) + assertTrue(result.head.isInstanceOf[Long]) + assertTrue(result(3).isInstanceOf[Double]) + assertEquals(java.util.Arrays.asList(2L, new java.util.ArrayList[Any](), "nested"), result(6)) + assertEquals(result, roundTrip(result)) + } + + @Test + def emptyCollection(): Unit = { + assertTrue(values("[]").isEmpty) + assertTrue(roundTrip(values("[]")).isEmpty) + } + + @Test + def rejectsUnsupportedValues(): Unit = { + Seq("null", "{}", "1", "[null]", "[{}]", "[9223372036854775808]", "[1e400]").foreach { json => + assertThrows(classOf[IllegalArgumentException], () => { values(json); () }, json) + } + assertThrows(classOf[IllegalArgumentException], () => InputCollectionFromJson.Data().collection) + assertThrows(classOf[IllegalArgumentException], () => + InputCollectionFromJson.Data("List(1)", mapper.readTree("[]")).collection) + } + + @Test + def legacyIterableSerialization(): Unit = { + val iterable = SerializableIterable.create("List(1, 2, 3)") + assertEquals(List(1, 2, 3), roundTrip(iterable).get) + assertEquals(List(1, 2, 3), iterable.get) + assertEquals(List(1, 2, 3), roundTrip(iterable).get) + } + + @Test + def operatorJsonRoundTrip(): Unit = { + Seq( + InputCollectionFromJson.Data("List(1, 2, 3)"), + InputCollectionFromJson.Data(values = mapper.readTree("[1,2,3]")), + InputCollectionFromJson.Data(values = mapper.readTree("[]")) + ).foreach { data => + val source = InputCollectionFromJson(1L, Array.empty[Long], Array(2L), "input", "inputCollection", data) + val replaced = source.replaceId(3L).asInstanceOf[InputCollectionFromJson] + assertEquals(3L, replaced.id) + assertEquals(data.collection.toList, replaced.data.collection.toList) + } + } + + private def execute(data: String, filename: String, origin: String = "python"): String = { + val output = directory.resolve(filename) + val plan = ParsePlanFromJson.parsePlanFromString(s"""{ + "context": {"origin": "$origin", "platforms": ["java"], "configuration": {}}, + "operators": [ + {"id": 1, "operatorName": "inputCollection", "cat": "input", + "input": [], "output": [2], "data": $data, "executionPlatform": "java"}, + {"id": 2, "operatorName": "textFileOutput", "cat": "output", + "input": [1], "output": [], "data": {"filename": "${output.toUri}"}} + ] + }""").get + val builder = new JsonPlanBuilder() + builder.configuration = new Configuration() + builder.setPlatforms(plan.context.platforms).setOrigin(plan.context.origin).setOperators(plan.operators).execute() + new String(Files.readAllBytes(output), StandardCharsets.UTF_8) + } + + @Test + def literalPlanWritesFile(): Unit = { + assertEquals("1\n2\n3\n", execute("""{"values": [1,2,3]}""", "integers.txt")) + assertEquals("", execute("""{"values": []}""", "empty.txt")) + assertEquals("[1, 2]\n[]\n", execute("""{"values": [[1,2],[]]}""", "nested.txt")) + } + + @Test + def stringsAreData(): Unit = { + val text = "quotes \" and \\ and newline\n雪 ${System.exit(1)}" + val data = mapper.createObjectNode() + data.putArray("values").add(text).add("") + assertEquals(text + "\n\n", execute(data.toString, "strings.txt")) + } + + @Test + def legacyPlansStillWriteFiles(): Unit = { + assertEquals("1\n2\n3\n", execute("""{"udf": "List(1, 2, 3)"}""", "legacy.txt", "scala")) + assertEquals("", execute("""{"udf": "List.empty[Int]"}""", "legacy-empty.txt", "scala")) + } +} From 59f16e678692f9d687ac2a1a5607e69f0f934e4f Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Thu, 5 Nov 2026 10:57:15 +0530 Subject: [PATCH 2/3] Implement PyWayang DataQuanta.collect() Return supported in-memory results through the existing JSON execution path, preserving terminal Python values and legacy REST/text-sink behavior. Refs #869; stacked on #870. Generated-by: OpenAI Codex (GPT-6) --- python/README.md | 32 ++++++++ python/src/pywy/core/core.py | 35 +++++++-- python/src/pywy/core/functions.py | 20 +++++ python/src/pywy/core/serializer.py | 17 +++-- python/src/pywy/dataquanta.py | 12 +++ python/src/pywy/execution/util.py | 4 + python/src/pywy/execution/worker.py | 6 +- wayang-api/wayang-api-json/README.md | 28 +++++++ .../src/main/scala/CollectResult.scala | 74 +++++++++++++++++++ .../wayang-api-json/src/main/scala/Main.scala | 24 ++++-- .../scala/operatorfromjson/PlanFromJson.scala | 3 +- .../unary/MapPartitionsOperatorFromJson.scala | 3 +- 12 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 wayang-api/wayang-api-json/src/main/scala/CollectResult.scala diff --git a/python/README.md b/python/README.md index 9c3388519..60ae92885 100644 --- a/python/README.md +++ b/python/README.md @@ -164,6 +164,38 @@ request, so this source is intended for small local datasets. This API requires a backend supporting literal `inputCollection.data.values`. The backend continues to accept existing `inputCollection.data.udf` plans. +### Collecting results + +Use `collect()` to execute a pipeline and receive a Python list: + +```python +result = (ctx.load_collection([1, 2, 3]) + .map(lambda value: int(value) * 2, str, int) + .collect()) +# [2, 4, 6] +``` + +Empty results return `[]`. Supported result values are the same as collection +inputs: strings, booleans, signed 64-bit integers, finite floats, and nested +lists/tuples (tuples become lists). Unsupported values cause an error instead +of being converted to strings. Each call executes the pipeline again without +changing the graph. Result order depends on the operators and platform. + +Adjacent Python transformations at the end of a collected pipeline run together +and encode their results before crossing the worker boundary. This preserves +their runtime types, including nested values, and supports standalone maps and +filters. Inputs to those transformations still follow the existing string-based +worker behavior: filtering `[1, 2, 3]` with `lambda x: int(x) > 1` yields +`["2", "3"]`. Map with `int` before filtering to retain numeric results in the +fused pipeline. Earlier worker boundaries, for example before a join, retain +their existing limitations; collecting cannot restore types already lost there. + +The backend, Python client, and worker must all include collect support. +Execution failures raise `requests.HTTPError` with the backend error response; +invalid or unsupported responses raise `ValueError`. Results are materialized +in backend and client memory, so use this method for results that fit in memory. +`store_textfile()` and legacy REST responses retain their existing behavior. + ## Docker The Python API can be run with the Wayang REST API as a separate container. diff --git a/python/src/pywy/core/core.py b/python/src/pywy/core/core.py index 2babc52cd..381949094 100644 --- a/python/src/pywy/core/core.py +++ b/python/src/pywy/core/core.py @@ -24,7 +24,7 @@ from pywy.core.serializer import JSONSerializer from pywy.graph.graph import WayangGraph from pywy.graph.types import WGraphOfVec -from pywy.operators import SinkOperator, UnaryToUnaryOperator +from pywy.operators import PywyOperator, CollectionSource, UnaryToUnaryOperator class Plugin: @@ -64,13 +64,13 @@ class PywyPlan: plugins : :obj:`set` of :py:class:`pywy.core.plugin.Plugin` plugins is the set of possible platforms that can be uses to execute the PywyPlan - sinks : :py:class:`typing.Iterable` of :py:class:`pywy.operators.sink.SinkOperator` - The list of sink operators, this describe the end of the pipeline, and - they are used to build the `graph` + sinks : :py:class:`typing.Iterable` of :py:class:`pywy.operators.base.PywyOperator` + Terminal operators used to build the graph. For collect(), the terminal + operator need not be a sink. """ graph: WayangGraph - def __init__(self, plugins: Set[Plugin], configuration: Configuration, sinks: Iterable[SinkOperator]): + def __init__(self, plugins: Set[Plugin], configuration: Configuration, sinks: Iterable[PywyOperator]): """basic Constructor of PywyPlan this constructor set the plugins and sinks element, and it prepares @@ -93,7 +93,7 @@ def set_graph(self): """ self.graph = WGraphOfVec(self.sinks) - def execute(self): + def execute(self, collect: bool = False): """ Transform the plan into topologies to group pipelines into one MapPartition operator Transform the plan into json objects to send it to Wayang @@ -109,6 +109,8 @@ def execute(self): json_data["context"] = context json_data["operators"] = [] + if collect: + json_data["collect"] = True nodes = [] pipeline = [] @@ -127,6 +129,9 @@ def execute(self): json_data["operators"].append(serializer.serialize(operator)) pipeline = [] + if pipeline: + json_data["operators"].append(serializer.serialize_pipeline(pipeline, json_output=collect)) + api_url = self.configuration.get_property("wayang.api.python.url") or os.environ.get("WAYANG_API_URL") if api_url is None: host = ( @@ -145,3 +150,21 @@ def execute(self): json_body = json.dumps(json_data) print(json_body) response = requests.post(api_url, headers=headers, json=json_data) + if collect: + if not response.ok: + raise requests.HTTPError( + f"Wayang execution failed (HTTP {response.status_code}): {response.text}", + response=response + ) + try: + body = response.json() + except ValueError as error: + raise ValueError( + "Wayang collect expected a JSON response; check the backend version" + ) from error + if not isinstance(body, dict) or not isinstance(body.get("results"), list): + raise ValueError("Wayang collect expected a JSON object with a results array") + try: + return CollectionSource._snapshot(body["results"], set(), "results") + except (TypeError, ValueError) as error: + raise ValueError(f"Unsupported Wayang collect response: {error}") from error diff --git a/python/src/pywy/core/functions.py b/python/src/pywy/core/functions.py index 7a0d6e6c1..d9cfd0146 100644 --- a/python/src/pywy/core/functions.py +++ b/python/src/pywy/core/functions.py @@ -14,6 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import json + +from pywy.execution.util import JsonRecord +from pywy.operators.source import CollectionSource + + class ChainedFunctions: def __init__(self): self.functions = [] @@ -27,3 +33,17 @@ def execute(self, input_data): for function in self.functions: result = function(result) return result + + def execute_json(self, input_data): + # Encode before the worker's legacy str()/NumPy conversion loses types. + try: + for value in self.execute(input_data): + value = CollectionSource._snapshot(value, set(), "collect result") + yield JsonRecord( + json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") + ) + except Exception as error: + # Preserve the Python cause instead of exposing only a socket EOF. + yield JsonRecord( + json.dumps({"error": f"{type(error).__name__}: {error}"}).encode("utf-8") + ) diff --git a/python/src/pywy/core/serializer.py b/python/src/pywy/core/serializer.py index c2305a7f6..2f3548fd8 100644 --- a/python/src/pywy/core/serializer.py +++ b/python/src/pywy/core/serializer.py @@ -48,7 +48,9 @@ def serialize(self, operator): json_operator["input"] = [] if operator.cat != "output": - json_operator["output"] = list(map(lambda x: self.id_table[x], operator.outputOperator)) + json_operator["output"] = [ + self.id_table[output] for output in operator.outputOperator if output in self.id_table + ] else: json_operator["output"] = [] @@ -96,7 +98,7 @@ def serialize(self, operator): return json_operator - def serialize_pipeline(self, pipeline): + def serialize_pipeline(self, pipeline, json_output=False): assert len(pipeline) > 0 json_operator = {} @@ -105,9 +107,11 @@ def serialize_pipeline(self, pipeline): json_operator["cat"] = "unary" json_operator["data"] = {} json_operator["input"] = list(map(lambda x: self.id_table[x], pipeline[0].inputOperator)) - json_operator["output"] = list(map(lambda x: self.id_table[x], pipeline[-1].outputOperator)) + json_operator["output"] = [ + self.id_table[output] for output in pipeline[-1].outputOperator if output in self.id_table + ] - if len(pipeline) == 1: + if len(pipeline) == 1 and not json_output: return self.serialize(pipeline[0]) chainedUdfs = ChainedFunctions() @@ -117,6 +121,9 @@ def serialize_pipeline(self, pipeline): if hasattr(operator, "get_udf"): chainedUdfs.add_function(operator.get_udf) - json_operator["data"]["udf"] = base64.b64encode(cloudpickle.dumps(chainedUdfs.execute)).decode('utf-8') + udf = chainedUdfs.execute_json if json_output else chainedUdfs.execute + json_operator["data"]["udf"] = base64.b64encode(cloudpickle.dumps(udf)).decode('utf-8') + if json_output: + json_operator["data"]["jsonOutput"] = True return json_operator diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py index 4b98b4807..6bb8aa560 100644 --- a/python/src/pywy/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -234,6 +234,18 @@ def train_linear_svc( + def collect(self: "DataQuanta[T]") -> List[T]: + """Execute this pipeline and return its results in memory. + + Supports the same values as load_collection(); tuples become lists. + Result order depends on the operators and execution platform. Each call + executes the pipeline again. Backend failures raise requests.HTTPError; + invalid or unsupported results raise ValueError. + """ + return PywyPlan( + self.context.plugins, self.context.configuration, [self.operator] + ).execute(collect=True) + def store_textfile(self: "DataQuanta[In]", path: str, input_type: GenericTco = None) -> None: last: List[SinkOperator] = [ cast( diff --git a/python/src/pywy/execution/util.py b/python/src/pywy/execution/util.py index 32c7a5b96..63138410b 100644 --- a/python/src/pywy/execution/util.py +++ b/python/src/pywy/execution/util.py @@ -16,6 +16,10 @@ # +class JsonRecord(bytes): + """An encoded collect record, distinct from a legacy UDF returning bytes.""" + + class SpecialLengths(object): END_OF_DATA_SECTION = -1 PYTHON_EXCEPTION_THROWN = -2 diff --git a/python/src/pywy/execution/worker.py b/python/src/pywy/execution/worker.py index 44b5941b8..e21eab149 100755 --- a/python/src/pywy/execution/worker.py +++ b/python/src/pywy/execution/worker.py @@ -22,7 +22,7 @@ import struct import numpy as np -from pywy.execution.util import SpecialLengths +from pywy.execution.util import JsonRecord, SpecialLengths def read_int(stream): @@ -69,7 +69,9 @@ def write_int(p, outfile): def write_with_length(obj, stream): - if type(obj) is list: + if type(obj) is JsonRecord: + serialized = obj + elif type(obj) is list: arr = np.array(obj) serialized = arr.tobytes() else: diff --git a/wayang-api/wayang-api-json/README.md b/wayang-api/wayang-api-json/README.md index aef777e08..5025410ed 100644 --- a/wayang-api/wayang-api-json/README.md +++ b/wayang-api/wayang-api-json/README.md @@ -35,3 +35,31 @@ tar -xvf wayang-assembly-1.1.2-SNAPSHOT-dist.tar.gz cd wayang-1.1.2-SNAPSHOT ./bin/wayang-submit org.apache.wayang.api.json.Main ``` + +## Collection responses + +`POST /wayang-api-json/submit-plan/json` accepts an optional top-level +`"collect": true` field. Such plans must have exactly one terminal operator and +no output sink. Execution uses the existing `DataQuanta.collect()` sink and +returns `application/json` with `{"results": [...]}`, including an empty array +for empty results. Failures use the existing HTTP 400 error response. + +Result values may be strings, booleans, signed 64-bit integers, finite floats, +and nested lists, arrays, or pairs. Pairs become JSON arrays. Other objects and +null values are rejected; results never use Java object serialization or +Scala expression evaluation. + +PyWayang marks a terminal `mapPartitions` with `data.jsonOutput: true` when its +UDF emits UTF-8 JSON records through the existing length-prefixed worker stream. +The REST handler decodes these records as JSON data before constructing the +response. This marker is not needed for native JVM results such as collection +sources. It does not change intermediate operators or legacy worker encoding. +If that UDF fails, its final record is `{"error": "ExceptionType: message"}`; +the handler fails the entire request rather than returning partial results. +Mappings are not supported result values, so this record cannot be mistaken +for a user value. + +Omitting `collect` (or setting it to false) preserves the existing text response: +`Success` for output sinks, and the Scala collection representation otherwise. +The Drawflow endpoint is unchanged. Collection responses are held in memory and +do not guarantee ordering beyond that provided by the selected execution plan. diff --git a/wayang-api/wayang-api-json/src/main/scala/CollectResult.scala b/wayang-api/wayang-api-json/src/main/scala/CollectResult.scala new file mode 100644 index 000000000..0e5747358 --- /dev/null +++ b/wayang-api/wayang-api-json/src/main/scala/CollectResult.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.wayang.api.json + +import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode} +import com.fasterxml.jackson.databind.json.JsonMapper + +import scala.collection.JavaConverters._ + +/** JSON values supported by the in-memory collection API; never serialize arbitrary objects. */ +private[json] object CollectResult { + private val mapper = JsonMapper.builder() + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .build() + + def response(values: Iterable[Any], jsonOutput: Boolean): String = { + val results = mapper.createArrayNode() + values.foreach { value => + val decoded = if (jsonOutput) { + require(value.isInstanceOf[String], "Expected JSON-encoded Python collect results") + mapper.readValue(value.asInstanceOf[String], classOf[Object]) match { + case error: java.util.Map[_, _] if error.size() == 1 && error.get("error").isInstanceOf[String] => + throw new IllegalArgumentException(s"Python collect failed: ${error.get("error")}") + case decoded => decoded + } + } else value + results.add(toJson(decoded)) + } + val response = mapper.createObjectNode() + response.set[JsonNode]("results", results) + mapper.writeValueAsString(response) + } + + private def toJson(value: Any): JsonNode = { + val nodes = mapper.getNodeFactory + value match { + case text: String => nodes.textNode(text) + case boolean: Boolean => nodes.booleanNode(boolean) + case number: Byte => nodes.numberNode(number.toLong) + case number: Short => nodes.numberNode(number.toLong) + case number: Int => nodes.numberNode(number.toLong) + case number: Long => nodes.numberNode(number) + case number: Float => toJson(number.toDouble) + case number: Double => + require(!number.isNaN && !number.isInfinity, "Collect results require finite floats") + nodes.numberNode(number) + case values: java.util.List[_] => toJson(values.asScala.toVector) + case values: Array[_] => toJson(values.toVector) + case pair: Tuple2[_, _] => toJson(Vector(pair._1, pair._2)) + case values: Seq[_] => + val array = mapper.createArrayNode() + values.foreach(value => array.add(toJson(value))) + array + case _ => + val name = if (value == null) "null" else value.getClass.getName + throw new IllegalArgumentException(s"Unsupported collect result type: $name") + } + } +} diff --git a/wayang-api/wayang-api-json/src/main/scala/Main.scala b/wayang-api/wayang-api-json/src/main/scala/Main.scala index 88b7b0f86..833173ad3 100644 --- a/wayang-api/wayang-api-json/src/main/scala/Main.scala +++ b/wayang-api/wayang-api-json/src/main/scala/Main.scala @@ -25,6 +25,7 @@ import java.io.{StringWriter, PrintWriter} import org.apache.wayang.api.json.builder.JsonPlanBuilder import org.apache.wayang.api.json.operatorfromdrawflow.OperatorFromDrawflowConverter import org.apache.wayang.api.json.operatorfromjson.OperatorFromJson +import org.apache.wayang.api.json.operatorfromjson.unary.MapPartitionsOperatorFromJson import org.apache.wayang.api.json.parserutil.ParseOperatorsFromDrawflow import org.apache.wayang.api.json.parserutil.ParsePlanFromJson @@ -51,17 +52,30 @@ object Main extends ZIOAppDefault { (for { requestBody <- req.body.asString planFromJson <- ZIO.fromTry(Try(ParsePlanFromJson.parsePlanFromString(requestBody).get)) + _ <- ZIO.attempt { + if (planFromJson.collect) { + require(!planFromJson.operators.exists(_.cat == OperatorFromJson.Categories.Output), + "Collect plans cannot contain output sinks") + require(planFromJson.operators.count(_.output.isEmpty) == 1, + "Collect plans require exactly one terminal operator") + } + } result <- ZIO.attempt(new JsonPlanBuilder().fromPlan(planFromJson).execute()) responseBody <- ZIO.attempt { - if (planFromJson.operators.exists(op => op.cat == OperatorFromJson.Categories.Output)) { + if (planFromJson.collect) { + val jsonOutput = planFromJson.operators.exists { + case op: MapPartitionsOperatorFromJson => op.output.isEmpty && op.data.jsonOutput + case _ => false + } + Response.json(CollectResult.response(result.collect(), jsonOutput)) + } else if (planFromJson.operators.exists(op => op.cat == OperatorFromJson.Categories.Output)) { println("Successfully executed WayangJob") - "Success" + Response.text("Success") } else { - result.collect().toString() + Response.text(result.collect().toString()) } } - resBody <- ZIO.succeed(Response.text(responseBody)) - } yield resBody).catchAll(t => { + } yield responseBody).catchAll(t => { // Error messages printed in server t.printStackTrace diff --git a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/PlanFromJson.scala b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/PlanFromJson.scala index 8693b5673..56bccab60 100644 --- a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/PlanFromJson.scala +++ b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/PlanFromJson.scala @@ -24,7 +24,8 @@ import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import com.fasterxml.jackson.module.scala.DefaultScalaModule class PlanFromJson(val context: ContextFromJson, - val operators: List[OperatorFromJson] + val operators: List[OperatorFromJson], + val collect: Boolean = false ) extends Serializable { private def getJsonMapper: JsonMapper = { diff --git a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/unary/MapPartitionsOperatorFromJson.scala b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/unary/MapPartitionsOperatorFromJson.scala index 9901c1762..ba14e929e 100644 --- a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/unary/MapPartitionsOperatorFromJson.scala +++ b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/unary/MapPartitionsOperatorFromJson.scala @@ -33,5 +33,6 @@ case class MapPartitionsOperatorFromJson(override val id: Long, } object MapPartitionsOperatorFromJson { - case class Data(udf: String, val inputType: scala.Option[NDimArray], val outputType: scala.Option[NDimArray]) + case class Data(udf: String, val inputType: scala.Option[NDimArray], val outputType: scala.Option[NDimArray], + jsonOutput: Boolean = false) } From 91df2b153c3ccd8d6d64f61d6501cf3d579e1977 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Sun, 8 Nov 2026 10:06:44 +0530 Subject: [PATCH 3/3] Test PyWayang collect and preserve earlier pipeline branches Derive serialized output connections from the selected plan's inputs so collecting an earlier branch still works after another consumer is attached. Add client, worker, Scala result conversion, and real backend regression coverage, and document how to run the collection tests. Refs #869; follow-up to #870. --- python/README.md | 12 +- python/src/pywy/core/serializer.py | 17 +- python/src/pywy/tests/test_collect.py | 164 ++++++++++++ .../pywy/tests/test_collect_integration.py | 235 ++++++++++++++++++ .../wayang/api/json/CollectResultTest.scala | 125 ++++++++++ 5 files changed, 545 insertions(+), 8 deletions(-) create mode 100644 python/src/pywy/tests/test_collect.py create mode 100644 python/src/pywy/tests/test_collect_integration.py create mode 100644 wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/CollectResultTest.scala diff --git a/python/README.md b/python/README.md index 60ae92885..59d32d695 100644 --- a/python/README.md +++ b/python/README.md @@ -229,7 +229,8 @@ You can run the python tests by using pytest, the requirements for the tests are The collection unit tests run without a backend: ```bash -PYTHONPATH=python/src pytest -q python/src/pywy/tests/test_load_collection.py +PYTHONPATH=python/src pytest -q python/src/pywy/tests/test_load_collection.py \ + python/src/pywy/tests/test_collect.py ``` For the collection integration tests, build and install the JSON API and its @@ -251,7 +252,14 @@ on the same machine (the tests read its output files): ```bash PYTHONPATH=python/src \ PYWAYANG_INTEGRATION_URL=http://localhost:8080/wayang-api-json/submit-plan/json \ -pytest -q python/src/pywy/tests/test_load_collection_integration.py +pytest -q python/src/pywy/tests/test_load_collection_integration.py \ + python/src/pywy/tests/test_collect_integration.py ``` Without `PYWAYANG_INTEGRATION_URL`, these integration tests explicitly skip. +The collect tests cover typed results, repeated execution, backend failures, +and compatibility with text sinks and legacy REST clients. Set +`PYWAYANG_TEST_SPARK=1` to also exercise collection on local Spark; the backend +must be configured to run Spark with the JVM options required by JDK 17. +The [wayang-submit launcher](../bin/wayang-submit) supplies these options. When +starting the backend with Maven, pass its `--add-opens` options in `MAVEN_OPTS`. diff --git a/python/src/pywy/core/serializer.py b/python/src/pywy/core/serializer.py index 2f3548fd8..060c9b9ab 100644 --- a/python/src/pywy/core/serializer.py +++ b/python/src/pywy/core/serializer.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import Set, Iterable +from typing import Set, Dict import json import base64 import cloudpickle @@ -28,14 +28,19 @@ from pywy.graph.graph import WayangGraph from pywy.graph.types import WGraphOfVec, NodeOperator, NodeVec from pywy.types import get_java_type, NDimArray, ndim_from_type -from pywy.operators import SinkOperator, UnaryToUnaryOperator, SourceUnaryOperator +from pywy.operators import PywyOperator, SinkOperator, UnaryToUnaryOperator, SourceUnaryOperator class JSONSerializer: - id_table: Iterable[int] + id_table: Dict[PywyOperator, int] - def __init__(self, id_table: Iterable[int]): + def __init__(self, id_table: Dict[PywyOperator, int]): self.id_table = id_table + # Forward links retain only the latest consumer, so derive edges from this plan's inputs. + self.consumers = {operator: [] for operator in id_table} + for operator in id_table: + for upstream in operator.inputOperator: + self.consumers[upstream].append(operator) def serialize(self, operator): json_operator = {} @@ -49,7 +54,7 @@ def serialize(self, operator): if operator.cat != "output": json_operator["output"] = [ - self.id_table[output] for output in operator.outputOperator if output in self.id_table + self.id_table[output] for output in self.consumers[operator] ] else: json_operator["output"] = [] @@ -108,7 +113,7 @@ def serialize_pipeline(self, pipeline, json_output=False): json_operator["data"] = {} json_operator["input"] = list(map(lambda x: self.id_table[x], pipeline[0].inputOperator)) json_operator["output"] = [ - self.id_table[output] for output in pipeline[-1].outputOperator if output in self.id_table + self.id_table[output] for output in self.consumers[pipeline[-1]] ] if len(pipeline) == 1 and not json_output: diff --git a/python/src/pywy/tests/test_collect.py b/python/src/pywy/tests/test_collect.py new file mode 100644 index 000000000..7cc047830 --- /dev/null +++ b/python/src/pywy/tests/test_collect.py @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import io +import json +import struct +from unittest.mock import Mock + +import cloudpickle +import pytest +import requests + +from pywy.dataquanta import WayangContext +from pywy.execution.util import JsonRecord +from pywy.execution.worker import dump_stream, write_with_length +from pywy.platforms.java import JavaPlugin + + +@pytest.fixture +def post(monkeypatch): + response = requests.Response() + response.status_code = 200 + response._content = b'{"results": []}' + post = Mock(return_value=response) + monkeypatch.setattr("pywy.core.core.requests.post", post) + return post + + +@pytest.fixture +def context(): + return WayangContext().register({JavaPlugin}) + + +@pytest.mark.parametrize("values", [ + [], + [-(2 ** 63), 0, 2 ** 63 - 1, 1.25, -0.0], + [True, False, "123", "true", "null", "", 'quote " \\', "雪 😀", "line\n\t\0"], + [[1, [False, "雪"]], []], +]) +def test_collect_response(context, post, values): + post.return_value._content = json.dumps({"results": values}).encode("utf-8") + result = context.load_collection([]).collect() + # JSON equality also distinguishes booleans, integers, floats and negative zero. + assert json.dumps(result) == json.dumps(values) + + +@pytest.mark.parametrize("body", [ + b"Success", b"Buffer(1, 2)", b"{", b"", + b"[]", b"null", b"{}", b'{"results": null}', b'{"results": {}}', + b'{"results": "[]"}', +]) +def test_rejects_malformed_responses(context, post, body): + post.return_value._content = body + with pytest.raises(ValueError, match="Wayang collect expected"): + context.load_collection([]).collect() + + +@pytest.mark.parametrize("body", [ + b'{"results": [null]}', b'{"results": [{}]}', + b'{"results": [9223372036854775808]}', b'{"results": [-9223372036854775809]}', + b'{"results": [NaN]}', b'{"results": [1e400]}', +]) +def test_rejects_unsupported_response_values(context, post, body): + post.return_value._content = body + with pytest.raises(ValueError, match="Unsupported Wayang collect response: results"): + context.load_collection([]).collect() + + +@pytest.mark.parametrize("status", [400, 500]) +def test_reports_backend_error(context, post, status): + post.return_value.status_code = status + post.return_value._content = b"Python collect failed: ZeroDivisionError: division by zero" + with pytest.raises(requests.HTTPError, match="ZeroDivisionError") as failure: + context.load_collection([1]).collect() + assert failure.value.response is post.return_value + assert f"HTTP {status}" in str(failure.value) + + +def test_propagates_connection_failure(context, post): + post.side_effect = requests.ConnectionError("backend unavailable") + with pytest.raises(requests.ConnectionError, match="backend unavailable"): + context.load_collection([]).collect() + + +def test_serializes_terminal_pipeline(context, post): + (context.load_collection([1, 2, 3]) + .map(lambda value: int(value) * 2, str, int) + .filter(lambda value: value > 2, int) + .collect()) + plan = post.call_args.kwargs["json"] + source, terminal = plan["operators"] + assert plan["collect"] is True + assert plan["context"]["platforms"] == ["java"] + assert source["output"] == [terminal["id"]] + assert terminal["input"] == [source["id"]] + assert terminal["output"] == [] + assert terminal["operatorName"] == "mapPartitions" + assert terminal["data"]["jsonOutput"] is True + assert "outputType" not in terminal["data"] + udf = cloudpickle.loads(base64.b64decode(terminal["data"]["udf"])) + records = list(udf(iter(["1", "2", "3"]))) + assert all(type(record) is JsonRecord for record in records) + assert [json.loads(record) for record in records] == [4, 6] + + +def test_collect_does_not_change_graph(context, post): + source = context.load_collection([1]) + mapped = source.map(lambda value: int(value), str, int) + source.collect() + source_plan = post.call_args.kwargs["json"] + assert len(source_plan["operators"]) == 1 + assert source_plan["operators"][0]["output"] == [] + assert source.operator.outputOperator == [mapped.operator] + mapped.collect() + mapped.collect() + assert post.call_count == 3 + assert source.operator.outputOperator == [mapped.operator] + assert mapped.operator.outputOperator == [None] + + +def test_text_sink_keeps_legacy_request(context, post): + assert context.load_collection([1]).store_textfile("file:///tmp/output.txt") is None + plan = post.call_args.kwargs["json"] + assert "collect" not in plan + assert plan["operators"][-1]["operatorName"] == "textFileOutput" + + +def test_collect_earlier_branch(context, post): + source = context.load_collection([1]) + first = source.map(lambda value: int(value) * 2, str, int) + second = source.map(lambda value: int(value) * 3, str, int) + first.collect() + source_plan, terminal = post.call_args.kwargs["json"]["operators"] + assert source_plan["output"] == [terminal["id"]] + assert terminal["input"] == [source_plan["id"]] + assert source.operator.outputOperator == [second.operator] + + +@pytest.mark.parametrize("payload", [b'""', b"false", b"123", b'[1,true]', '"雪"'.encode()]) +def test_worker_frames_each_json_record_once(payload): + stream = io.BytesIO() + dump_stream(iter([JsonRecord(payload)]), stream) + assert stream.getvalue() == struct.pack("!i", len(payload)) + payload + struct.pack("!i", -1) + + +def test_worker_preserves_legacy_byte_values(): + stream = io.BytesIO() + write_with_length(b"ab", stream) + assert stream.getvalue() == b"\x00\x00\x00\x05b'ab'" diff --git a/python/src/pywy/tests/test_collect_integration.py b/python/src/pywy/tests/test_collect_integration.py new file mode 100644 index 000000000..1ddd6f1ac --- /dev/null +++ b/python/src/pywy/tests/test_collect_integration.py @@ -0,0 +1,235 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Collect regression tests against PYWAYANG_INTEGRATION_URL, never a mocked backend.""" + +import json +import math +import os + +import pytest +import requests + +from pywy.configuration import Configuration +from pywy.dataquanta import WayangContext +from pywy.platforms.java import JavaPlugin +from pywy.platforms.spark import SparkPlugin + + +@pytest.fixture +def api(monkeypatch): + url = os.environ.get("PYWAYANG_INTEGRATION_URL") + if not url: + pytest.skip("Set PYWAYANG_INTEGRATION_URL to run against a local Wayang backend") + post = requests.post + + def submit(*args, **kwargs): + kwargs.setdefault("timeout", 120) + return post(*args, **kwargs) + + monkeypatch.setattr(requests, "post", submit) + return url + + +@pytest.fixture +def context(api): + configuration = Configuration() + configuration.set_property("wayang.api.python.url", api) + return WayangContext(configuration).register({JavaPlugin}) + + +@pytest.mark.parametrize("values", [ + [], + [-(2 ** 63), -1, 0, 2 ** 63 - 1], + [1.5, -0.25, -0.0, 1.23456789012345], + [True, False, "123", "true", "null", ""], + ['quotes " and \\', "line\nfeed\ttab\0", "雪 😀", '${System.exit(1)}'], + [[1, [False, "雪"]], [], [2.5, True]], +]) +def test_native_collection_round_trip(context, values): + result = context.load_collection(values).collect() + assert json.dumps(result) == json.dumps(values) + + +@pytest.mark.parametrize("function, expected", [ + (lambda value: int(value) * 2, [2, 4, 6]), + (lambda value: float(value) / 2, [0.5, 1.0, 1.5]), + (lambda value: int(value) > 1, [False, True, True]), + (lambda value: "", ["", "", ""]), + (lambda value: "雪\n\t\x00" + value, ["雪\n\t\x001", "雪\n\t\x002", "雪\n\t\x003"]), + (lambda value: [int(value), (True, ["雪", 2.5])], + [[1, [True, ["雪", 2.5]]], [2, [True, ["雪", 2.5]]], [3, [True, ["雪", 2.5]]]]), +], ids=["integers", "floats", "booleans", "empty-strings", "unicode", "nested"]) +def test_transformed_values(context, function, expected): + result = context.load_collection([1, 2, 3]).map(function, str, str).collect() + assert json.dumps(result) == json.dumps(expected) + + +def test_required_example_and_runtime_type(context): + result = (context.load_collection([1, 2, 3]) + .map(lambda value: int(value) * 2, str, int).collect()) + assert result == [2, 4, 6] + assert all(type(value) is int for value in result) + assert context.load_collection([1]).map(lambda value: "123", str, int).collect() == ["123"] + + +def test_filter_and_map(context): + source = context.load_collection([1, 2, 3]) + assert source.filter(lambda value: int(value) > 1, str).collect() == ["2", "3"] + assert (source.map(lambda value: int(value) * 2, str, int) + .filter(lambda value: value > 2, int).collect()) == [4, 6] + assert (source.filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int).collect()) == [4, 6] + + +def test_empty_transformations(context): + assert context.load_collection([]).map(lambda value: int(value), str, int).collect() == [] + assert context.load_collection([1, 2]).filter(lambda value: False, str).collect() == [] + assert (context.load_collection([1, 2]).map(lambda value: int(value), str, int) + .filter(lambda value: value > 10, int).collect()) == [] + + +def test_repeat_collect_and_text_sink(context, tmp_path): + source = context.load_collection([1, 2, 3]) + mapped = source.map(lambda value: int(value) * 2, str, int) + assert source.collect() == [1, 2, 3] + assert mapped.collect() == [2, 4, 6] + assert mapped.collect() == [2, 4, 6] + output = tmp_path / "values.txt" + assert source.store_textfile(output.as_uri()) is None + assert output.read_text() == "1\n2\n3\n" + assert source.collect() == [1, 2, 3] + assert mapped.collect() == [2, 4, 6] + + +def test_collect_earlier_branch(context): + source = context.load_collection([1, 2, 3]) + first = source.map(lambda value: int(value) * 2, str, int) + second = source.map(lambda value: int(value) * 3, str, int) + assert first.collect() == [2, 4, 6] + assert second.collect() == [3, 6, 9] + assert first.collect() == [2, 4, 6] + + +def test_text_source(context, tmp_path): + path = tmp_path / "input.txt" + path.write_text("1\n2\n3\n", encoding="utf-8") + assert context.textfile(path.as_uri()).collect() == ["1", "2", "3"] + assert (context.textfile(path.as_uri()) + .map(lambda value: int(value) * 2, str, int).collect()) == [2, 4, 6] + + +def test_native_cartesian_pairs(context): + result = (context.load_collection([1, 2]) + .cartesian(context.load_collection(["a", "b"])).collect()) + assert sorted(result) == [[1, "a"], [1, "b"], [2, "a"], [2, "b"]] + + +@pytest.mark.parametrize("value, message", [ + (None, "NoneType"), ({"error": "user data"}, "dict"), ({1}, "set"), (b"data", "bytes"), + (object(), "object"), (math.nan, "finite"), (math.inf, "finite"), + (2 ** 63, "signed 64 bits"), +]) +def test_rejects_unsupported_udf_results(context, value, message): + with pytest.raises(requests.HTTPError, match=message) as failure: + context.load_collection([1]).map(lambda item: value, str, str).collect() + assert failure.value.response.status_code == 400 + + +def test_rejects_cyclic_udf_result(context): + cycle = [] + cycle.append(cycle) + with pytest.raises(requests.HTTPError, match="cyclic"): + context.load_collection([1]).map(lambda value: cycle, str, str).collect() + + +def test_failure_discards_partial_results(context): + with pytest.raises(requests.HTTPError, match="ZeroDivisionError") as failure: + (context.load_collection([1, 2, 3]) + .map(lambda value: 1 / (int(value) - 2), str, float).collect()) + assert failure.value.response.status_code == 400 + assert "division by zero" in failure.value.response.text + + +@pytest.fixture +def native_plan(): + return { + "context": {"origin": "scala", "platforms": ["java"], "configuration": {}}, + "operators": [{ + "id": 1, "operatorName": "inputCollection", "cat": "input", + "input": [], "output": [], "data": {"values": [1, 2, 3]}, + }], + } + + +@pytest.mark.parametrize("collect", [None, False, True]) +@pytest.mark.parametrize("data", [{"values": [1, 2, 3]}, {"udf": "List(1, 2, 3)"}]) +def test_legacy_and_collection_responses(api, native_plan, collect, data): + native_plan["operators"][0]["data"] = data + if collect is not None: + native_plan["collect"] = collect + response = requests.post(api, json=native_plan) + assert response.status_code == 200, response.text + if collect: + assert response.headers["Content-Type"].startswith("application/json") + assert response.json() == {"results": [1, 2, 3]} + else: + assert response.headers["Content-Type"].startswith("text/plain") + assert response.text == "Buffer(1, 2, 3)" + + +def test_collect_rejects_sink_before_execution(api, native_plan, tmp_path): + output = tmp_path / "must-not-exist.txt" + native_plan["collect"] = True + native_plan["operators"][0]["output"] = [2] + native_plan["operators"].append({ + "id": 2, "operatorName": "textFileOutput", "cat": "output", + "input": [1], "output": [], "data": {"filename": output.as_uri()}, + }) + response = requests.post(api, json=native_plan) + assert response.status_code == 400 + assert "Collect plans cannot contain output sinks" in response.text + assert not output.exists() + del native_plan["collect"] + response = requests.post(api, json=native_plan) + assert response.status_code == 200, response.text + assert response.text == "Success" + assert output.read_text() == "1\n2\n3\n" + + +@pytest.mark.parametrize("terminals", [0, 2]) +def test_rejects_ambiguous_terminals(api, native_plan, terminals): + source = native_plan["operators"][0] + native_plan["collect"] = True + native_plan["operators"] = [dict(source, id=index + 1) for index in range(terminals)] + response = requests.post(api, json=native_plan) + assert response.status_code == 400 + assert "exactly one terminal operator" in response.text + + +def test_collect_on_spark(api): + if os.environ.get("PYWAYANG_TEST_SPARK") != "1": + pytest.skip("Set PYWAYANG_TEST_SPARK=1 with a backend configured for local Spark") + configuration = Configuration() + configuration.set_property("wayang.api.python.url", api) + context = WayangContext(configuration).register({SparkPlugin}) + result = (context.load_collection([1, 2, 3]) + .map(lambda value: int(value) * 2, str, int).collect()) + assert sorted(result) == [2, 4, 6] + assert all(type(value) is int for value in result) + assert context.load_collection([1]).filter(lambda value: False, str).collect() == [] + assert context.load_collection([[1, [False, "雪"]], []]).collect() == [[1, [False, "雪"]], []] diff --git a/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/CollectResultTest.scala b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/CollectResultTest.scala new file mode 100644 index 000000000..5b1eb478c --- /dev/null +++ b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/CollectResultTest.scala @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.wayang.api.json + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.wayang.api.json.operatorfromjson.unary.MapPartitionsOperatorFromJson +import org.apache.wayang.api.json.parserutil.ParsePlanFromJson +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.Test + +class CollectResultTest { + private val mapper = new ObjectMapper() + + @Test + def nativeValuesKeepTheirTypes(): Unit = { + val values = Vector[Any](1.toByte, 2.toShort, 3, Long.MinValue, Long.MaxValue, + 1.25f, 2.5d, -0.0d, true, false, "123", "true", "null", "") + val result = mapper.readTree(CollectResult.response(values, jsonOutput = false)).get("results") + assertEquals(values.size, result.size()) + (0 to 4).foreach(index => assertTrue(result.get(index).isIntegralNumber)) + assertEquals(Long.MinValue, result.get(3).longValue()) + assertEquals(Long.MaxValue, result.get(4).longValue()) + assertEquals(1.25, result.get(5).doubleValue()) + assertEquals(2.5, result.get(6).doubleValue()) + assertEquals(java.lang.Double.doubleToRawLongBits(-0.0), + java.lang.Double.doubleToRawLongBits(result.get(7).doubleValue())) + assertTrue(result.get(8).isBoolean) + assertFalse(result.get(9).booleanValue()) + (10 to 13).foreach(index => assertTrue(result.get(index).isTextual)) + } + + @Test + def nestedContainersBecomeArrays(): Unit = { + val values = Vector[Any](Array(1, 2), java.util.Arrays.asList("雪", false), + (3L, Vector("nested", Vector.empty[Any]))) + assertEquals(mapper.readTree("""{"results":[[1,2],["雪",false],[3,["nested",[]]]]}"""), + mapper.readTree(CollectResult.response(values, jsonOutput = false))) + } + + @Test + def emptyResults(): Unit = { + Seq(false, true).foreach { encoded => + assertEquals("""{"results":[]}""", CollectResult.response(Vector.empty, encoded)) + } + } + + @Test + def stringsAreNotInterpretedWithoutEncodingMarker(): Unit = { + val text = "quotes \" and \\ and newline\n雪 ${System.exit(1)}" + val values = Vector[Any](text, "123", "true", "[1,2]", "{\"error\":\"data\"}") + val result = mapper.readTree(CollectResult.response(values, jsonOutput = false)).get("results") + values.zipWithIndex.foreach { case (value, index) => + assertTrue(result.get(index).isTextual) + assertEquals(value, result.get(index).textValue()) + } + } + + @Test + def decodesPythonRecords(): Unit = { + val values = Vector[Any]("1", "1.25", "true", "\"123\"", "\"\"", "[1,[false,\"雪\"]]") + assertEquals(mapper.readTree("""{"results":[1,1.25,true,"123","",[1,[false,"雪"]]]}"""), + mapper.readTree(CollectResult.response(values, jsonOutput = true))) + } + + @Test + def rejectsUnsupportedNativeValues(): Unit = { + Seq[Any](null, new Object(), Map("key" -> 1), java.math.BigInteger.TEN, + Double.NaN, Double.PositiveInfinity, Float.NegativeInfinity).foreach { value => + assertThrows(classOf[IllegalArgumentException], () => { + CollectResult.response(Vector(value), jsonOutput = false); () + }) + } + } + + @Test + def rejectsMalformedAndUnsupportedRecords(): Unit = { + Seq("", "not json", "1 2", "{").foreach { record => + assertThrows(classOf[JsonProcessingException], () => { + CollectResult.response(Vector(record), jsonOutput = true); () + }) + } + Seq[Any](1, "null", "{}", "[null]", "[{}]", "9223372036854775808", "1e400", + "{\"error\":1}", "{\"error\":\"message\",\"extra\":true}").foreach { record => + assertThrows(classOf[IllegalArgumentException], () => { + CollectResult.response(Vector(record), jsonOutput = true); () + }) + } + } + + @Test + def pythonFailureDiscardsEarlierResults(): Unit = { + val failure = assertThrows(classOf[IllegalArgumentException], () => { + CollectResult.response(Vector("1", "{\"error\":\"ZeroDivisionError: division by zero\"}"), + jsonOutput = true); () + }) + assertTrue(failure.getMessage.contains("Python collect failed: ZeroDivisionError")) + } + + @Test + def legacyPlansDefaultToTextAndUnencodedRecords(): Unit = { + val plan = ParsePlanFromJson.parsePlanFromString("""{ + "context": {"origin":"python", "platforms":["java"], "configuration":{}}, + "operators": [{"id":1, "operatorName":"mapPartitions", "cat":"unary", + "input":[2], "output":[], "data":{"udf":"unused"}}] + }""").get + assertFalse(plan.collect) + assertFalse(plan.operators.head.asInstanceOf[MapPartitionsOperatorFromJson].data.jsonOutput) + } +}