From 9b5453f9bb69bb850609e81193a67216c7202990 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Mon, 2 Nov 2026 11:58:20 +0530 Subject: [PATCH 1/2] Add PyWayang load_collection using literal JSON input --- python/README.md | 75 ++++++++++ python/src/pywy/core/serializer.py | 5 +- python/src/pywy/dataquanta.py | 17 ++- python/src/pywy/operators/__init__.py | 3 +- python/src/pywy/operators/source.py | 35 +++++ python/src/pywy/tests/test_load_collection.py | 112 +++++++++++++++ .../tests/test_load_collection_integration.py | 82 +++++++++++ .../main/scala/builder/JsonPlanBuilder.scala | 8 +- .../input/InputCollectionFromJson.scala | 36 ++++- .../wayang/api/json/InputCollectionTest.scala | 135 ++++++++++++++++++ 10 files changed, 499 insertions(+), 9 deletions(-) create mode 100644 python/src/pywy/tests/test_load_collection.py create mode 100644 python/src/pywy/tests/test_load_collection_integration.py create mode 100644 wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala diff --git a/python/README.md b/python/README.md index eafa2e7ee..9c3388519 100644 --- a/python/README.md +++ b/python/README.md @@ -119,6 +119,51 @@ configuration.set_property("wayang.api.python.port", "8080") The same values can be supplied through environment variables: `WAYANG_API_URL`, or `WAYANG_API_HOST` and `WAYANG_API_PORT`. +## In-memory collections + +`load_collection()` snapshots a Python list or tuple and returns a `DataQuanta`: + +```python +from pywy.dataquanta import WayangContext +from pywy.platforms.java import JavaPlugin + +ctx = WayangContext().register({JavaPlugin}) +ctx.load_collection([1, 2, 3]).store_textfile("file:///tmp/numbers.txt") +``` + +Supported values are built-in strings, booleans, signed 64-bit integers, finite +floats, and nested lists or tuples. Tuples become lists. Empty and mixed +collections are supported. Input order and nested structure are preserved by +the source; subsequent operators/platforms determine result ordering. `None`, +mappings, sets, generators, custom objects, cyclic containers, out-of-range +integers, and non-finite floats are rejected before submission. + +The current Python worker passes records to UDFs as strings, including records +from collection sources. Convert numeric inputs explicitly: + +```python +(ctx.load_collection([1, 2, 3]) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile("file:///tmp/doubled.txt")) +``` + +This example uses the existing fusion of adjacent Python transformations into +one worker call. The standalone Python filter and primitive-typed map-to-text +paths have existing limitations (empty filter output and incompatible sink +types, respectively), also reproducible with text-file sources. Collection +input does not change those paths. + +Nested values remain structured on the JVM, but reach Python UDFs and text sinks +in Java list notation, such as `[1, 2]`. This is not a JSON round-trip format. +Text sinks use the existing JVM string representation, one record per line; +embedded newlines in strings produce additional lines. File paths refer to the +backend's filesystem. Collection data is held in memory and sent in one JSON +request, so this source is intended for small local datasets. + +This API requires a backend supporting literal `inputCollection.data.values`. +The backend continues to accept existing `inputCollection.data.udf` plans. + ## Docker The Python API can be run with the Wayang REST API as a separate container. @@ -148,3 +193,33 @@ docker network rm wayang-python ### Testing python code You can run the python tests by using pytest, the requirements for the tests are listed in `python/src/pywy/requirements.txt`. To run the tests navigate to the base wayang folder, e.g. `/var/www/html` and run `pytest -s python/src/pywy` if you need to pass a specific configuration for your use case you can also add a config flag `pytest -s --config=pathToYourConfig python/src/pywy/` + +The collection unit tests run without a backend: + +```bash +PYTHONPATH=python/src pytest -q python/src/pywy/tests/test_load_collection.py +``` + +For the collection integration tests, build and install the JSON API and its +dependencies with JDK 17 (`./mvnw install -pl :wayang-api-json -am -DskipTests`). +Start `org.apache.wayang.api.json.Main` as described in the JSON API README, or use: + +```bash +./mvnw -pl :wayang-api-json exec:java \ + -Dexec.mainClass=org.apache.wayang.api.json.Main -Dexec.args=8080 \ + -Dwayang.configuration=file:///absolute/path/to/wayang.properties +``` + +The properties file must set `wayang.api.python.path` to the Python executable, +`wayang.api.python.worker` to the absolute path of +`python/src/pywy/execution/worker.py`, and `wayang.api.python.env.path` to the +absolute path of `python/src`. Run from the repository root against a backend +on the same machine (the tests read its output files): + +```bash +PYTHONPATH=python/src \ +PYWAYANG_INTEGRATION_URL=http://localhost:8080/wayang-api-json/submit-plan/json \ +pytest -q python/src/pywy/tests/test_load_collection_integration.py +``` + +Without `PYWAYANG_INTEGRATION_URL`, these integration tests explicitly skip. diff --git a/python/src/pywy/core/serializer.py b/python/src/pywy/core/serializer.py index 7e727c4bf..c2305a7f6 100644 --- a/python/src/pywy/core/serializer.py +++ b/python/src/pywy/core/serializer.py @@ -60,7 +60,10 @@ def serialize(self, operator): if hasattr(operator, "output_type") and operator.output_type is not None: json_operator["data"]["outputType"] = ndim_from_type(operator.output_type).to_json() - if operator.json_name == "filter": + if operator.json_name == "inputCollection": + json_operator["data"]["values"] = operator.values + + elif operator.json_name == "filter": json_operator["data"]["udf"] = base64.b64encode(cloudpickle.dumps(operator.use_predicate)).decode('utf-8') elif operator.json_name == "reduceBy": diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py index dd8490ba8..4b98b4807 100644 --- a/python/src/pywy/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import Set, List, Optional, cast +from typing import Set, List, Optional, Tuple, Union, cast from pywy.core.core import Plugin, PywyPlan from pywy.operators.base import PO_T @@ -56,6 +56,21 @@ def unregister(self, *plugins: Plugin): def textfile(self, file_path: str) -> "DataQuanta[str]": return DataQuanta(self, TextFileSource(file_path)) + def load_collection(self, collection: Union[List[T], Tuple[T, ...]]) -> "DataQuanta[T]": + """Create a source from a snapshot of an ordered list or tuple. + + Values may be strings, booleans, signed 64-bit integers, finite floats, + or nested lists/tuples (tuples become lists). Empty collections are valid. + Other values, including None, mappings, sets, and cyclic containers, + raise TypeError or ValueError before a plan is submitted. + + As with other sources, the current Python worker passes records to UDFs + as strings. Numeric UDFs must convert their input explicitly with int() + or float(). See the README for a fused transformation example and + existing worker limitations. + """ + return DataQuanta(self, CollectionSource(collection)) + def parquet( self, file_path: str, projection: Optional[List[str]] = None, column_names: Optional[List[str]] = None ) -> "DataQuanta[Record]": diff --git a/python/src/pywy/operators/__init__.py b/python/src/pywy/operators/__init__.py index b4740fd3c..7a349cd03 100644 --- a/python/src/pywy/operators/__init__.py +++ b/python/src/pywy/operators/__init__.py @@ -19,7 +19,7 @@ from pywy.operators.binary import BinaryToUnaryOperator, JoinOperator, DLTrainingOperator, PredictOperator, \ CartesianOperator from pywy.operators.sink import TextFileSink, SinkOperator -from pywy.operators.source import TextFileSource, ParquetSource, SourceUnaryOperator +from pywy.operators.source import CollectionSource, TextFileSource, ParquetSource, SourceUnaryOperator from pywy.operators.unary import UnaryToUnaryOperator, FilterOperator, MapOperator, FlatmapOperator, \ ReduceByKeyOperator, SortOperator @@ -29,6 +29,7 @@ BinaryToUnaryOperator, TextFileSink, TextFileSource, + CollectionSource, ParquetSource, FilterOperator, SinkOperator, diff --git a/python/src/pywy/operators/source.py b/python/src/pywy/operators/source.py index d55caba44..87330c48b 100644 --- a/python/src/pywy/operators/source.py +++ b/python/src/pywy/operators/source.py @@ -15,6 +15,7 @@ # limitations under the License. # +import math from typing import List, Optional from pywy.operators.base import PywyOperator @@ -44,6 +45,40 @@ def __repr__(self): return super().__repr__() +class CollectionSource(SourceUnaryOperator): + def __init__(self, collection): + # Empty and heterogeneous collections use the backend's generic type. + super().__init__('Collection', output_type=None) + self.json_name = "inputCollection" + if type(collection) not in (list, tuple): + raise TypeError("load_collection expects a list or tuple") + self.values = self._snapshot(collection, set(), "collection") + + @staticmethod + def _snapshot(value, ancestors, path): + if type(value) in (str, bool): + return value + if type(value) is int: + if not -(2 ** 63) <= value < 2 ** 63: + raise ValueError(f"{path}: integers must fit in signed 64 bits") + return value + if type(value) is float: + if not math.isfinite(value): + raise ValueError(f"{path}: floats must be finite") + return value + if type(value) in (list, tuple): + if id(value) in ancestors: + raise ValueError(f"{path}: cyclic collections are not supported") + ancestors.add(id(value)) + result = [ + CollectionSource._snapshot(item, ancestors, f"{path}[{index}]") + for index, item in enumerate(value) + ] + ancestors.remove(id(value)) + return result + raise TypeError(f"{path}: unsupported collection value of type {type(value).__name__}") + + class TextFileSource(SourceUnaryOperator): path: str json_name: str diff --git a/python/src/pywy/tests/test_load_collection.py b/python/src/pywy/tests/test_load_collection.py new file mode 100644 index 000000000..757119b67 --- /dev/null +++ b/python/src/pywy/tests/test_load_collection.py @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import json +from unittest.mock import patch + +import cloudpickle +import pytest + +from pywy.dataquanta import DataQuanta, WayangContext +from pywy.operators import CollectionSource, SourceUnaryOperator +from pywy.platforms.java import JavaPlugin + + +@pytest.mark.parametrize("values", [ + [1, -2, 0, -(2 ** 63), 2 ** 63 - 1], + [1.5, -0.0, 1e-300, 1e300], + ['quotes " and \\', "line\nfeed\ttab", "雪 😀", "", '${System.exit(1)}'], + [], + [True, False], + [[1, 2], [], [3, ["nested", False]]], + [1, "two", 3.5, True], +]) +def test_collection_plan(values): + context = WayangContext().register({JavaPlugin}) + data = context.load_collection(values) + assert isinstance(data, DataQuanta) + assert data.context is context + assert isinstance(data.operator, SourceUnaryOperator) + assert data.operator.inputs == 0 + assert data.operator.outputs == 1 + + with patch("pywy.core.core.requests.post") as post: + data.store_textfile("file:///tmp/collection.txt") + + plan = json.loads(json.dumps(post.call_args.kwargs["json"], allow_nan=False)) + assert plan["context"]["origin"] == "python" + assert plan["context"]["platforms"] == ["java"] + source, sink = plan["operators"] + assert source == { + "id": 1, "operatorName": "inputCollection", "cat": "input", + "input": [], "output": [2], "data": {"values": values}, + } + assert sink["operatorName"] == "textFileOutput" + assert sink["input"] == [1] + assert sink["data"]["filename"] == "file:///tmp/collection.txt" + + +def test_snapshot_and_tuples(): + nested = [1, (2, 3)] + values = [nested, nested] + source = CollectionSource(values) + nested.append(4) + values.clear() + assert source.values == [[1, [2, 3]], [1, [2, 3]]] + assert CollectionSource((1, 2)).values == [1, 2] + + +@pytest.mark.parametrize("values", ["abc", b"abc", {1, 2}, {"a": 1}, None, iter([1, 2])]) +def test_reject_unordered_or_non_collection_inputs(values): + with pytest.raises(TypeError, match="expects a list or tuple"): + WayangContext().load_collection(values) + + +@pytest.mark.parametrize("value", [None, {}, {1}, b"bytes", complex(1, 2), object()]) +def test_reject_unsupported_values(value): + with pytest.raises(TypeError, match=r"collection\[0\]\[0\]: unsupported"): + WayangContext().load_collection([[value]]) + + +@pytest.mark.parametrize("value", [-(2 ** 63) - 1, 2 ** 63, float("nan"), float("inf"), -float("inf")]) +def test_reject_out_of_range_values(value): + with pytest.raises(ValueError, match=r"collection\[0\]"): + WayangContext().load_collection([value]) + + +def test_reject_cycles(): + values = [] + values.append(values) + with pytest.raises(ValueError, match="cyclic"): + WayangContext().load_collection(values) + + +def test_collection_transformation(): + with patch("pywy.core.core.requests.post") as post: + (WayangContext().register({JavaPlugin}).load_collection([3, 1, 2]) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile("file:///tmp/doubled.txt")) + source, transform, sink = post.call_args.kwargs["json"]["operators"] + assert source["output"] == [transform["id"]] + assert transform["input"] == [source["id"]] + assert transform["output"] == [sink["id"]] + assert sink["input"] == [transform["id"]] + assert transform["operatorName"] == "mapPartitions" + udf = cloudpickle.loads(base64.b64decode(transform["data"]["udf"])) + assert list(udf(iter(["3", "1", "2"]))) == [6, 4] diff --git a/python/src/pywy/tests/test_load_collection_integration.py b/python/src/pywy/tests/test_load_collection_integration.py new file mode 100644 index 000000000..17f14f51e --- /dev/null +++ b/python/src/pywy/tests/test_load_collection_integration.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Run against a local backend using PYWAYANG_INTEGRATION_URL (see README).""" + +import os + +import pytest +import requests + +from pywy.configuration import Configuration +from pywy.dataquanta import WayangContext +from pywy.platforms.java import JavaPlugin + + +@pytest.fixture +def collection_context(monkeypatch): + url = os.environ.get("PYWAYANG_INTEGRATION_URL") + if not url: + pytest.skip("Set PYWAYANG_INTEGRATION_URL to run against a local Wayang backend") + configuration = Configuration() + configuration.set_property("wayang.api.python.url", url) + post = requests.post + + def checked_post(*args, **kwargs): + response = post(*args, **kwargs, timeout=120) + assert response.status_code == 200, response.text + assert response.text == "Success" + return response + + # Execute real HTTP requests and check errors that PywyPlan currently ignores. + monkeypatch.setattr("pywy.core.core.requests.post", checked_post) + return WayangContext(configuration).register({JavaPlugin}) + + +@pytest.mark.parametrize("values, expected", [ + ([1, -2, 0, -(2 ** 63), 2 ** 63 - 1], "1\n-2\n0\n-9223372036854775808\n9223372036854775807\n"), + ([1.5, -0.25, -0.0, 1.23456789012345], "1.5\n-0.25\n-0.0\n1.23456789012345\n"), + (['quotes " and \\', "雪 😀", "line\nfeed\ttab", "", '${System.exit(1)}'], + 'quotes " and \\\n雪 😀\nline\nfeed\ttab\n\n${System.exit(1)}\n'), + ([], ""), + ([True, False], "true\nfalse\n"), + ([[1, 2], [], [3, ["nested", False]]], "[1, 2]\n[]\n[3, [nested, false]]\n"), +]) +def test_collection_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "collection.txt" + collection_context.load_collection(values).store_textfile(output.as_uri()) + assert output.read_text(encoding="utf-8") == expected + + +@pytest.mark.parametrize("values, expected", [([1, 2, 3], ["3", "5", "7"]), ([], [])]) +def test_collection_maps_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "doubled.txt" + (collection_context.load_collection(values) + .map(lambda value: int(value) * 2, str, int) + .map(lambda value: value + 1, int, int) + .store_textfile(output.as_uri())) + assert output.read_text(encoding="utf-8").splitlines() == expected + + +@pytest.mark.parametrize("values, expected", [([3, 1, 2], ["6", "4"]), ([], [])]) +def test_collection_filter_map_to_textfile(collection_context, tmp_path, values, expected): + output = tmp_path / "filtered.txt" + (collection_context.load_collection(values) + .filter(lambda value: int(value) > 1, str) + .map(lambda value: int(value) * 2, str, int) + .store_textfile(output.as_uri())) + assert output.read_text(encoding="utf-8").splitlines() == expected diff --git a/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala b/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala index d9d5a2a62..ec9dc3530 100644 --- a/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala +++ b/wayang-api/wayang-api-json/src/main/scala/builder/JsonPlanBuilder.scala @@ -18,7 +18,7 @@ package org.apache.wayang.api.json.builder import org.apache.wayang.api.json.operatorfromjson.OperatorFromJson.ExecutionPlatforms -import org.apache.wayang.api.json.parserutil.{SerializableIterable, SerializableLambda, SerializableLambda2} +import org.apache.wayang.api.json.parserutil.{SerializableLambda, SerializableLambda2} import org.apache.wayang.api.json.operatorfromjson.{ComposedOperatorFromJson, OperatorFromJson} import org.apache.wayang.api.json.operatorfromjson.binary.{CartesianOperatorFromJson, CoGroupOperatorFromJson, IntersectOperatorFromJson, JoinOperatorFromJson, PredictOperatorFromJson, DLTrainingOperatorFromJson, UnionOperatorFromJson} import org.apache.wayang.api.json.operatorfromjson.other.KMeansFromJson @@ -213,11 +213,11 @@ class JsonPlanBuilder() { } private def visit(operator: InputCollectionFromJson, planBuilder: PlanBuilder): DataQuanta[Any] = { - val iterable = SerializableIterable.create(operator.data.udf) + val iterable = operator.data.collection if (!ExecutionPlatforms.All.contains(operator.executionPlatform)) - planBuilder.loadCollection(iterable.get) + planBuilder.loadCollection(iterable) else - planBuilder.loadCollection(iterable.get).withTargetPlatforms(getExecutionPlatform(operator.executionPlatform)) + planBuilder.loadCollection(iterable).withTargetPlatforms(getExecutionPlatform(operator.executionPlatform)) } private def visit(operator: TableInputFromJson, planBuilder: PlanBuilder): DataQuanta[Any] = { diff --git a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala index 9d514643d..8522b229b 100644 --- a/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala +++ b/wayang-api/wayang-api-json/src/main/scala/operatorfromjson/input/InputCollectionFromJson.scala @@ -18,7 +18,11 @@ package org.apache.wayang.api.json.operatorfromjson.input import com.fasterxml.jackson.annotation.JsonTypeName +import com.fasterxml.jackson.databind.JsonNode import org.apache.wayang.api.json.operatorfromjson.OperatorFromJson +import org.apache.wayang.api.json.parserutil.SerializableIterable + +import scala.collection.JavaConverters._ @JsonTypeName(OperatorFromJson.OperatorNames.InputCollection) case class InputCollectionFromJson(override val id: Long, @@ -32,7 +36,35 @@ case class InputCollectionFromJson(override val id: Long, } object InputCollectionFromJson { - case class Data(udf: String) -} + case class Data(udf: String = null, values: JsonNode = null) { + def collection: Iterable[Any] = { + require((udf != null) != (values != null), + "inputCollection requires exactly one of data.udf or data.values") + if (values == null) return SerializableIterable.create(udf).get + require(values.isArray, "inputCollection data.values must be an array") + values.elements().asScala.map(readValue).toVector + } + } + + // Use serializable JVM values, not Jackson nodes or generated Scala code. + private def readValue(value: JsonNode): Any = { + if (value.isTextual) value.textValue() + else if (value.isBoolean) value.booleanValue() + else if (value.isIntegralNumber) { + require(value.canConvertToLong, "inputCollection integers must fit in signed 64 bits") + value.longValue() + } else if (value.isFloatingPointNumber) { + val number = value.doubleValue() + require(!number.isNaN && !number.isInfinity, "inputCollection floats must be finite") + number + } else if (value.isArray) { + val result = new java.util.ArrayList[Any]() + value.elements().asScala.foreach(item => result.add(readValue(item))) + result + } else { + throw new IllegalArgumentException(s"Unsupported inputCollection value: $value") + } + } +} diff --git a/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala new file mode 100644 index 000000000..c129a085d --- /dev/null +++ b/wayang-api/wayang-api-json/src/test/scala/org/apache/wayang/api/json/InputCollectionTest.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.wayang.api.json + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.wayang.api.json.builder.JsonPlanBuilder +import org.apache.wayang.api.json.operatorfromjson.input.InputCollectionFromJson +import org.apache.wayang.api.json.parserutil.{ParsePlanFromJson, SerializableIterable} +import org.apache.wayang.core.api.Configuration +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} + +class InputCollectionTest { + @TempDir + var directory: Path = _ + + private val mapper = new ObjectMapper() + + private def values(json: String): Iterable[Any] = + InputCollectionFromJson.Data(values = mapper.readTree(json)).collection + + private def roundTrip[T](value: T): T = { + val bytes = new ByteArrayOutputStream() + val output = new ObjectOutputStream(bytes) + try output.writeObject(value) finally output.close() + val input = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray)) + try input.readObject().asInstanceOf[T] finally input.close() + } + + @Test + def literalTypesAndSerialization(): Unit = { + val result = values("""[1,-9223372036854775808,9223372036854775807,1.25,true,"雪",[2,[],"nested"]]""").toVector + assertEquals(Vector(1L, Long.MinValue, Long.MaxValue, 1.25, true, "雪"), result.take(6)) + assertTrue(result.head.isInstanceOf[Long]) + assertTrue(result(3).isInstanceOf[Double]) + assertEquals(java.util.Arrays.asList(2L, new java.util.ArrayList[Any](), "nested"), result(6)) + assertEquals(result, roundTrip(result)) + } + + @Test + def emptyCollection(): Unit = { + assertTrue(values("[]").isEmpty) + assertTrue(roundTrip(values("[]")).isEmpty) + } + + @Test + def rejectsUnsupportedValues(): Unit = { + Seq("null", "{}", "1", "[null]", "[{}]", "[9223372036854775808]", "[1e400]").foreach { json => + assertThrows(classOf[IllegalArgumentException], () => { values(json); () }, json) + } + assertThrows(classOf[IllegalArgumentException], () => InputCollectionFromJson.Data().collection) + assertThrows(classOf[IllegalArgumentException], () => + InputCollectionFromJson.Data("List(1)", mapper.readTree("[]")).collection) + } + + @Test + def legacyIterableSerialization(): Unit = { + val iterable = SerializableIterable.create("List(1, 2, 3)") + assertEquals(List(1, 2, 3), roundTrip(iterable).get) + assertEquals(List(1, 2, 3), iterable.get) + assertEquals(List(1, 2, 3), roundTrip(iterable).get) + } + + @Test + def operatorJsonRoundTrip(): Unit = { + Seq( + InputCollectionFromJson.Data("List(1, 2, 3)"), + InputCollectionFromJson.Data(values = mapper.readTree("[1,2,3]")), + InputCollectionFromJson.Data(values = mapper.readTree("[]")) + ).foreach { data => + val source = InputCollectionFromJson(1L, Array.empty[Long], Array(2L), "input", "inputCollection", data) + val replaced = source.replaceId(3L).asInstanceOf[InputCollectionFromJson] + assertEquals(3L, replaced.id) + assertEquals(data.collection.toList, replaced.data.collection.toList) + } + } + + private def execute(data: String, filename: String, origin: String = "python"): String = { + val output = directory.resolve(filename) + val plan = ParsePlanFromJson.parsePlanFromString(s"""{ + "context": {"origin": "$origin", "platforms": ["java"], "configuration": {}}, + "operators": [ + {"id": 1, "operatorName": "inputCollection", "cat": "input", + "input": [], "output": [2], "data": $data, "executionPlatform": "java"}, + {"id": 2, "operatorName": "textFileOutput", "cat": "output", + "input": [1], "output": [], "data": {"filename": "${output.toUri}"}} + ] + }""").get + val builder = new JsonPlanBuilder() + builder.configuration = new Configuration() + builder.setPlatforms(plan.context.platforms).setOrigin(plan.context.origin).setOperators(plan.operators).execute() + new String(Files.readAllBytes(output), StandardCharsets.UTF_8) + } + + @Test + def literalPlanWritesFile(): Unit = { + assertEquals("1\n2\n3\n", execute("""{"values": [1,2,3]}""", "integers.txt")) + assertEquals("", execute("""{"values": []}""", "empty.txt")) + assertEquals("[1, 2]\n[]\n", execute("""{"values": [[1,2],[]]}""", "nested.txt")) + } + + @Test + def stringsAreData(): Unit = { + val text = "quotes \" and \\ and newline\n雪 ${System.exit(1)}" + val data = mapper.createObjectNode() + data.putArray("values").add(text).add("") + assertEquals(text + "\n\n", execute(data.toString, "strings.txt")) + } + + @Test + def legacyPlansStillWriteFiles(): Unit = { + assertEquals("1\n2\n3\n", execute("""{"udf": "List(1, 2, 3)"}""", "legacy.txt", "scala")) + assertEquals("", execute("""{"udf": "List.empty[Int]"}""", "legacy-empty.txt", "scala")) + } +} From 00d852db4277d612c1d84e40234152c384bedbae Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Fri, 13 Nov 2026 21:31:21 +0530 Subject: [PATCH 2/2] Support NumPy collection inputs and address review feedback --- python/README.md | 74 +++-------------- python/src/pywy/dataquanta.py | 15 ++-- python/src/pywy/operators/source.py | 28 +++++-- python/src/pywy/tests/test_load_collection.py | 83 ++++++++++++++++++- .../tests/test_load_collection_integration.py | 31 ++++++- 5 files changed, 153 insertions(+), 78 deletions(-) diff --git a/python/README.md b/python/README.md index 9c3388519..d990aeb38 100644 --- a/python/README.md +++ b/python/README.md @@ -121,49 +121,23 @@ The same values can be supplied through environment variables: ## In-memory collections -`load_collection()` snapshots a Python list or tuple and returns a `DataQuanta`: +`load_collection()` creates a `DataQuanta` from a Python list, tuple, or NumPy +array. It copies the input, preserving order and nesting; empty inputs are valid. +Supported elements are strings, booleans, signed 64-bit integers, finite floats, +and nested lists, tuples, or arrays. NumPy boolean, integer, float16/32/64, and +Unicode values are supported within the same limits. Unsupported values raise +`TypeError` or `ValueError` before execution. ```python +import numpy as np from pywy.dataquanta import WayangContext from pywy.platforms.java import JavaPlugin ctx = WayangContext().register({JavaPlugin}) ctx.load_collection([1, 2, 3]).store_textfile("file:///tmp/numbers.txt") +ctx.load_collection(np.array([1.5, 2.5])).store_textfile("file:///tmp/floats.txt") ``` -Supported values are built-in strings, booleans, signed 64-bit integers, finite -floats, and nested lists or tuples. Tuples become lists. Empty and mixed -collections are supported. Input order and nested structure are preserved by -the source; subsequent operators/platforms determine result ordering. `None`, -mappings, sets, generators, custom objects, cyclic containers, out-of-range -integers, and non-finite floats are rejected before submission. - -The current Python worker passes records to UDFs as strings, including records -from collection sources. Convert numeric inputs explicitly: - -```python -(ctx.load_collection([1, 2, 3]) - .filter(lambda value: int(value) > 1, str) - .map(lambda value: int(value) * 2, str, int) - .store_textfile("file:///tmp/doubled.txt")) -``` - -This example uses the existing fusion of adjacent Python transformations into -one worker call. The standalone Python filter and primitive-typed map-to-text -paths have existing limitations (empty filter output and incompatible sink -types, respectively), also reproducible with text-file sources. Collection -input does not change those paths. - -Nested values remain structured on the JVM, but reach Python UDFs and text sinks -in Java list notation, such as `[1, 2]`. This is not a JSON round-trip format. -Text sinks use the existing JVM string representation, one record per line; -embedded newlines in strings produce additional lines. File paths refer to the -backend's filesystem. Collection data is held in memory and sent in one JSON -request, so this source is intended for small local datasets. - -This API requires a backend supporting literal `inputCollection.data.values`. -The backend continues to accept existing `inputCollection.data.udf` plans. - ## Docker The Python API can be run with the Wayang REST API as a separate container. @@ -194,32 +168,6 @@ docker network rm wayang-python ### Testing python code You can run the python tests by using pytest, the requirements for the tests are listed in `python/src/pywy/requirements.txt`. To run the tests navigate to the base wayang folder, e.g. `/var/www/html` and run `pytest -s python/src/pywy` if you need to pass a specific configuration for your use case you can also add a config flag `pytest -s --config=pathToYourConfig python/src/pywy/` -The collection unit tests run without a backend: - -```bash -PYTHONPATH=python/src pytest -q python/src/pywy/tests/test_load_collection.py -``` - -For the collection integration tests, build and install the JSON API and its -dependencies with JDK 17 (`./mvnw install -pl :wayang-api-json -am -DskipTests`). -Start `org.apache.wayang.api.json.Main` as described in the JSON API README, or use: - -```bash -./mvnw -pl :wayang-api-json exec:java \ - -Dexec.mainClass=org.apache.wayang.api.json.Main -Dexec.args=8080 \ - -Dwayang.configuration=file:///absolute/path/to/wayang.properties -``` - -The properties file must set `wayang.api.python.path` to the Python executable, -`wayang.api.python.worker` to the absolute path of -`python/src/pywy/execution/worker.py`, and `wayang.api.python.env.path` to the -absolute path of `python/src`. Run from the repository root against a backend -on the same machine (the tests read its output files): - -```bash -PYTHONPATH=python/src \ -PYWAYANG_INTEGRATION_URL=http://localhost:8080/wayang-api-json/submit-plan/json \ -pytest -q python/src/pywy/tests/test_load_collection_integration.py -``` - -Without `PYWAYANG_INTEGRATION_URL`, these integration tests explicitly skip. +Collection unit tests run without a backend. See the setup instructions in +[`test_load_collection_integration.py`](src/pywy/tests/test_load_collection_integration.py) +for running the collection tests against a local REST backend. diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py index 4b98b4807..702331378 100644 --- a/python/src/pywy/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -17,6 +17,8 @@ from typing import Set, List, Optional, Tuple, Union, cast +from numpy import ndarray + from pywy.core.core import Plugin, PywyPlan from pywy.operators.base import PO_T from pywy.types import (GenericTco, Predicate, Function, BiFunction, FlatmapFunction, IterableOut, T, In, Out) @@ -56,18 +58,19 @@ def unregister(self, *plugins: Plugin): def textfile(self, file_path: str) -> "DataQuanta[str]": return DataQuanta(self, TextFileSource(file_path)) - def load_collection(self, collection: Union[List[T], Tuple[T, ...]]) -> "DataQuanta[T]": - """Create a source from a snapshot of an ordered list or tuple. + def load_collection(self, collection: Union[List[T], Tuple[T, ...], ndarray]) -> "DataQuanta[T]": + """Create a source from a snapshot of a list, tuple, or NumPy array. Values may be strings, booleans, signed 64-bit integers, finite floats, - or nested lists/tuples (tuples become lists). Empty collections are valid. + or nested lists/tuples/arrays (containers become lists). NumPy boolean, + integer, float16/32/64, and Unicode scalars are converted to Python values. + Arrays must have at least one dimension at the top level; nested scalar + arrays become scalar values. Empty collections are valid. Other values, including None, mappings, sets, and cyclic containers, raise TypeError or ValueError before a plan is submitted. As with other sources, the current Python worker passes records to UDFs - as strings. Numeric UDFs must convert their input explicitly with int() - or float(). See the README for a fused transformation example and - existing worker limitations. + as strings. Numeric UDFs must convert their input with int() or float(). """ return DataQuanta(self, CollectionSource(collection)) diff --git a/python/src/pywy/operators/source.py b/python/src/pywy/operators/source.py index 87330c48b..5db70dede 100644 --- a/python/src/pywy/operators/source.py +++ b/python/src/pywy/operators/source.py @@ -18,6 +18,8 @@ import math from typing import List, Optional +import numpy as np + from pywy.operators.base import PywyOperator from pywy.basic.data.record import Record from pywy.types import GenericTco @@ -50,12 +52,20 @@ def __init__(self, collection): # Empty and heterogeneous collections use the backend's generic type. super().__init__('Collection', output_type=None) self.json_name = "inputCollection" - if type(collection) not in (list, tuple): - raise TypeError("load_collection expects a list or tuple") + if type(collection) not in (list, tuple, np.ndarray): + raise TypeError("load_collection expects a list, tuple, or NumPy array") + if type(collection) is np.ndarray and collection.ndim == 0: + raise TypeError("load_collection expects a NumPy array with at least one dimension") self.values = self._snapshot(collection, set(), "collection") @staticmethod def _snapshot(value, ancestors, path): + # Apply the same JSON type and range checks after NumPy conversion. + if isinstance(value, np.generic) and value.dtype.kind in "biufU": + value = value.item() + if type(value) is np.ndarray: + if value.dtype.kind not in "biufUO" or (value.dtype.kind == "f" and value.dtype.itemsize > 8): + raise TypeError(f"{path}: unsupported NumPy dtype {value.dtype}") if type(value) in (str, bool): return value if type(value) is int: @@ -66,14 +76,18 @@ def _snapshot(value, ancestors, path): if not math.isfinite(value): raise ValueError(f"{path}: floats must be finite") return value - if type(value) in (list, tuple): + if type(value) in (list, tuple, np.ndarray): if id(value) in ancestors: raise ValueError(f"{path}: cyclic collections are not supported") ancestors.add(id(value)) - result = [ - CollectionSource._snapshot(item, ancestors, f"{path}[{index}]") - for index, item in enumerate(value) - ] + items = value.tolist() if type(value) is np.ndarray else value + if type(value) is np.ndarray and value.ndim == 0: + result = CollectionSource._snapshot(items, ancestors, path) + else: + result = [ + CollectionSource._snapshot(item, ancestors, f"{path}[{index}]") + for index, item in enumerate(items) + ] ancestors.remove(id(value)) return result raise TypeError(f"{path}: unsupported collection value of type {type(value).__name__}") diff --git a/python/src/pywy/tests/test_load_collection.py b/python/src/pywy/tests/test_load_collection.py index 757119b67..7b0cb22b3 100644 --- a/python/src/pywy/tests/test_load_collection.py +++ b/python/src/pywy/tests/test_load_collection.py @@ -20,6 +20,7 @@ from unittest.mock import patch import cloudpickle +import numpy as np import pytest from pywy.dataquanta import DataQuanta, WayangContext @@ -45,9 +46,12 @@ def test_collection_plan(values): assert data.operator.inputs == 0 assert data.operator.outputs == 1 + # Capture the submitted JSON without requiring a running REST server. + # Actual HTTP execution and file contents are covered by the integration tests. with patch("pywy.core.core.requests.post") as post: data.store_textfile("file:///tmp/collection.txt") + post.assert_called_once() plan = json.loads(json.dumps(post.call_args.kwargs["json"], allow_nan=False)) assert plan["context"]["origin"] == "python" assert plan["context"]["platforms"] == ["java"] @@ -73,7 +77,7 @@ def test_snapshot_and_tuples(): @pytest.mark.parametrize("values", ["abc", b"abc", {1, 2}, {"a": 1}, None, iter([1, 2])]) def test_reject_unordered_or_non_collection_inputs(values): - with pytest.raises(TypeError, match="expects a list or tuple"): + with pytest.raises(TypeError, match="expects a list, tuple, or NumPy array"): WayangContext().load_collection(values) @@ -97,6 +101,7 @@ def test_reject_cycles(): def test_collection_transformation(): + # Inspect the serialized pipeline; only the HTTP request is mocked. with patch("pywy.core.core.requests.post") as post: (WayangContext().register({JavaPlugin}).load_collection([3, 1, 2]) .filter(lambda value: int(value) > 1, str) @@ -110,3 +115,79 @@ def test_collection_transformation(): assert transform["operatorName"] == "mapPartitions" udf = cloudpickle.loads(base64.b64decode(transform["data"]["udf"])) assert list(udf(iter(["3", "1", "2"]))) == [6, 4] + + +@pytest.mark.parametrize("values, expected", [ + (np.array([1, -2], dtype=np.int32), [1, -2]), + (np.array([-(2 ** 63), 2 ** 63 - 1], dtype=np.int64), [-(2 ** 63), 2 ** 63 - 1]), + (np.array([0, 2 ** 63 - 1], dtype=np.uint64), [0, 2 ** 63 - 1]), + (np.array([1.5, -0.0], dtype=np.float16), [1.5, -0.0]), + (np.array([1.5, 2.25], dtype=np.float32), [1.5, 2.25]), + (np.array([1.23456789012345], dtype=np.float64), [1.23456789012345]), + (np.array([True, False]), [True, False]), + (np.array(['"quoted"', "雪\n\\"]), ['"quoted"', "雪\n\\"]), + (np.array([[1, 2], [3, 4]]).T, [[1, 3], [2, 4]]), + (np.arange(6)[::2], [0, 2, 4]), + (np.empty((0, 2)), []), + (np.empty((2, 0)), [[], []]), + ([np.int64(1), np.float32(1.5), np.bool_(True), np.str_("雪")], [1, 1.5, True, "雪"]), + ([np.array(3), (np.array([4, 5]),)], [3, [[4, 5]]]), + (np.array([1, "two", [np.float64(3.5)]], dtype=object), [1, "two", [3.5]]), +]) +def test_numpy_collection_plan(values, expected): + # Capture the JSON request, including conversion of every NumPy element. + with patch("pywy.core.core.requests.post") as post: + WayangContext().register({JavaPlugin}).load_collection(values).store_textfile("file:///tmp/numpy.txt") + post.assert_called_once() + actual = post.call_args.kwargs["json"]["operators"][0]["data"]["values"] + assert json.dumps(actual, allow_nan=False) == json.dumps(expected, allow_nan=False) + + +def test_numpy_snapshot(): + array = np.array([[1, 2], [3, 4]]) + data = WayangContext().load_collection([array, array]) + array[:] = 0 + assert data.operator.values == [[[1, 2], [3, 4]], [[1, 2], [3, 4]]] + + +@pytest.mark.parametrize("values", [np.int64(1), np.array(1)]) +def test_reject_scalar_collection(values): + with pytest.raises(TypeError, match="load_collection expects"): + WayangContext().load_collection(values) + + +@pytest.mark.parametrize("value", [ + np.complex64(1j), np.bytes_(b"bytes"), np.datetime64(1, "ns"), + np.timedelta64(1, "ns"), np.array([np.datetime64(1, "ns")]), + np.array([1j]), np.array([], dtype=complex), + np.array([("a", 1)], dtype=[("name", "U1"), ("value", "i4")]), + np.ma.array([1, 2], mask=[False, True]), +]) +def test_reject_unsupported_numpy_values(value): + with pytest.raises(TypeError, match=r"collection\[0\]: unsupported"): + WayangContext().load_collection([value]) + + +@pytest.mark.parametrize("value", [ + np.uint64(2 ** 63), np.float32("nan"), np.float64("inf"), + np.array([np.uint64(2 ** 63)]), np.array([float("-inf")]), +]) +def test_reject_out_of_range_numpy_values(value): + with pytest.raises(ValueError, match=r"collection\[0\]"): + WayangContext().load_collection([value]) + + +@pytest.mark.parametrize("shape", [(1,), ()]) +def test_reject_numpy_object_cycles(shape): + array = np.empty(shape, dtype=object) + array.flat[0] = array + with pytest.raises(ValueError, match="cyclic"): + WayangContext().load_collection([array]) + + +def test_reject_extended_precision_numpy_values(): + if np.dtype(np.longdouble).itemsize <= 8: + pytest.skip("longdouble has no additional precision on this platform") + for value in (np.longdouble(1), np.array([1], dtype=np.longdouble)): + with pytest.raises(TypeError, match="unsupported"): + WayangContext().load_collection([value]) diff --git a/python/src/pywy/tests/test_load_collection_integration.py b/python/src/pywy/tests/test_load_collection_integration.py index 17f14f51e..bb759d50c 100644 --- a/python/src/pywy/tests/test_load_collection_integration.py +++ b/python/src/pywy/tests/test_load_collection_integration.py @@ -15,10 +15,32 @@ # limitations under the License. # -"""Run against a local backend using PYWAYANG_INTEGRATION_URL (see README).""" +r"""Collection tests using real HTTP requests and backend output files. + +From the repository root, with JDK 17 and the Python requirements installed: + + ./mvnw install -pl :wayang-api-json -am -DskipTests + ./mvnw -pl :wayang-api-json exec:java \ + -Dexec.mainClass=org.apache.wayang.api.json.Main -Dexec.args=8080 \ + -Dwayang.configuration=file:///absolute/path/to/wayang.properties + +Set these properties to absolute paths in wayang.properties: + wayang.api.python.path = + wayang.api.python.worker = /python/src/pywy/execution/worker.py + wayang.api.python.env.path = /python/src + +In another terminal, run against that backend on the same machine: + + PYTHONPATH=python/src \ + PYWAYANG_INTEGRATION_URL=http://localhost:8080/wayang-api-json/submit-plan/json \ + pytest -q python/src/pywy/tests/test_load_collection_integration.py + +These tests skip when PYWAYANG_INTEGRATION_URL is unset. +""" import os +import numpy as np import pytest import requests @@ -55,6 +77,13 @@ def checked_post(*args, **kwargs): ([], ""), ([True, False], "true\nfalse\n"), ([[1, 2], [], [3, ["nested", False]]], "[1, 2]\n[]\n[3, [nested, false]]\n"), + (np.array([1, -2, 2 ** 63 - 1], dtype=np.int64), "1\n-2\n9223372036854775807\n"), + (np.array([1.5, -0.0], dtype=np.float32), "1.5\n-0.0\n"), + (np.array([True, False]), "true\nfalse\n"), + (np.array(["雪", '"quoted"']), '雪\n"quoted"\n'), + (np.array([[1, 2], [3, 4]]), "[1, 2]\n[3, 4]\n"), + (np.empty((0, 2)), ""), + ([np.int32(1), np.float64(2.5), np.bool_(True), np.str_("雪")], "1\n2.5\ntrue\n雪\n"), ]) def test_collection_to_textfile(collection_context, tmp_path, values, expected): output = tmp_path / "collection.txt"