diff --git a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ObjectFileSerialization.java b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ObjectFileSerialization.java index 46d2c152e..5327f18c5 100644 --- a/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ObjectFileSerialization.java +++ b/wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ObjectFileSerialization.java @@ -25,20 +25,57 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InvalidClassException; +import java.io.ObjectInputFilter; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Collection; import java.util.Collections; +import java.util.Currency; +import java.util.Date; +import java.util.Iterator; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; + +import org.apache.wayang.core.api.Configuration; /** * Utility methods that convert between Java objects and the on-disk representation used by {@link ObjectFileSink}. */ public final class ObjectFileSerialization { + public static final String ALLOWLIST_PROPERTY = "wayang.basic.serialization.allowlist"; + public static final String LEGACY_ALLOWLIST_PROPERTY = "wayang.serialization.allowlist"; + public static final String FLINK_ALLOWLIST_PROPERTY = "wayang.flink.serialization.allowlist"; + + public static final String EXPECTED_TYPE_PROPERTY = "wayang.basic.serialization.expected-type"; + public static final String LEGACY_EXPECTED_TYPE_PROPERTY = "wayang.serialization.expected-type"; + public static final String FLINK_EXPECTED_TYPE_PROPERTY = "wayang.flink.value.expected-type"; + + public static final String MAX_DEPTH_PROPERTY = "wayang.basic.serialization.max-depth"; + public static final String LEGACY_MAX_DEPTH_PROPERTY = "wayang.serialization.max-depth"; + public static final String MAX_ARRAY_LENGTH_PROPERTY = "wayang.basic.serialization.max-array-length"; + public static final String LEGACY_MAX_ARRAY_LENGTH_PROPERTY = "wayang.serialization.max-array-length"; + public static final String MAX_REFERENCES_PROPERTY = "wayang.basic.serialization.max-references"; + public static final String LEGACY_MAX_REFERENCES_PROPERTY = "wayang.serialization.max-references"; + public static final String MAX_BYTES_PROPERTY = "wayang.basic.serialization.max-bytes"; + public static final String LEGACY_MAX_BYTES_PROPERTY = "wayang.serialization.max-bytes"; + + static final long DEFAULT_MAX_DEPTH = 100L; + static final long DEFAULT_MAX_ARRAY_LENGTH = 1_048_576L; + static final long DEFAULT_MAX_REFERENCES = 100_000L; + static final long DEFAULT_MAX_BYTES = 67_108_864L; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); @@ -88,7 +125,7 @@ public static List deserializeChunk(byte[] payload, case JSON: return deserializeJson(payload, elementType); case LEGACY_JAVA_SERIALIZATION: - return deserializeLegacy(payload); + return deserializeLegacy(payload, elementType); default: throw new IllegalArgumentException("Unknown serialization mode: " + mode); } @@ -104,9 +141,14 @@ private static byte[] serializeLegacy(Object[] chunk, int validLength) throws IO } } - private static List deserializeLegacy(byte[] payload) throws IOException, ClassNotFoundException { + private static List deserializeLegacy(byte[] payload, Class elementType) throws IOException, ClassNotFoundException { + long maxBytes = getMaxBytes(); + if (payload != null && payload.length > maxBytes) { + throw new InvalidClassException("Serialized payload length " + payload.length + " exceeds max bytes limit " + maxBytes); + } try (ByteArrayInputStream bis = new ByteArrayInputStream(payload); ObjectInputStream ois = new ObjectInputStream(bis)) { + ois.setObjectInputFilter(createObjectInputFilter(elementType)); Object tmp = ois.readObject(); if (tmp == null) { return Collections.emptyList(); @@ -121,6 +163,304 @@ private static List deserializeLegacy(byte[] payload) throws IOException } } + /** + * Creates an {@link ObjectInputFilter} enforcing resource limits and class allowlists for legacy Java deserialization. + * + * @param elementType the expected element type (optional, may be {@code null}) + * @return an {@link ObjectInputFilter} that permits only expected, safe classes within configured resource bounds + */ + public static ObjectInputFilter createObjectInputFilter(Class elementType) { + final long maxDepth = getMaxDepth(); + final long maxArrayLength = getMaxArrayLength(); + final long maxReferences = getMaxReferences(); + final long maxBytes = getMaxBytes(); + + ObjectInputFilter filter = filterInfo -> { + // Check stream/graph resource bounds before serialClass() since limit callbacks may pass serialClass == null. + if (filterInfo.depth() > maxDepth || + filterInfo.arrayLength() > maxArrayLength || + filterInfo.references() > maxReferences || + filterInfo.streamBytes() > maxBytes) { + return ObjectInputFilter.Status.REJECTED; + } + + Class clazz = filterInfo.serialClass(); + if (clazz == null) { + return ObjectInputFilter.Status.UNDECIDED; + } + if (isAllowedClass(clazz, elementType)) { + return ObjectInputFilter.Status.ALLOWED; + } + return ObjectInputFilter.Status.REJECTED; + }; + + ObjectInputFilter globalFilter = ObjectInputFilter.Config.getSerialFilter(); + if (globalFilter != null) { + return ObjectInputFilter.merge(globalFilter, filter); + } + return filter; + } + + private static boolean isAllowedClass(Class clazz, Class elementType) { + Class current = clazz; + while (current.isArray()) { + current = current.getComponentType(); + } + if (current.isPrimitive()) { + return true; + } + + // Standard base object and scalar types + if (current == Object.class || + current == String.class || + current == Boolean.class || + current == Character.class || + current == Class.class || + current == Void.class || + current == java.lang.Record.class || + isStandardNumber(current) || + Enum.class.isAssignableFrom(current) || + isStandardCharSequence(current)) { + return true; + } + + // Standard math + if (current.getName().startsWith("java.math.")) { + return true; + } + + // Standard date and time types + if (current.getName().startsWith("java.time.") || + (Date.class.isAssignableFrom(current) && + (current.getName().startsWith("java.util.") || current.getName().startsWith("java.sql.")))) { + return true; + } + + // Standard collections, maps, and common utilities + if (((Collection.class.isAssignableFrom(current) || + Map.class.isAssignableFrom(current) || + Map.Entry.class.isAssignableFrom(current)) && + current.getName().startsWith("java.util.")) || + current.getName().equals("java.util.CollSer") || + current.getName().startsWith("java.util.Collections$") || + current.getName().startsWith("java.util.Arrays$") || + current.getName().startsWith("java.util.ImmutableCollections$") || + current == UUID.class || + current == BitSet.class || + current == Locale.class || + current == Currency.class || + current == Optional.class) { + return true; + } + + // Wayang internal data quantum types + if (current.getName().equals("org.apache.wayang.basic.data.Record") || + current.getName().equals("org.apache.wayang.basic.data.Tuple2") || + current.getName().equals("org.apache.wayang.basic.data.Tuple5") || + current.getName().equals("org.apache.wayang.core.util.Tuple") || + current.getName().equals("org.apache.wayang.basic.model.op.Op") || + current.getName().equals("org.apache.wayang.flink.compiler.criterion.WayangValue") || + current.getName().equals("org.apache.wayang.flink.compiler.criterion.WayangListValue")) { + return true; + } + + // Scala tuples and basic collections (for Spark / Scala platform compatibility) + if (current.getName().startsWith("scala.Tuple") || + current.getName().startsWith("scala.collection.")) { + return true; + } + + // Expected element type if specified + if (elementType != null && !isBroadType(elementType)) { + Class targetComponent = elementType; + while (targetComponent.isArray()) { + targetComponent = targetComponent.getComponentType(); + } + if (!isBroadType(targetComponent) && + !Number.class.isAssignableFrom(targetComponent) && + !Collection.class.isAssignableFrom(targetComponent) && + !Map.class.isAssignableFrom(targetComponent)) { + if (current.equals(targetComponent) || + (current.getName().startsWith(targetComponent.getName() + "$") && + !Number.class.isAssignableFrom(current) && + !Collection.class.isAssignableFrom(current) && + !Map.class.isAssignableFrom(current))) { + return true; + } + } + } + + // Trusted configuration allowlist (comma/semicolon-separated patterns via system property or Wayang Configuration) + String basicAllowlist = resolveConfigProperty(ALLOWLIST_PROPERTY, LEGACY_ALLOWLIST_PROPERTY); + if (matchesAllowlistPatterns(basicAllowlist, current.getName())) { + return true; + } + String flinkAllowlist = resolveConfigProperty(FLINK_ALLOWLIST_PROPERTY); + return matchesAllowlistPatterns(flinkAllowlist, current.getName()); + } + + /** + * Resolves the first non-blank value among {@code keys}, checking JVM system properties before Wayang's + * default {@link Configuration}. + * + * @param keys ordered property keys from highest to lowest precedence + * @return trimmed property value, or {@code null} if unset or blank + */ + public static String resolveConfigProperty(String... keys) { + if (keys == null) { + return null; + } + for (String key : keys) { + if (key == null) { + continue; + } + String sysValue = System.getProperty(key); + if (sysValue != null && !sysValue.trim().isEmpty()) { + return sysValue.trim(); + } + } + Configuration defaultConfig = Configuration.getDefaultConfiguration(); + if (defaultConfig != null) { + for (String key : keys) { + if (key == null) { + continue; + } + Optional configValue = defaultConfig.getOptionalStringProperty(key); + if (configValue.isPresent() && !configValue.get().trim().isEmpty()) { + return configValue.get().trim(); + } + } + } + return null; + } + + /** + * Returns the configured maximum object graph nesting depth for legacy Java deserialization. + * + * @return positive depth limit + */ + public static long getMaxDepth() { + return getPositiveLongProperty(DEFAULT_MAX_DEPTH, MAX_DEPTH_PROPERTY, LEGACY_MAX_DEPTH_PROPERTY); + } + + /** + * Returns the configured maximum array length for legacy Java deserialization. + * + * @return positive array-length limit + */ + public static long getMaxArrayLength() { + return getPositiveLongProperty(DEFAULT_MAX_ARRAY_LENGTH, MAX_ARRAY_LENGTH_PROPERTY, LEGACY_MAX_ARRAY_LENGTH_PROPERTY); + } + + /** + * Returns the configured maximum object reference count for legacy Java deserialization. + * + * @return positive reference-count limit + */ + public static long getMaxReferences() { + return getPositiveLongProperty(DEFAULT_MAX_REFERENCES, MAX_REFERENCES_PROPERTY, LEGACY_MAX_REFERENCES_PROPERTY); + } + + /** + * Returns the configured maximum stream byte count for legacy Java deserialization. + * + * @return positive stream-bytes limit + */ + public static long getMaxBytes() { + return getPositiveLongProperty(DEFAULT_MAX_BYTES, MAX_BYTES_PROPERTY, LEGACY_MAX_BYTES_PROPERTY); + } + + private static long getPositiveLongProperty(long defaultValue, String... keys) { + String raw = resolveConfigProperty(keys); + if (raw == null) { + return defaultValue; + } + try { + long parsed = Long.parseLong(raw); + if (parsed <= 0) { + throw new IllegalArgumentException("Serialization limit property must be positive: " + raw); + } + return parsed; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid numeric serialization limit property: " + raw, e); + } + } + + private static boolean matchesAllowlistPatterns(String configuredAllowlist, String className) { + if (configuredAllowlist == null || configuredAllowlist.trim().isEmpty()) { + return false; + } + String[] patterns = configuredAllowlist.split("[,;]"); + for (String pattern : patterns) { + if (matchesPattern(pattern.trim(), className)) { + return true; + } + } + return false; + } + + private static boolean isStandardNumber(Class clazz) { + return clazz == Number.class || + clazz == Byte.class || + clazz == Short.class || + clazz == Integer.class || + clazz == Long.class || + clazz == Float.class || + clazz == Double.class; + } + + private static boolean isStandardCharSequence(Class clazz) { + return clazz == String.class || + clazz == StringBuilder.class || + clazz == StringBuffer.class || + clazz == CharSequence.class; + } + + private static boolean isBroadType(Class clazz) { + return clazz == Object.class || + clazz == Serializable.class || + clazz == Comparable.class || + clazz == Cloneable.class || + clazz == AutoCloseable.class || + clazz == Number.class || + clazz == CharSequence.class || + clazz == Collection.class || + clazz == List.class || + clazz == Set.class || + clazz == Queue.class || + clazz == Map.class || + clazz == Map.Entry.class || + clazz == Iterable.class || + clazz == Iterator.class || + clazz == Throwable.class || + clazz == Exception.class || + clazz == Error.class || + clazz == Enum.class || + clazz == java.lang.Record.class; + } + + private static boolean matchesPattern(String pattern, String className) { + if (pattern.isEmpty()) { + return false; + } + if (pattern.equals("*") || pattern.equals("**")) { + return true; + } + if (pattern.endsWith(".**")) { + String prefix = pattern.substring(0, pattern.length() - 3); + return className.equals(prefix) || className.startsWith(prefix + "."); + } + if (pattern.endsWith(".*")) { + String prefix = pattern.substring(0, pattern.length() - 2); + if (className.startsWith(prefix + ".")) { + String remainder = className.substring(prefix.length() + 1); + return !remainder.contains("."); + } + return false; + } + return pattern.equals(className); + } + private static byte[] serializeJson(Object[] chunk, int validLength) throws IOException { Object[] payload = Arrays.copyOf(chunk, validLength); return OBJECT_MAPPER.writeValueAsBytes(payload); diff --git a/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ObjectFileSerializationTest.java b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ObjectFileSerializationTest.java index 0cc3a830d..fa9cb1619 100644 --- a/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ObjectFileSerializationTest.java +++ b/wayang-commons/wayang-basic/src/test/java/org/apache/wayang/basic/operators/ObjectFileSerializationTest.java @@ -18,11 +18,18 @@ package org.apache.wayang.basic.operators; +import org.apache.wayang.core.api.Configuration; import org.junit.Assert; import org.junit.Test; +import testpayload.LegitimatePayload; +import testpayload.SubclassPayloadWithHook; +import testpayload.UnrelatedPayloadWithHook; import java.io.IOException; +import java.io.InvalidClassException; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class ObjectFileSerializationTest { @@ -67,7 +74,463 @@ public void legacyRoundtrip() throws Exception { Assert.assertEquals(42, deserialized.value); } + @Test(expected = java.io.InvalidClassException.class) + public void legacyDeserializationRejectsDisallowedClass() throws Exception { + java.util.concurrent.atomic.AtomicBoolean disallowed = new java.util.concurrent.atomic.AtomicBoolean(true); + Object[] chunk = new Object[]{disallowed}; + + byte[] bytes = ObjectFileSerialization.serializeChunk( + chunk, + chunk.length, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + // Deserializing with an unrelated elementType (or null) should be rejected by the filter + ObjectFileSerialization.deserializeChunk( + bytes, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, + String.class); + } + + @Test + public void legacyDeserializationAllowedViaElementType() throws Exception { + java.util.concurrent.atomic.AtomicBoolean payload = new java.util.concurrent.atomic.AtomicBoolean(true); + Object[] chunk = new Object[]{payload}; + + byte[] bytes = ObjectFileSerialization.serializeChunk( + chunk, + chunk.length, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + // When elementType explicitly specifies AtomicBoolean, it should be permitted + List result = ObjectFileSerialization.deserializeChunk( + bytes, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, + java.util.concurrent.atomic.AtomicBoolean.class); + + Assert.assertEquals(1, result.size()); + Assert.assertTrue(((java.util.concurrent.atomic.AtomicBoolean) result.get(0)).get()); + } + + @Test + public void legacyDeserializationAllowedViaSystemProperty() throws Exception { + java.util.concurrent.atomic.AtomicBoolean payload = new java.util.concurrent.atomic.AtomicBoolean(true); + Object[] chunk = new Object[]{payload}; + + byte[] bytes = ObjectFileSerialization.serializeChunk( + chunk, + chunk.length, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + String propKey = "wayang.basic.serialization.allowlist"; + String originalVal = System.getProperty(propKey); + try { + System.setProperty(propKey, "java.util.concurrent.atomic.**"); + List result = ObjectFileSerialization.deserializeChunk( + bytes, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, + String.class); + Assert.assertEquals(1, result.size()); + Assert.assertTrue(((java.util.concurrent.atomic.AtomicBoolean) result.get(0)).get()); + } finally { + if (originalVal != null) { + System.setProperty(propKey, originalVal); + } else { + System.clearProperty(propKey); + } + } + } + + @Test + public void legacyDeserializationRejectsCustomNumberSubclass() throws Exception { + CustomNumberPayload.hookExecuted = false; + CustomNumberPayload payload = new CustomNumberPayload(123); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + // Subclass rejection when expected type is Number.class + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, Number.class); + Assert.fail("Expected InvalidClassException when elementType is Number.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for Number.class", CustomNumberPayload.hookExecuted); + + // Also rejected when expected type is broad types Object.class, Serializable.class, Comparable.class + for (Class broad : new Class[]{Object.class, Serializable.class, Comparable.class}) { + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, broad); + Assert.fail("Expected InvalidClassException when elementType is " + broad.getSimpleName()); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for " + broad.getSimpleName(), CustomNumberPayload.hookExecuted); + } + + // Also rejected if expected type directly specifies CustomNumberPayload.class + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, CustomNumberPayload.class); + Assert.fail("Expected InvalidClassException when elementType is CustomNumberPayload.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for CustomNumberPayload.class", CustomNumberPayload.hookExecuted); + } + + @Test + public void legacyDeserializationRejectsCustomCollectionSubclass() throws Exception { + CustomCollectionPayload.hookExecuted = false; + CustomCollectionPayload payload = new CustomCollectionPayload<>(); + payload.add("item"); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, java.util.Collection.class); + Assert.fail("Expected InvalidClassException when elementType is Collection.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for Collection.class", CustomCollectionPayload.hookExecuted); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, java.util.List.class); + Assert.fail("Expected InvalidClassException when elementType is List.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for List.class", CustomCollectionPayload.hookExecuted); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, CustomCollectionPayload.class); + Assert.fail("Expected InvalidClassException when elementType is CustomCollectionPayload.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for CustomCollectionPayload.class", CustomCollectionPayload.hookExecuted); + } + + @Test + public void legacyDeserializationRejectsCustomMapSubclass() throws Exception { + CustomMapPayload.hookExecuted = false; + CustomMapPayload payload = new CustomMapPayload<>(); + payload.put("k", "v"); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, java.util.Map.class); + Assert.fail("Expected InvalidClassException when elementType is Map.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for Map.class", CustomMapPayload.hookExecuted); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, CustomMapPayload.class); + Assert.fail("Expected InvalidClassException when elementType is CustomMapPayload.class"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook should not execute for CustomMapPayload.class", CustomMapPayload.hookExecuted); + } + + @Test + public void legacyDeserializationAllowsLegitimatePayload() throws Exception { + LegitimatePayload payload = new LegitimatePayload("safe-data"); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + List result = ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, LegitimatePayload.class); + Assert.assertEquals(1, result.size()); + Assert.assertEquals("safe-data", ((LegitimatePayload) result.get(0)).value); + } + + @Test + public void legacyDeserializationRejectsUnrelatedClassInSamePackage() throws Exception { + UnrelatedPayloadWithHook.hookExecuted = false; + UnrelatedPayloadWithHook payload = new UnrelatedPayloadWithHook(); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, LegitimatePayload.class); + Assert.fail("Expected InvalidClassException when deserializing unrelated class in same package"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Unrelated class in same package must not execute readObject() hook", + UnrelatedPayloadWithHook.hookExecuted); + } + + @Test + public void legacyDeserializationRejectsSubclassOfExpectedPayload() throws Exception { + SubclassPayloadWithHook.hookExecuted = false; + SubclassPayloadWithHook payload = new SubclassPayloadWithHook("subclass-payload"); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, LegitimatePayload.class); + Assert.fail("Expected InvalidClassException when deserializing arbitrary subclass of expected type"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Subclass readObject() hook must not execute", + SubclassPayloadWithHook.hookExecuted); + } + + @Test + public void legacyDeserializationAllowsImmutableCollections() throws Exception { + List list = List.of("alpha", "beta"); + java.util.Set set = java.util.Set.of("gamma", "delta"); + java.util.Map map = java.util.Map.of("key1", 10, "key2", 20); + + Object[] chunk = new Object[]{list, set, map}; + byte[] bytes = ObjectFileSerialization.serializeChunk( + chunk, chunk.length, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + List result = ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, Object.class); + + Assert.assertEquals(3, result.size()); + Assert.assertEquals(list, result.get(0)); + Assert.assertEquals(set, result.get(1)); + Assert.assertEquals(map, result.get(2)); + } + + @Test + public void legacyDeserializationRejectsUnrelatedWayangClassWithHook() throws Exception { + CustomWayangPayloadWithHook.hookExecuted = false; + CustomWayangPayloadWithHook payload = new CustomWayangPayloadWithHook(); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, String.class); + Assert.fail("Expected InvalidClassException for arbitrary class in org.apache.wayang"); + } catch (java.io.InvalidClassException e) { + // Expected + } + Assert.assertFalse("Arbitrary class in org.apache.wayang must not execute readObject() hook", + CustomWayangPayloadWithHook.hookExecuted); + } + + @Test + public void legacyDeserializationAllowsJava17Record() throws Exception { + SampleJavaRecord record = new SampleJavaRecord("record-data", 100); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{record}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + List result = ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, SampleJavaRecord.class); + Assert.assertEquals(1, result.size()); + SampleJavaRecord deserialized = (SampleJavaRecord) result.get(0); + Assert.assertEquals("record-data", deserialized.name()); + Assert.assertEquals(100, deserialized.count()); + } + + @Test + public void legacyDeserializationAllowsObjectAndListDeclaredTypes() throws Exception { + byte[] objectBytes = ObjectFileSerialization.serializeChunk( + new Object[]{"hello", 42}, 2, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + List objects = ObjectFileSerialization.deserializeChunk( + objectBytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, Object.class); + Assert.assertEquals(Arrays.asList("hello", 42), objects); + + List arraysList = Arrays.asList("a", "b"); + List immutableList = List.of("x", "y"); + ArrayList arrayList = new ArrayList<>(Arrays.asList("z")); + byte[] listBytes = ObjectFileSerialization.serializeChunk( + new Object[]{arraysList, immutableList, arrayList}, + 3, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + List lists = ObjectFileSerialization.deserializeChunk( + listBytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, List.class); + Assert.assertEquals(3, lists.size()); + Assert.assertEquals(arraysList, lists.get(0)); + Assert.assertEquals(immutableList, lists.get(1)); + Assert.assertEquals(arrayList, lists.get(2)); + + // Broad Object/List expected types must still reject non-allowlisted custom subclasses with hooks + CustomCollectionPayload.hookExecuted = false; + CustomCollectionPayload badList = new CustomCollectionPayload<>(); + badList.add("evil"); + byte[] badListBytes = ObjectFileSerialization.serializeChunk( + new Object[]{badList}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + try { + ObjectFileSerialization.deserializeChunk( + badListBytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, List.class); + Assert.fail("Expected InvalidClassException when deserializing custom List subclass under List.class"); + } catch (InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom List hook must not execute", CustomCollectionPayload.hookExecuted); + } + + @Test + public void legacyDeserializationEnforcesDepthLimit() throws Exception { + Object shallow = createNestedLists(8); + byte[] shallowBytes = ObjectFileSerialization.serializeChunk( + new Object[]{shallow}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + // Allowed under default limit (100) + List ok = ObjectFileSerialization.deserializeChunk( + shallowBytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, List.class); + Assert.assertEquals(1, ok.size()); + + // Rejected under configured small limit + System.setProperty(ObjectFileSerialization.MAX_DEPTH_PROPERTY, "4"); + try { + expectLegacyInvalidClass(shallowBytes, List.class); + } finally { + System.clearProperty(ObjectFileSerialization.MAX_DEPTH_PROPERTY); + } + + // Rejected under default depth limit (100) without custom property + Object tooDeep = createNestedLists(105); + byte[] tooDeepBytes = ObjectFileSerialization.serializeChunk( + new Object[]{tooDeep}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + expectLegacyInvalidClass(tooDeepBytes, List.class); + } + + @Test + public void legacyDeserializationEnforcesArrayLengthLimit() throws Exception { + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{new int[]{1, 2, 3, 4, 5}}, + 1, + ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + System.setProperty(ObjectFileSerialization.MAX_ARRAY_LENGTH_PROPERTY, "3"); + try { + expectLegacyInvalidClass(bytes, int[].class); + } finally { + System.clearProperty(ObjectFileSerialization.MAX_ARRAY_LENGTH_PROPERTY); + } + + // Crafted byte[] stream declaring Integer.MAX_VALUE length must be rejected under default limit before allocation + byte[] forgedStream = forgeOversizedByteArrayStream(Integer.MAX_VALUE); + try (java.io.ByteArrayInputStream bis = new java.io.ByteArrayInputStream(forgedStream); + java.io.ObjectInputStream ois = new java.io.ObjectInputStream(bis)) { + ois.setObjectInputFilter(ObjectFileSerialization.createObjectInputFilter(Object.class)); + ois.readObject(); + Assert.fail("Expected InvalidClassException for oversized array header"); + } catch (InvalidClassException e) { + // Expected + } + } + + @Test + public void legacyDeserializationEnforcesReferencesAndBytesLimits() throws Exception { + ArrayList manyRefs = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + manyRefs.add(new StringBuilder("item-" + i)); + } + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{manyRefs}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + System.setProperty(ObjectFileSerialization.MAX_REFERENCES_PROPERTY, "10"); + try { + expectLegacyInvalidClass(bytes, List.class); + } finally { + System.clearProperty(ObjectFileSerialization.MAX_REFERENCES_PROPERTY); + } + + System.setProperty(ObjectFileSerialization.MAX_BYTES_PROPERTY, "32"); + try { + expectLegacyInvalidClass(bytes, List.class); + } finally { + System.clearProperty(ObjectFileSerialization.MAX_BYTES_PROPERTY); + } + } + + @Test + public void legacyDeserializationUsesWayangConfigurationAllowlist() throws Exception { + SerializablePayload payload = new SerializablePayload("cfg-allowed", 77); + byte[] bytes = ObjectFileSerialization.serializeChunk( + new Object[]{payload}, 1, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION); + + expectLegacyInvalidClass(bytes, Object.class); + + Configuration cfg = Configuration.getDefaultConfiguration(); + String prev = cfg.getOptionalStringProperty(ObjectFileSerialization.ALLOWLIST_PROPERTY).orElse(""); + cfg.setProperty(ObjectFileSerialization.ALLOWLIST_PROPERTY, SerializablePayload.class.getName()); + try { + List result = ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, Object.class); + Assert.assertEquals(1, result.size()); + SerializablePayload deserialized = (SerializablePayload) result.get(0); + Assert.assertEquals("cfg-allowed", deserialized.text); + Assert.assertEquals(77, deserialized.value); + } finally { + cfg.setProperty(ObjectFileSerialization.ALLOWLIST_PROPERTY, prev); + } + } + + private static void expectLegacyInvalidClass(byte[] bytes, Class elementType) throws Exception { + try { + ObjectFileSerialization.deserializeChunk( + bytes, ObjectFileSerializationMode.LEGACY_JAVA_SERIALIZATION, elementType); + Assert.fail("Expected InvalidClassException"); + } catch (InvalidClassException e) { + // Expected + } + } + + private static ArrayList createNestedLists(int levels) { + ArrayList current = new ArrayList<>(); + current.add("leaf"); + for (int i = 0; i < levels; i++) { + ArrayList parent = new ArrayList<>(); + parent.add(current); + current = parent; + } + return current; + } + + private static byte[] forgeOversizedByteArrayStream(int declaredLength) throws IOException { + java.io.ByteArrayOutputStream bos = new java.io.ByteArrayOutputStream(); + try (java.io.ObjectOutputStream oos = new java.io.ObjectOutputStream(bos)) { + oos.writeObject(new byte[]{1, 2, 3, 4, 5}); + oos.flush(); + } + byte[] serialized = bos.toByteArray(); + int lenOffset = serialized.length - 5 - 4; + serialized[lenOffset] = (byte) (declaredLength >>> 24); + serialized[lenOffset + 1] = (byte) (declaredLength >>> 16); + serialized[lenOffset + 2] = (byte) (declaredLength >>> 8); + serialized[lenOffset + 3] = (byte) declaredLength; + return serialized; + } + + public record SampleJavaRecord(String name, int count) implements Serializable { + private static final long serialVersionUID = 1L; + } + + public static class CustomWayangPayloadWithHook implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + private static class SerializablePayload implements Serializable { + private static final long serialVersionUID = 1L; final String text; final int value; @@ -76,4 +539,48 @@ private SerializablePayload(String text, int value) { this.value = value; } } + + public static class CustomNumberPayload extends Number implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + private final int val; + + public CustomNumberPayload(int val) { + this.val = val; + } + + @Override + public int intValue() { return val; } + @Override + public long longValue() { return val; } + @Override + public float floatValue() { return val; } + @Override + public double doubleValue() { return val; } + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + + public static class CustomCollectionPayload extends java.util.ArrayList implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + + public static class CustomMapPayload extends java.util.HashMap implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } } diff --git a/wayang-commons/wayang-basic/src/test/java/testpayload/LegitimatePayload.java b/wayang-commons/wayang-basic/src/test/java/testpayload/LegitimatePayload.java new file mode 100644 index 000000000..b7448e5e2 --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/testpayload/LegitimatePayload.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testpayload; + +import java.io.Serializable; + +public class LegitimatePayload implements Serializable { + private static final long serialVersionUID = 1L; + public final String value; + + public LegitimatePayload(String value) { + this.value = value; + } +} + diff --git a/wayang-commons/wayang-basic/src/test/java/testpayload/SubclassPayloadWithHook.java b/wayang-commons/wayang-basic/src/test/java/testpayload/SubclassPayloadWithHook.java new file mode 100644 index 000000000..84e9943a7 --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/testpayload/SubclassPayloadWithHook.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testpayload; + +import java.io.IOException; +import java.io.ObjectInputStream; + +public class SubclassPayloadWithHook extends LegitimatePayload { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + public SubclassPayloadWithHook(String value) { + super(value); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } +} + diff --git a/wayang-commons/wayang-basic/src/test/java/testpayload/UnrelatedPayloadWithHook.java b/wayang-commons/wayang-basic/src/test/java/testpayload/UnrelatedPayloadWithHook.java new file mode 100644 index 000000000..dd02a08fd --- /dev/null +++ b/wayang-commons/wayang-basic/src/test/java/testpayload/UnrelatedPayloadWithHook.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testpayload; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; + +public class UnrelatedPayloadWithHook implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } +} + diff --git a/wayang-platforms/wayang-flink/src/main/java/org/apache/wayang/flink/compiler/criterion/WayangValue.java b/wayang-platforms/wayang-flink/src/main/java/org/apache/wayang/flink/compiler/criterion/WayangValue.java index 2fc0d6b5f..f4f8d30bf 100644 --- a/wayang-platforms/wayang-flink/src/main/java/org/apache/wayang/flink/compiler/criterion/WayangValue.java +++ b/wayang-platforms/wayang-flink/src/main/java/org/apache/wayang/flink/compiler/criterion/WayangValue.java @@ -26,14 +26,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.ObjectInput; +import java.io.InvalidClassException; import java.io.ObjectInputStream; -import java.io.ObjectOutput; import java.io.ObjectOutputStream; +import org.apache.wayang.basic.operators.ObjectFileSerialization; +import org.apache.wayang.core.api.exception.WayangException; + /** * Implementation of {@link Value} of flink for use in Wayang */ public class WayangValue implements Value { + private static final long serialVersionUID = 1L; + private static final int HIGH_BIT = 0x1 << 7; private static final int HIGH_BIT2 = 0x1 << 13; @@ -42,17 +46,50 @@ public class WayangValue implements Value { private T data; + private Class type; + public WayangValue(){ this.data = null; + this.type = null; + } + + public WayangValue(Class type){ + this.data = null; + this.type = type; } + @SuppressWarnings("unchecked") public WayangValue(T element){ this.data = element; + if (element != null) { + this.type = (Class) element.getClass(); + } + } + + public WayangValue(T element, Class type){ + this.data = element; + this.type = type; + } + + public Class getType() { + return this.type; + } + + public void setType(Class type) { + this.type = type; } @Override public void write(DataOutputView dataOutputView) throws IOException { - byte[] element = convertToByte(); + byte[] element; + try { + element = convertToByte(); + } catch (WayangException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + throw new IOException("Failed to serialize WayangValue payload", e); + } // write the length, variable-length encoded int len = element.length; while (len >= HIGH_BIT) { @@ -61,13 +98,10 @@ public void write(DataOutputView dataOutputView) throws IOException { } dataOutputView.write(len); dataOutputView.write(element); - - } @Override public void read(DataInputView dataInputView) throws IOException { - int len = dataInputView.readUnsignedByte(); if (len >= HIGH_BIT) { @@ -75,61 +109,105 @@ public void read(DataInputView dataInputView) throws IOException { int curr; len = len & 0x7f; while ((curr = dataInputView.readUnsignedByte()) >= HIGH_BIT) { + if (shift >= 28) { + throw new IOException("Malformed WayangValue length header"); + } len |= (curr & 0x7f) << shift; shift += 7; } len |= curr << shift; } + long maxBytes = ObjectFileSerialization.getMaxBytes(); + if (len < 0 || len > maxBytes) { + throw new InvalidClassException( + "WayangValue payload length " + len + " exceeds allowed bounds [0, " + maxBytes + "]"); + } + byte[] array = new byte[len]; - dataInputView.read(array); + dataInputView.readFully(array); - this.data = convertToObject(array); + try { + this.data = convertToObject(array, this.type); + } catch (WayangException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + throw new IOException("Failed to deserialize WayangValue payload", e); + } } - public byte[] convertToByte(){ - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutput out = null; - byte[] result = null; - try { - out = new ObjectOutputStream(bos); + public byte[] convertToByte() { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bos)) { out.writeObject(this.data); out.flush(); - result = bos.toByteArray(); + return bos.toByteArray(); } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - bos.close(); - } catch (IOException ex) { - ex.printStackTrace(); + throw new WayangException("Failed to serialize WayangValue payload", e); + } + } + + public T convertToObject(byte[] array) { + return convertToObject(array, this.type); + } + + @SuppressWarnings("unchecked") + public T convertToObject(byte[] array, Class expectedType) { + if (array == null || array.length == 0) { + return null; + } + long maxBytes = ObjectFileSerialization.getMaxBytes(); + if (array.length > maxBytes) { + throw new WayangException( + "Failed to deserialize WayangValue payload", + new InvalidClassException( + "Payload length " + array.length + " exceeds max bytes limit " + maxBytes)); + } + Class targetType = resolveExpectedType(expectedType); + try (ByteArrayInputStream bis = new ByteArrayInputStream(array); + ObjectInputStream in = new ObjectInputStream(bis)) { + in.setObjectInputFilter(ObjectFileSerialization.createObjectInputFilter(targetType)); + Object result = in.readObject(); + if (targetType != null && !targetType.isPrimitive() && result != null && !targetType.isInstance(result)) { + throw new InvalidClassException( + result.getClass().getName(), + "Deserialized payload is not assignable to expected type " + targetType.getName()); } + return (T) result; + } catch (IOException | ClassNotFoundException e) { + throw new WayangException("Failed to deserialize WayangValue payload", e); + } + } + + private Class resolveExpectedType(Class explicitExpectedType) { + if (explicitExpectedType != null) { + return explicitExpectedType; + } + if (this.type != null) { + return this.type; } - return result; + return resolveConfiguredExpectedType(); } - public T convertToObject(byte[] array){ - ByteArrayInputStream bis = new ByteArrayInputStream(array); - ObjectInput in = null; - T object = null; + private Class resolveConfiguredExpectedType() { + String configuredType = ObjectFileSerialization.resolveConfigProperty( + ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, + ObjectFileSerialization.EXPECTED_TYPE_PROPERTY, + ObjectFileSerialization.LEGACY_EXPECTED_TYPE_PROPERTY); + if (configuredType == null) { + return null; + } try { - in = new ObjectInputStream(bis); - object = (T) in.readObject(); - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } finally { - try { - if (in != null) { - in.close(); - } - } catch (IOException ex) { - ex.printStackTrace(); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if (cl == null) { + cl = getClass().getClassLoader(); } + return Class.forName(configuredType, false, cl); + } catch (ClassNotFoundException | LinkageError e) { + throw new WayangException("Could not load configured expected payload class: " + configuredType, e); } - return object; } public String toString(){ diff --git a/wayang-platforms/wayang-flink/src/test/java/org/apache/wayang/flink/compiler/criterion/WayangValueTest.java b/wayang-platforms/wayang-flink/src/test/java/org/apache/wayang/flink/compiler/criterion/WayangValueTest.java new file mode 100644 index 000000000..582e7da33 --- /dev/null +++ b/wayang-platforms/wayang-flink/src/test/java/org/apache/wayang/flink/compiler/criterion/WayangValueTest.java @@ -0,0 +1,444 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.flink.compiler.criterion; + +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.wayang.basic.operators.ObjectFileSerialization; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.exception.WayangException; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InvalidClassException; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class WayangValueTest { + + @Test + public void testRoundtripAllowedType() { + WayangValue value = new WayangValue<>("hello"); + byte[] bytes = value.convertToByte(); + Assert.assertNotNull(bytes); + + WayangValue target = new WayangValue<>(); + String deserialized = target.convertToObject(bytes); + Assert.assertEquals("hello", deserialized); + } + + @Test + public void testFreshWayangValueWithCustomPayloadTypeConstructor() { + WayangValue source = new WayangValue<>(new CustomPayload("custom-constructor")); + byte[] bytes = source.convertToByte(); + Assert.assertNotNull(bytes); + + WayangValue fresh = new WayangValue<>(CustomPayload.class); + CustomPayload deserialized = fresh.convertToObject(bytes); + Assert.assertNotNull(deserialized); + Assert.assertEquals("custom-constructor", deserialized.msg); + } + + @Test + public void testFreshWayangValueWithSetType() { + WayangValue source = new WayangValue<>(new CustomPayload("custom-setter")); + byte[] bytes = source.convertToByte(); + Assert.assertNotNull(bytes); + + WayangValue fresh = new WayangValue<>(); + fresh.setType(CustomPayload.class); + CustomPayload deserialized = fresh.convertToObject(bytes); + Assert.assertNotNull(deserialized); + Assert.assertEquals("custom-setter", deserialized.msg); + } + + @Test + public void testFreshWayangValueWithExplicitExpectedTypeInConvert() { + WayangValue source = new WayangValue<>(new CustomPayload("custom-explicit")); + byte[] bytes = source.convertToByte(); + Assert.assertNotNull(bytes); + + WayangValue fresh = new WayangValue<>(); + CustomPayload deserialized = fresh.convertToObject(bytes, CustomPayload.class); + Assert.assertNotNull(deserialized); + Assert.assertEquals("custom-explicit", deserialized.msg); + } + + @Test + public void testFlinkReadWriteRoundtripWithTypedReceiver() throws Exception { + WayangValue source = new WayangValue<>(new CustomPayload("flink-stream"), CustomPayload.class); + byte[] bytes = writeToBytes(source); + + WayangValue fresh = readFromBytes(bytes, new WayangValue<>(CustomPayload.class)); + Assert.assertNotNull(fresh.get()); + Assert.assertEquals("flink-stream", fresh.get().msg); + Assert.assertEquals(CustomPayload.class, fresh.getType()); + } + + @Test + public void testFreshNoArgWayangValueRejectsUntrustedCustomPayloadWithHook() throws Exception { + CustomPayloadWithHook.hookExecuted = false; + WayangValue source = new WayangValue<>(new CustomPayloadWithHook("evil")); + byte[] bytes = writeToBytes(source); + + WayangValue fresh = new WayangValue<>(); + try { + readFromBytes(bytes, fresh); + Assert.fail("Expected InvalidClassException when deserializing untrusted custom class into no-arg WayangValue"); + } catch (InvalidClassException e) { + // Expected + } + Assert.assertFalse("Custom readObject() hook must not execute", CustomPayloadWithHook.hookExecuted); + } + + @Test + public void testWayangListValueRejectsUntrustedAndRoundtripsWithAllowlistOrExpectedType() throws Exception { + WayangListValue list = new WayangListValue(); + list.add(new WayangValue<>(new CustomPayload("item-1"))); + list.add(new WayangValue<>(new CustomPayload("item-2"))); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(bos); + list.write(outView); + byte[] listBytes = bos.toByteArray(); + + // Untrusted custom elements must be rejected when no allowlist / expected type is configured + try { + readListFromBytes(listBytes); + Assert.fail("Expected InvalidClassException when deserializing unconfigured custom payloads in WayangListValue"); + } catch (InvalidClassException e) { + // Expected + } + + // Trusted allowlist enables deserialization in fresh no-arg WayangValue instances created by WayangListValue + System.setProperty(ObjectFileSerialization.FLINK_ALLOWLIST_PROPERTY, CustomPayload.class.getName()); + try { + WayangListValue freshList = readListFromBytes(listBytes); + Assert.assertEquals(2, freshList.size()); + Assert.assertEquals("item-1", ((CustomPayload) freshList.get(0).get()).msg); + Assert.assertEquals("item-2", ((CustomPayload) freshList.get(1).get()).msg); + } finally { + System.clearProperty(ObjectFileSerialization.FLINK_ALLOWLIST_PROPERTY); + } + + // Trusted expected-type in Wayang Configuration also enables WayangListValue roundtrip + Configuration cfg = Configuration.getDefaultConfiguration(); + String prev = cfg.getOptionalStringProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY).orElse(""); + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, CustomPayload.class.getName()); + try { + WayangListValue freshList = readListFromBytes(listBytes); + Assert.assertEquals(2, freshList.size()); + Assert.assertEquals("item-1", ((CustomPayload) freshList.get(0).get()).msg); + Assert.assertEquals("item-2", ((CustomPayload) freshList.get(1).get()).msg); + } finally { + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, prev); + } + } + + @Test + public void testFreshNoArgWayangValueRejectsCustomNumberSubclass() throws Exception { + CustomNumberWithHook.hookExecuted = false; + WayangValue source = new WayangValue<>(new CustomNumberWithHook(42)); + byte[] bytes = writeToBytes(source); + + WayangValue fresh = new WayangValue<>(); + try { + readFromBytes(bytes, fresh); + Assert.fail("Expected IOException / InvalidClassException when deserializing CustomNumberWithHook"); + } catch (IOException e) { + // Expected + } + Assert.assertFalse("Custom Number readObject() hook must not execute", CustomNumberWithHook.hookExecuted); + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testObjectAndListDeclaredPayloadsRoundtripAndRejectMismatchOrHooks() throws Exception { + WayangValue objSource = new WayangValue<>("hello-object", Object.class); + byte[] objBytes = writeToBytes(objSource); + Assert.assertEquals("hello-object", readFromBytes(objBytes, new WayangValue<>(Object.class)).get()); + Assert.assertEquals("hello-object", readFromBytes(objBytes, new WayangValue<>()).get()); + + List arraysList = Arrays.asList("a", "b"); + byte[] arraysListBytes = writeToBytes(new WayangValue<>(arraysList, (Class) List.class)); + Assert.assertEquals(arraysList, readFromBytes(arraysListBytes, new WayangValue<>((Class) List.class)).get()); + Assert.assertEquals(arraysList, readFromBytes(arraysListBytes, new WayangValue<>()).get()); + + List immutableList = List.of("x", "y"); + byte[] immutableListBytes = writeToBytes(new WayangValue<>(immutableList, (Class) List.class)); + Assert.assertEquals(immutableList, readFromBytes(immutableListBytes, new WayangValue<>((Class) List.class)).get()); + + ArrayList arrayList = new ArrayList<>(Arrays.asList("p", "q")); + byte[] arrayListBytes = writeToBytes(new WayangValue<>(arrayList, (Class) List.class)); + Assert.assertEquals(arrayList, readFromBytes(arrayListBytes, new WayangValue<>((Class) List.class)).get()); + + // Configured expected-type List.class or Object.class works cleanly on no-arg WayangValue + Configuration cfg = Configuration.getDefaultConfiguration(); + String prev = cfg.getOptionalStringProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY).orElse(""); + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, List.class.getName()); + try { + Assert.assertEquals(arraysList, readFromBytes(arraysListBytes, new WayangValue<>()).get()); + try { + readFromBytes(objBytes, new WayangValue<>()); + Assert.fail("Expected InvalidClassException when actual String is not assignable to List.class"); + } catch (InvalidClassException e) { + // Expected + } + } finally { + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, prev); + } + + // Broad Object.class / List.class typed receivers must still reject custom hooks + CustomListWithHook.hookExecuted = false; + byte[] badListBytes = writeToBytes(new WayangValue<>(new CustomListWithHook(), (Class) List.class)); + try { + readFromBytes(badListBytes, new WayangValue<>((Class) List.class)); + Assert.fail("Expected InvalidClassException for CustomListWithHook under List.class"); + } catch (InvalidClassException e) { + // Expected + } + Assert.assertFalse("CustomListWithHook readObject() hook must not execute", CustomListWithHook.hookExecuted); + } + + @Test + public void testFreshNoArgWayangValueResolvesTrustedConfigAndRejectsOtherCustomClasses() throws Exception { + byte[] customBytes = writeToBytes(new WayangValue<>(new CustomPayload("trusted-cfg"))); + byte[] hookBytes = writeToBytes(new WayangValue<>(new CustomPayloadWithHook("untrusted-hook"))); + + Configuration cfg = Configuration.getDefaultConfiguration(); + String prev = cfg.getOptionalStringProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY).orElse(""); + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, CustomPayload.class.getName()); + try { + CustomPayloadWithHook.hookExecuted = false; + WayangValue fresh = readFromBytes(customBytes, new WayangValue<>()); + Assert.assertEquals("trusted-cfg", fresh.get().msg); + + try { + readFromBytes(hookBytes, new WayangValue<>()); + Assert.fail("Expected InvalidClassException for unconfigured CustomPayloadWithHook"); + } catch (InvalidClassException e) { + // Expected + } + Assert.assertFalse("Unconfigured hook class must not execute", CustomPayloadWithHook.hookExecuted); + } finally { + cfg.setProperty(ObjectFileSerialization.FLINK_EXPECTED_TYPE_PROPERTY, prev); + } + + System.setProperty(ObjectFileSerialization.ALLOWLIST_PROPERTY, CustomPayload.class.getName()); + try { + CustomPayloadWithHook.hookExecuted = false; + WayangValue fresh = readFromBytes(customBytes, new WayangValue<>()); + Assert.assertEquals("trusted-cfg", fresh.get().msg); + + try { + readFromBytes(hookBytes, new WayangValue<>()); + Assert.fail("Expected InvalidClassException for non-allowlisted CustomPayloadWithHook"); + } catch (InvalidClassException e) { + // Expected + } + Assert.assertFalse("Non-allowlisted hook class must not execute", CustomPayloadWithHook.hookExecuted); + } finally { + System.clearProperty(ObjectFileSerialization.ALLOWLIST_PROPERTY); + } + } + + @Test + public void testWayangValueEnforcesLengthBoundsAndResourceLimits() throws Exception { + // Oversized varint length header (Integer.MAX_VALUE) must fail before array allocation + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(bos); + int len = Integer.MAX_VALUE; + while (len >= 128) { + outView.write(len | 128); + len >>>= 7; + } + outView.write(len); + outView.flush(); + try { + readFromBytes(bos.toByteArray(), new WayangValue<>()); + Assert.fail("Expected InvalidClassException for oversized WayangValue length header"); + } catch (InvalidClassException e) { + // Expected + } + + // Depth limit + ArrayList deepList = new ArrayList<>(); + ArrayList cursor = deepList; + for (int i = 0; i < 6; i++) { + ArrayList next = new ArrayList<>(); + cursor.add(next); + cursor = next; + } + byte[] deepBytes = writeToBytes(new WayangValue<>(deepList, ArrayList.class)); + System.setProperty(ObjectFileSerialization.MAX_DEPTH_PROPERTY, "4"); + try { + readFromBytes(deepBytes, new WayangValue<>(ArrayList.class)); + Assert.fail("Expected InvalidClassException when depth limit is exceeded"); + } catch (InvalidClassException e) { + // Expected + } finally { + System.clearProperty(ObjectFileSerialization.MAX_DEPTH_PROPERTY); + } + + // Array-length limit + byte[] arrayBytes = writeToBytes(new WayangValue<>(new int[]{1, 2, 3, 4, 5}, int[].class)); + System.setProperty(ObjectFileSerialization.MAX_ARRAY_LENGTH_PROPERTY, "3"); + try { + readFromBytes(arrayBytes, new WayangValue<>(int[].class)); + Assert.fail("Expected InvalidClassException when array-length limit is exceeded"); + } catch (InvalidClassException e) { + // Expected + } finally { + System.clearProperty(ObjectFileSerialization.MAX_ARRAY_LENGTH_PROPERTY); + } + + // References limit + ArrayList refList = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + refList.add(new StringBuilder("ref-" + i)); + } + byte[] refBytes = writeToBytes(new WayangValue<>(refList, ArrayList.class)); + System.setProperty(ObjectFileSerialization.MAX_REFERENCES_PROPERTY, "6"); + try { + readFromBytes(refBytes, new WayangValue<>(ArrayList.class)); + Assert.fail("Expected InvalidClassException when references limit is exceeded"); + } catch (InvalidClassException e) { + // Expected + } finally { + System.clearProperty(ObjectFileSerialization.MAX_REFERENCES_PROPERTY); + } + } + + @Test + public void testReusedNoArgWayangValueReadsMultipleAllowedTypes() throws Exception { + byte[] intBytes = writeToBytes(new WayangValue<>(Integer.valueOf(7))); + byte[] strBytes = writeToBytes(new WayangValue<>("second-record")); + + WayangValue reused = new WayangValue<>(); + readFromBytes(intBytes, reused); + Assert.assertEquals(7, reused.get()); + readFromBytes(strBytes, reused); + Assert.assertEquals("second-record", reused.get()); + } + + @Test + public void testRejectsDisallowedTypeThrowsException() throws Exception { + AtomicBoolean disallowed = new AtomicBoolean(true); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bos)) { + out.writeObject(disallowed); + out.flush(); + } + + WayangValue target = new WayangValue<>(); + try { + target.convertToObject(bos.toByteArray()); + Assert.fail("Expected WayangException due to disallowed class"); + } catch (WayangException e) { + Assert.assertTrue(e.getCause() instanceof InvalidClassException); + } + } + + public static class CustomPayload implements Serializable { + private static final long serialVersionUID = 1L; + final String msg; + + public CustomPayload(String msg) { + this.msg = msg; + } + } + + public static class CustomNumberWithHook extends Number implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + private final int val; + + public CustomNumberWithHook(int val) { + this.val = val; + } + + @Override + public int intValue() { return val; } + @Override + public long longValue() { return val; } + @Override + public float floatValue() { return val; } + @Override + public double doubleValue() { return val; } + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + + public static class CustomPayloadWithHook implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + final String msg; + + public CustomPayloadWithHook(String msg) { + this.msg = msg; + } + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + + public static class CustomListWithHook extends ArrayList implements Serializable { + private static final long serialVersionUID = 1L; + public static boolean hookExecuted = false; + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + hookExecuted = true; + in.defaultReadObject(); + } + } + + private static byte[] writeToBytes(WayangValue value) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(bos); + value.write(outView); + return bos.toByteArray(); + } + + private static WayangValue readFromBytes(byte[] bytes, WayangValue target) throws IOException { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bis); + target.read(inView); + return target; + } + + private static WayangListValue readListFromBytes(byte[] bytes) throws IOException { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bis); + WayangListValue list = new WayangListValue(); + list.read(inView); + return list; + } +}