docs: publish matched XML and JSON serializer comparisons

.agents/skills/polyxml-benchmark-workflow/SKILL.md

@@ -72,3 +72,42 @@ description: Use when running, adding, or publishing PolyXML language and runtim
     Its allocation and Callgrind consumers are diagnostic only. Confirm wins
     with the uninstrumented Criterion runner, and anchor write-only filters as
     `^serialization/` because `serialization` also matches deserialization.
+
+12. For matched serializer comparisons, use `benchmarks/serializer-comparison/`.
+    Measure uninstrumented builds; the Rust allocation feature is a separate
+    diagnostic binary, and Python timings must not run under `tracemalloc`.
+    Typed stdlib/DOM adapters must include model construction and integer
+    conversion. Check every decoded field, escaped text, declared XML root,
+    and outputs with an independent XML implementation before publishing.
+    Generated Rust JSON and handwritten Serde JSON use the same `serde_json`
+    engine; Go/C# generated and handwritten models use the same native XML
+    serializer. Label these model comparisons, not separate engine wins.
+    Keep dynamic value graphs and Pydantic validation distinct from dataclass
+    binding. Report medians of process medians and their range; raw samples
+    within one process are correlated. Current and historical host/revision
+    results belong in separate tables.
+
+13. C# `XmlSerializer` timings require enough warmup for tiered compilation:
+    at least 500 read/write pairs and two seconds per model, plus alternating
+    model order in fresh processes. Include `StringWriter.ToString()` in
+    timed writes when the promised result is a string. Record the actual
+    runtime (a net8.0 target rolled forward to .NET 10 measures .NET 10).
+    Go full-field assertions belong in preflight, outside measured reads.
+
+    In the October 4 .NET 10 refresh, that longer warmup still left a
+    first-model effect for small operations. Do not call it model overhead:
+    preserve the default runs as diagnostics and use the separate
+    `serializer-comparison/control_csharp.py` full-JIT comparison, which
+    explicitly sets `DOTNET_TieredCompilation=0` in six balanced processes.
+    Label this configuration; it is not a default-tiered-PGO benchmark.
+
+14. Record the actual StAX providers on the benchmark classpath, not just the
+    JDK version. Jackson brings Woodstox into the shaded JMH jar, and generated
+    `XMLInputFactory.newFactory()` codecs select it through SPI as well. The
+    direct codec uses the JDK API but need not use the JDK default parser.
+    `serializer-comparison/Providers.java` records both factory classes.
+
+15. Retained benchmark-source READMEs contain repository-relative links. Store
+    them byte-for-byte as `.md.txt` inside docs data artifacts so the docs site
+    does not render them in the wrong relative-link context. Keep actual
+    result summaries as Markdown and check the final site strictly.

benchmarks/README.md

@@ -34,6 +34,7 @@ equivalent) and never at this root.
 
 | Suite | Location | Tooling | How to run |
 | :--- | :--- | :--- | :--- |
+| Matched serializer comparison | [`serializer-comparison/`](serializer-comparison/README.md) | Fresh-process Rust/Python, native Go/C#, Java JMH | See suite README |
 | Rust core engine | [`crates/polyxml-core/benches/`](../crates/polyxml-core/benches/) | [Criterion.rs](https://github.com/bheisler/criterion.rs) | `cargo bench --bench core_benchmarks` |
 | Rust same-host XML/Serde regression | [`rust-xml-regression/`](rust-xml-regression/README.md) | Generated consumers + Criterion | See suite README |
 | Rust runtime investigation | [`rust-runtime-investigation/`](rust-runtime-investigation/README.md) | Dynamic core constraints, allocations and instruction profiles | See suite README |

benchmarks/csharp/Program.cs

@@ -29,36 +29,45 @@ static void Measure<T>(string name, int count, Func<int,T> make) where T : class
         using (var check = new StringReader(xml))
         {
             var decoded = serializer.Deserialize(check) as T ?? throw new Exception("Round trip failed");
-            int mid = count / 2;
-            if (decoded is Batch<Sensor> generated && (
-                generated.Sensors.Length != count ||
-                generated.Sensors[0].Id != "sensor-0" || generated.Sensors[0].Value != 0 ||
-                generated.Sensors[mid].Id != $"sensor-{mid}" || generated.Sensors[mid].Value != mid ||
-                generated.Sensors[^1].Id != $"sensor-{count-1}" || generated.Sensors[^1].Value != count-1))
-                throw new Exception("Generated batch mismatch");
-            if (decoded is Batch<BaselineSensor> baseline && (
-                baseline.Sensors.Length != count ||
-                baseline.Sensors[0].Id != "sensor-0" || baseline.Sensors[0].Value != 0 ||
-                baseline.Sensors[mid].Id != $"sensor-{mid}" || baseline.Sensors[mid].Value != mid ||
-                baseline.Sensors[^1].Id != $"sensor-{count-1}" || baseline.Sensors[^1].Value != count-1))
-                throw new Exception("Baseline batch mismatch");
+            var fields = decoded switch {
+                Batch<Sensor> generated => generated.Sensors.Select(s => (s.Id, s.Value)),
+                Batch<BaselineSensor> baseline => baseline.Sensors.Select(s => (s.Id, s.Value)),
+                _ => throw new Exception("Unknown model")
+            };
+            int index = 0;
+            foreach (var (id, number) in fields) {
+                if (id != $"sensor-{index}" || number != index) throw new Exception($"Field mismatch at {index}");
+                index++;
+            }
+            if (index != count) throw new Exception("Sensor count mismatch");
             using var verify = new StringWriter();
             serializer.Serialize(verify, decoded);
             if (verify.ToString() != initial.ToString()) throw new Exception("Round trip mismatch");
         }
-        for (var i = 0; i < 100; i++) { using var reader = new StringReader(xml); _ = serializer.Deserialize(reader); using var writer = new StringWriter(); serializer.Serialize(writer, value); }
+        var warmup = Stopwatch.StartNew();
+        int warmed = 0;
+        int warmupMs = int.TryParse(Environment.GetEnvironmentVariable("BENCH_WARMUP_MS"), out var ms) ? ms : 2000;
+        while (warmed < 500 || warmup.ElapsedMilliseconds < warmupMs) {
+            using var reader = new StringReader(xml);
+            GC.KeepAlive(serializer.Deserialize(reader));
+            using var writer = new StringWriter();
+            serializer.Serialize(writer, value);
+            GC.KeepAlive(writer.ToString());
+            warmed++;
+        }
+        warmup.Stop();
         foreach (var operation in new[] { "read", "write" })
         {
             GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect();
             var bytes = GC.GetAllocatedBytesForCurrentThread();
             var timer = Stopwatch.StartNew();
             for (var i = 0; i < Iterations; i++)
             {
-                if (operation == "read") { using var reader = new StringReader(xml); _ = serializer.Deserialize(reader); }
-                else { using var writer = new StringWriter(); serializer.Serialize(writer, value); }
+                if (operation == "read") { using var reader = new StringReader(xml); GC.KeepAlive(serializer.Deserialize(reader)); }
+                else { using var writer = new StringWriter(); serializer.Serialize(writer, value); GC.KeepAlive(writer.ToString()); }
             }
             timer.Stop();
-            Console.WriteLine($"{name},size={count},{operation},ns/op={timer.Elapsed.TotalNanoseconds / Iterations:F0},B/op={(GC.GetAllocatedBytesForCurrentThread()-bytes)/Iterations},xml_bytes={System.Text.Encoding.UTF8.GetByteCount(xml)}");
+            Console.WriteLine($"{name},size={count},{operation},ns/op={timer.Elapsed.TotalNanoseconds / Iterations:F0},B/op={(GC.GetAllocatedBytesForCurrentThread()-bytes)/Iterations},xml_bytes={System.Text.Encoding.UTF8.GetByteCount(xml)},output_chars={initial.ToString().Length},warmup_iterations={warmed},warmup_ms={warmup.ElapsedMilliseconds}");
         }
     }
 
@@ -67,8 +76,10 @@ public static void Main()
         Console.WriteLine($"dotnet={Environment.Version},os={System.Runtime.InteropServices.RuntimeInformation.OSDescription},cpu={Environment.ProcessorCount},iterations={Iterations}");
         foreach (var count in new[] { 1, 1000 })
         {
-            Measure("generated", count, n => new Batch<Sensor> { Sensors = Enumerable.Range(0, n).Select(i => new Sensor { Id = $"sensor-{i}", Value = i }).ToArray() });
-            Measure("baseline", count, n => new Batch<BaselineSensor> { Sensors = Enumerable.Range(0, n).Select(i => new BaselineSensor { Id = $"sensor-{i}", Value = i }).ToArray() });
+            Action generated = () => Measure("generated", count, n => new Batch<Sensor> { Sensors = Enumerable.Range(0, n).Select(i => new Sensor { Id = $"sensor-{i}", Value = i }).ToArray() });
+            Action baseline = () => Measure("baseline", count, n => new Batch<BaselineSensor> { Sensors = Enumerable.Range(0, n).Select(i => new BaselineSensor { Id = $"sensor-{i}", Value = i }).ToArray() });
+            if (Environment.GetEnvironmentVariable("BENCH_BASELINE_FIRST") == "1") { baseline(); generated(); }
+            else { generated(); baseline(); }
         }
     }
 }

benchmarks/csharp/README.md

@@ -15,3 +15,11 @@ native PolyXML codec.
 Use `BENCH_ITERATIONS=2 ./benchmarks/csharp/run.sh` for a smoke run. Repeat full
 runs on an idle host, retain raw output, and record Git revision, CPU, and
 `dotnet --version` before publishing numbers. Generated code stays in `target/`.
+
+For steady-state comparisons, warmup runs for at least two seconds and 500
+read/write pairs per model. Writes materialize a UTF-16 `string`; reads start
+from the shared XML decoded to a string outside timing. Set
+`BENCH_BASELINE_FIRST=1` in alternate fresh processes to rotate model order.
+`BENCH_ITERATIONS` defaults to 1,000 and `BENCH_WARMUP_MS` to 2,000. Every
+sensor field is checked before timing. Record the actual .NET runtime when
+rolling a `net8.0` build forward; it is not a .NET 8 timing in that case.

benchmarks/go/README.md

@@ -13,3 +13,9 @@ For a smoke run use `BENCH_TIME=1x ./benchmarks/go/run.sh`. For reportable
 measurements, run on an idle host with `BENCH_COUNT=5 BENCH_TIME=3s`, retain the
 raw output, and record the Git revision, OS/CPU, and `go version`. Generated
 sources are ignored under `target/`.
+
+Every sensor field is checked outside timing; timed reads only unmarshal.
+Set `BENCH_BASELINE_FIRST=1` in alternate fresh processes to rotate model
+order. For a repeated comparison, run five separate processes with
+`BENCH_COUNT=1 BENCH_TIME=2s` and retain every output, including `B/op` and
+`allocs/op`. Both models use the same native `encoding/xml` implementation.

benchmarks/go/bench_test.go

@@ -25,47 +25,59 @@ func BenchmarkXML(b *testing.B) {
 				generated[i] = models.Sensor{ID: fmt.Sprintf("sensor-%d", i), Value: int32(i)}
 				plain[i] = baseline{ID: generated[i].ID, Value: generated[i].Value}
 			}
-			for _, tc := range []struct {
+			cases := []struct {
 				name   string
 				values any
-				read   func([]byte) error
+				read   func([]byte, bool) error
 			}{
-				{"generated", generated, func(data []byte) error {
+				{"generated", generated, func(data []byte, verify bool) error {
 					var v struct {
 						Items []models.Sensor `xml:"Sensor"`
 					}
 					if err := xml.Unmarshal(data, &v); err != nil {
 						return err
 					}
-					if len(v.Items) != count ||
-						v.Items[0].ID != generated[0].ID || v.Items[0].Value != generated[0].Value ||
-						v.Items[count/2].ID != generated[count/2].ID || v.Items[count/2].Value != generated[count/2].Value ||
-						v.Items[count-1].ID != generated[count-1].ID || v.Items[count-1].Value != generated[count-1].Value {
-						return fmt.Errorf("generated round trip mismatch")
+					if verify {
+						if len(v.Items) != count {
+							return fmt.Errorf("generated count mismatch")
+						}
+						for i, sensor := range v.Items {
+							if sensor.ID != generated[i].ID || sensor.Value != generated[i].Value {
+								return fmt.Errorf("generated field mismatch at %d", i)
+							}
+						}
 					}
 					return nil
 				}},
-				{"baseline", plain, func(data []byte) error {
+				{"baseline", plain, func(data []byte, verify bool) error {
 					var v struct {
 						Items []baseline `xml:"Sensor"`
 					}
 					if err := xml.Unmarshal(data, &v); err != nil {
 						return err
 					}
-					if len(v.Items) != count ||
-						v.Items[0].ID != plain[0].ID || v.Items[0].Value != plain[0].Value ||
-						v.Items[count/2].ID != plain[count/2].ID || v.Items[count/2].Value != plain[count/2].Value ||
-						v.Items[count-1].ID != plain[count-1].ID || v.Items[count-1].Value != plain[count-1].Value {
-						return fmt.Errorf("baseline round trip mismatch")
+					if verify {
+						if len(v.Items) != count {
+							return fmt.Errorf("baseline count mismatch")
+						}
+						for i, sensor := range v.Items {
+							if sensor.ID != plain[i].ID || sensor.Value != plain[i].Value {
+								return fmt.Errorf("baseline field mismatch at %d", i)
+							}
+						}
 					}
 					return nil
 				}},
-			} {
+			}
+			if os.Getenv("BENCH_BASELINE_FIRST") == "1" {
+				cases[0], cases[1] = cases[1], cases[0]
+			}
+			for _, tc := range cases {
 				input, err := os.ReadFile(fmt.Sprintf("../workloads/sensor-batch/sensor-%d.xml", count))
 				if err != nil {
 					b.Fatal(err)
 				}
-				if err := tc.read(input); err != nil {
+				if err := tc.read(input, true); err != nil {
 					b.Fatal(err)
 				}
 				output, err := xml.Marshal(struct {
@@ -83,7 +95,7 @@ func BenchmarkXML(b *testing.B) {
 					b.ReportAllocs()
 					b.ResetTimer()
 					for i := 0; i < b.N; i++ {
-						if err := tc.read(input); err != nil {
+						if err := tc.read(input, false); err != nil {
 							b.Fatal(err)
 						}
 					}

benchmarks/java/README.md

@@ -91,3 +91,8 @@ harness; these JMH results do not measure them. Generated direct codecs have sta
 model calls, but StAX provider selection and application dependencies still determine
 native-image configuration. Do not infer universal zero-configuration AOT support
 from the absence of model reflection.
+
+The matched serializer refresh checks every input against all three typed
+readers before measurement. JMH logs include aggregate input bytes and
+per-message writer output lengths. Reads use varying IDs; writes repeatedly
+serialize the same preconstructed ID-0 model to fresh output bytes.

benchmarks/java/src/main/java/io/polyxml/bench/BindingBenchmark.java

@@ -58,13 +58,25 @@ record = io.polyxml.bench.records.MessageCodec.readXml(new ByteArrayInputStream(
         verify();
     }
     public void verify() throws Exception {
+        // Check every input and field across the matched typed readers before timing.
+        for (byte[] xml : documents) {
+            var expected = MessageCodec.readXml(new ByteArrayInputStream(xml));
+            if (!expected.equals(mapper.readValue(xml, Message.class))) throw new AssertionError("Jackson input mismatch");
+            var decoded = unmarshaller.unmarshal(new javax.xml.transform.stream.StreamSource(new ByteArrayInputStream(xml)), io.polyxml.bench.jaxb.Message.class).getValue();
+            var encoded = new ByteArrayOutputStream();
+            marshaller.marshal(new JAXBElement<>(root, io.polyxml.bench.jaxb.Message.class, decoded), encoded);
+            if (!expected.equals(MessageCodec.readXml(new ByteArrayInputStream(encoded.toByteArray())))) throw new AssertionError("JAXB input mismatch");
+        }
         var direct = new ByteArrayOutputStream(); MessageCodec.writeXml(pojo, direct);
         if (!pojo.equals(MessageCodec.readXml(new ByteArrayInputStream(direct.toByteArray())))) throw new AssertionError("direct round trip");
         if (!pojo.equals(mapper.readValue(mapper.writeValueAsBytes(pojo), Message.class))) throw new AssertionError("Jackson round trip");
         ObjectMapper json = new ObjectMapper().registerModule(new Jdk8Module());
         if (!pojo.equals(json.readValue(json.writeValueAsBytes(pojo), Message.class))) throw new AssertionError("JSON round trip");
         var jaxb = new ByteArrayOutputStream(); marshaller.marshal(new JAXBElement<>(root, io.polyxml.bench.jaxb.Message.class, legacy), jaxb);
         if (!pojo.equals(MessageCodec.readXml(new ByteArrayInputStream(jaxb.toByteArray())))) throw new AssertionError("JAXB round trip");
+        long inputBytes = 0; for (byte[] xml : documents) inputBytes += xml.length;
+        System.out.printf("fixture: workload=%s,batch=%d,input_bytes=%d,direct_output_bytes=%d,jackson_output_bytes=%d,jaxb_output_bytes=%d%n",
+            workload, batchSize, inputBytes, direct.size(), mapper.writeValueAsBytes(pojo).length, jaxb.size());
         var records = new ByteArrayOutputStream(); io.polyxml.bench.records.MessageCodec.writeXml(record,records);
         if (!pojo.equals(MessageCodec.readXml(new ByteArrayInputStream(records.toByteArray())))) throw new AssertionError("record round trip");
     }

benchmarks/serializer-comparison/.gitignore

@@ -0,0 +1 @@
+/target/

benchmarks/serializer-comparison/Providers.java

@@ -0,0 +1,7 @@
+/** Record the StAX provider actually selected by the shaded benchmark classpath. */
+public class Providers {
+    public static void main(String[] args) {
+        System.out.println("input=" + javax.xml.stream.XMLInputFactory.newFactory().getClass().getName());
+        System.out.println("output=" + javax.xml.stream.XMLOutputFactory.newFactory().getClass().getName());
+    }
+}