From 07ffc9bdb3a52c1564d704543ca53a9e031e56d6 Mon Sep 17 00:00:00 2001 From: Bundo Lee Date: Mon, 19 Oct 2026 18:35:36 +0900 Subject: [PATCH] Give each line chunk only the operator that painted it processLineArts copied the window's whole operator list onto every line chunk in it: L lines times P paint operators StreamInfo objects. A window is everything painted between two text or XObject boundaries, so one chart drawn without text in between reaches thousands of both. A 55-page paper (one form with 7,500 strokes) made 56 million copies and ran out of a 2 GB heap in parseChunks; a 2.6 MB document made 325 million and failed at 5 GB. Over a 200-document corpus, 5 documents needed more than 2 GB for these copies alone. A line now carries one StreamInfo, for its own paint operator, and the full list goes once onto the window's LineArtChunk, as it already did. Memory is L + P instead of L x P. This changes what a consumer sees on a line, and one relies on it: odl-pdfua's SemanticMapper builds a figure from a region's lines and needs the whole window, curves and fills included. It is changed alongside to find the window's LineArtChunk through the line's operator, and through the box when the key is ambiguous (a form drawn twice, or two forms sharing a resource name). Land that change before this one ships. Only data-loader mode is affected (StaticContainers.isDataLoader), so validation is unchanged. opendataloader-pdf's own tagger does not read line art stream infos: its tagged PDFs are byte-identical before and after on the documents that completed before, and the two that did not now complete in under 300 MB. Not changed here: parseLineArts, used when MCIDs are not ignored (core's --use-struct-tree), still copies the stream's unmarked operators onto every separate box, boxes x operators. Co-Authored-By: Claude Opus 5.5 --- .../gf/model/factory/chunks/ChunkParser.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java index af389f555..6626c9298 100644 --- a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java +++ b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java @@ -738,6 +738,15 @@ private void processLineChunk(BoundingBox boundingBox, Long mcid, LineChunk line lineArtContainer.add(mcid, lineChunk, operatorIndex, xObjectName); if (StaticStorages.getIsIgnoreMCIDs()) { boundingBox.union(lineChunk.getBoundingBox()); + // The line carries only the operator that painted it. The region's + // full list goes once onto the LineArtChunk that processLineArts + // builds, whose box holds the line, and a consumer that needs the + // whole region finds that chunk through this operator and that box. + // Giving every line the full list cost lines x operators objects: one + // chart of 7,500 strokes made 56 million and ran out of heap. + if (StaticContainers.isDataLoader()) { + lineChunk.getStreamInfos().add(new StreamInfo(operatorIndex, xObjectName, null)); + } } } @@ -1161,11 +1170,8 @@ public void processLineArts() { List pending = lineArtContainer.getStreamInfos(null); List lineChunks = lineArtContainer.getLineChunks(null); if (lineChunks != null && !lineChunks.isEmpty()) { - for (LineChunk lineChunk : lineChunks) { - for (StreamInfo info : pending) { - lineChunk.getStreamInfos().add(new StreamInfo(info)); - } - } + // Each line already holds its own operator (processLineChunk). The + // region's list is attached to the boxes below only, once per box. artifacts.addAll(lineChunks); lineChunks.clear(); }