From 39b273d69e08991e0140b676ad573a16c46065cb Mon Sep 17 00:00:00 2001 From: Ade Anom A Date: Thu, 22 Oct 2026 14:31:01 +0700 Subject: [PATCH 1/3] Add dump structure command for the tagged-PDF structure tree --- CHANGELOG.md | 1 + cmd/cli/cmd_docview.go | 7 +- cmd/cli/cmd_structure.go | 336 +++++++ cmd/cli/cmd_structure_plain_test.go | 360 +++++++ cmd/cli/cmd_structure_test.go | 55 ++ cmd/cli/doc.go | 4 +- cmd/cli/main.go | 3 + cmd/cli/usage.go | 2 + docs/cli-usage.md | 3 +- docs/cli/structure.md | 52 +- internal/pdfcore/markedcontent.go | 217 +++++ internal/pdfcore/markedcontent_test.go | 243 +++++ internal/pdfcore/markedcontent_tokens_test.go | 250 +++++ internal/pdfcore/model.go | 149 +++ internal/pdfcore/numbertree.go | 202 ++++ internal/pdfcore/numbertree_shapes_test.go | 88 ++ internal/pdfcore/numbertree_test.go | 244 +++++ internal/pdfcore/stream.go | 79 +- internal/pdfcore/structure.go | 837 ++++++++++++++++ internal/pdfcore/structure_bench_test.go | 130 +++ internal/pdfcore/structure_raw_test.go | 154 +++ internal/pdfcore/structure_shapes_test.go | 450 +++++++++ internal/pdfcore/structure_test.go | 898 ++++++++++++++++++ testdata/generate_test.go | 57 ++ testdata/structure-broken.pdf | Bin 0 -> 1856 bytes testdata/structure.pdf | Bin 0 -> 2233 bytes tests/dump-structure/dump_structure_test.go | 222 +++++ tests/dump-structure/go.mod | 3 + tests/dump-structure/helpers_test.go | 238 +++++ 29 files changed, 5243 insertions(+), 41 deletions(-) create mode 100644 cmd/cli/cmd_structure.go create mode 100644 cmd/cli/cmd_structure_plain_test.go create mode 100644 cmd/cli/cmd_structure_test.go create mode 100644 internal/pdfcore/markedcontent.go create mode 100644 internal/pdfcore/markedcontent_test.go create mode 100644 internal/pdfcore/markedcontent_tokens_test.go create mode 100644 internal/pdfcore/numbertree.go create mode 100644 internal/pdfcore/numbertree_shapes_test.go create mode 100644 internal/pdfcore/numbertree_test.go create mode 100644 internal/pdfcore/structure.go create mode 100644 internal/pdfcore/structure_bench_test.go create mode 100644 internal/pdfcore/structure_raw_test.go create mode 100644 internal/pdfcore/structure_shapes_test.go create mode 100644 internal/pdfcore/structure_test.go create mode 100644 testdata/structure-broken.pdf create mode 100644 testdata/structure.pdf create mode 100644 tests/dump-structure/dump_structure_test.go create mode 100644 tests/dump-structure/go.mod create mode 100644 tests/dump-structure/helpers_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 603132b..10485b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to UniDoc PDF Debugger are recorded here. Format follows Kee ### Added +- `dump structure [--json] ` prints a tagged PDF's structure tree and lines it up with the page content. Each element shows its `/S` (and the role it maps to), reference, page, `/T`, `/Lang`, `/Alt`, `/ActualText` and `/E`, with its MCID, MCR and OBJR kids underneath. Each page gets a table, one row per MCID, with how often the content marks it, the element in the page's ParentTree slot and the elements whose `/K` claims it; rows where those do not agree on one element are marked `*`. Cycles, a tree deeper than 256 levels, a `/Pg` that is not a page, an EMC with nothing open, a missing property list, a `/StructParents` key the ParentTree lacks and malformed ParentTree nodes are reported where they occur, and the command still exits 0. `--json` carries the full report. MCIDs inside Form XObjects are not aligned - File > "Reload from Disk" (Cmd+R on macOS, Ctrl+R on Windows and Linux, and F5 everywhere) opens the active tab's file again in place. The tab keeps its position and its left-rail view; the tree, selection, detail panel, history and recent jumps reset as for a fresh open. A short "reloaded - view reset" note appears at the top of the window. A reload that finishes after you switched to another tab replaces that tab without taking focus. If the file is gone or no longer parses, the old tab stays as it was and the error banner says why; on Windows a file another app has locked shows "in use by another app - try again" instead - Open files are watched for changes on disk, background tabs included. When a file is rewritten, replaced or deleted, its tab gets a small amber dot (2 to 4 seconds after the write while the window has focus, 10 to 20 seconds otherwise, also for a file a generator keeps rewriting) and a tooltip with the time it changed. The detail panel shows "Document updated" with Reload and Compare buttons, or "File not found". Nothing reloads by itself, and the marker never moves focus or shifts the layout. Detection compares size, modification time and file identity, so an in-place rewrite that keeps all three is not seen - File > "Compare with File on Disk" diffs the open version against the file as it is now on disk, in the Diff panel, labelled "On disk (14:05)". Files over 500 MB ask first. Reload in the Diff header then makes the compared copy the tab's document without parsing it again diff --git a/cmd/cli/cmd_docview.go b/cmd/cli/cmd_docview.go index 350d377..340611e 100644 --- a/cmd/cli/cmd_docview.go +++ b/cmd/cli/cmd_docview.go @@ -8,16 +8,17 @@ import ( ) // docViewFlags holds the parsed flags common to the document-level dump -// subcommands (xref, objects, pages, images, bytes, signatures). None take --ref. +// subcommands (xref, objects, pages, structure, images, bytes, signatures). None +// take --ref. type docViewFlags struct { pretty bool - json bool // opt into JSON: structured for xref/objects/pages/images/signatures, the text wrapper for bytes + json bool // opt into JSON: structured for xref/objects/pages/structure/images/signatures, the text wrapper for bytes } // parseDocViewFlags builds and parses a FlagSet for a document-level dump // subcommand. resource is the bare resource name used in the usage message. // --json is read into f.json uniformly; the handler decides its meaning: a -// format switch for xref/objects/pages/images/signatures (plain-text default +// format switch for xref/objects/pages/structure/images/signatures (plain-text default // -> JSON) or a payload wrapper for bytes (raw bytes default -> decoded JSON // payload). On failure it writes usage and returns ok=false. func parseDocViewFlags(resource string, args []string) (filePath string, f docViewFlags, ok bool) { diff --git a/cmd/cli/cmd_structure.go b/cmd/cli/cmd_structure.go new file mode 100644 index 0000000..4a6d163 --- /dev/null +++ b/cmd/cli/cmd_structure.go @@ -0,0 +1,336 @@ +package main + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + + "unidoc-pdf-debugger/internal/pdfcore" +) + +// runStructureDump parses flags and dispatches the tagged-structure dump. +func runStructureDump(args []string) int { + filePath, f, ok := parseDocViewFlags("structure", args) + if !ok { + return 1 + } + return execStructureDump(filePath, f) +} + +// execStructureDump opens the PDF and renders its structure report. +func execStructureDump(filePath string, f docViewFlags) (exitCode int) { + defer func() { + if r := recover(); r != nil { + writeJSONError(os.Stderr, fmt.Sprintf("internal error: %v", r)) + exitCode = 2 + } + }() + + ins, _, code := openForCLI(filePath) + if code != 0 { + return code + } + defer func() { _ = ins.Close("cli") }() + + report, err := ins.GetStructure("cli") + if err != nil { + writeJSONError(os.Stderr, err.Error()) + return 2 + } + return writeStructureDump(os.Stdout, os.Stderr, report, f) +} + +// writeStructureDump writes the report to stdout: the JSON object with +// --json, the plain-text view otherwise. Findings never change the exit code; +// a failed write exits 2 with the error object on stderr. +func writeStructureDump(stdout, stderr io.Writer, r *pdfcore.StructureReport, f docViewFlags) int { + var err error + if f.json { + err = emit(stdout, r, f.pretty) + } else { + _, err = io.WriteString(stdout, structurePlain(r)) + } + if err != nil { + writeJSONError(stderr, fmt.Sprintf("failed to write output: %v", err)) + return 2 + } + return 0 +} + +// structurePlain renders the report as plain text. NON-CONTRACTUAL; use +// --json to parse. +func structurePlain(r *pdfcore.StructureReport) string { + var b strings.Builder + if !r.HasStructTree { + b.WriteString("No structure tree (/StructTreeRoot absent)\n") + } else { + writeStructureHeader(&b, r) + } + if r.HasStructTree || len(r.Elements) > 0 { + c := r.Counts + fmt.Fprintf(&b, "\nElements (%d; kids: %d MCID, %d MCR, %d OBJR, %d error):\n", + c.Elements, c.MCIDKids, c.MCRKids, c.OBJRKids, c.ErrorKids) + writeStructKids(&b, r.Elements, 0) + } + b.WriteString("\n") + writeStructPages(&b, r.Pages) + if len(r.OtherParentTree) > 0 { + b.WriteString("\nOther ParentTree keys:\n") + for _, e := range r.OtherParentTree { + line := fmt.Sprintf(" %d -> %s", e.Key, joinNonEmpty(e.ElementRef, e.S)) + if e.Err != "" { + line += " error: " + e.Err + } + b.WriteString(asciiSafe(line) + "\n") + } + } + if len(r.Problems) > 0 { + fmt.Fprintf(&b, "\nProblems (%d):\n", len(r.Problems)) + for _, p := range r.Problems { + b.WriteString(" " + asciiSafe(p.Where+": "+p.Message) + "\n") + } + } + return b.String() +} + +// writeStructureHeader writes the document-level lines. +func writeStructureHeader(b *strings.Builder, r *pdfcore.StructureReport) { + line := "Structure tree: " + orText(r.RootRef, "(direct)") + if r.ParentTreeNextKey != nil { + line += fmt.Sprintf(" (ParentTreeNextKey %d)", *r.ParentTreeNextKey) + } + b.WriteString(line + "\n") + if r.HasParentTree { + b.WriteString("ParentTree: " + orText(r.ParentTreeRef, "(direct)") + "\n") + } else { + b.WriteString("ParentTree: absent\n") + } + marked := "absent" + if r.Marked != nil { + marked = strconv.FormatBool(*r.Marked) + } + b.WriteString("MarkInfo /Marked: " + marked + "\n") + b.WriteString("Document /Lang: " + asciiSafe(orText(r.DocLang, "absent")) + "\n") + if len(r.RoleMap) > 0 { + pairs := make([]string, len(r.RoleMap)) + for i, e := range r.RoleMap { + pairs[i] = e.From + " -> " + orText(e.To, "?") + } + b.WriteString("RoleMap: " + asciiSafe(strings.Join(pairs, ", ")) + "\n") + } +} + +// writeStructKids writes kids, two spaces of indent per level. +func writeStructKids(b *strings.Builder, kids []*pdfcore.StructKid, level int) { + indent := strings.Repeat(" ", level) + for _, k := range kids { + var line string + switch k.Kind { + case "element": + line = elementLine(k.Element) + case "mcid": + line = fmt.Sprintf("MCID %d", *k.MCID) + case "mcr": + line = "MCR" + if k.MCID != nil { + line += fmt.Sprintf(" MCID %d", *k.MCID) + } + if k.Page > 0 { + line += fmt.Sprintf(" page %d", k.Page) + } + if k.StmRef != "" { + line += " stream " + k.StmRef + } + case "objr": + line = "OBJR " + joinNonEmpty(orText(k.ObjRef, "-"), k.ObjType) + default: + line = "error: " + k.Err + } + if k.Kind != "error" && k.Kind != "element" && k.Err != "" { + line += " error: " + k.Err + } + b.WriteString(indent + asciiSafe(line) + "\n") + if k.Element != nil { + writeStructKids(b, k.Element.Kids, level+1) + } + } +} + +// elementLine renders one element: S, mapped S, ref, page and text fields. +func elementLine(e *pdfcore.StructElem) string { + parts := []string{orText(e.S, "(no /S)")} + if e.MappedS != "" { + parts[0] += " (" + e.MappedS + ")" + } + parts[0] += " " + orText(e.Ref, "(direct)") + if e.Page > 0 { + p := fmt.Sprintf("page %d", e.Page) + if e.PageInherited { + p += " (inherited)" + } + parts = append(parts, p) + } else if e.PgRef != "" { + parts = append(parts, "/Pg "+e.PgRef) + } + text := func(label, v string) { + c := pdfcore.ClampDisplayValue(v, pdfcore.TreeValueCap) + if s := asciiSafe(c); s != c { + parts = append(parts, label+" "+s) + } else { + parts = append(parts, label+" \""+c+"\"") + } + } + if e.Title != "" { + text("T", e.Title) + } + if e.Lang != "" { + text("Lang", e.Lang) + } + for _, f := range []struct { + label string + v *string + }{{"Alt", e.Alt}, {"ActualText", e.ActualText}, {"E", e.E}} { + if f.v != nil { + text(f.label, *f.v) + } + } + if e.Err != "" { + parts = append(parts, "error: "+e.Err) + } + return strings.Join(parts, " ") +} + +// pageIsEmpty reports whether a page has nothing to show: no /StructParents, +// no marked content, no alignment row (a tree element may claim an MCID on a +// page with neither) and no error. +func pageIsEmpty(p *pdfcore.StructPage) bool { + if p.StructParents != nil || p.ParentTreeErr != "" || p.Err != "" || len(p.ContentMCIDs) > 0 || len(p.Alignment) > 0 { + return false + } + for _, s := range p.Streams { + if s.Opens > 0 || s.Closes > 0 || s.Err != "" { + return false + } + } + return true +} + +// writeStructPages writes one block per page with structure content and +// collapses the rest into one range list. +func writeStructPages(b *strings.Builder, pages []*pdfcore.StructPage) { + var empty []int + legend := false + for _, p := range pages { + if pageIsEmpty(p) { + empty = append(empty, p.PageNum) + continue + } + opens, closes := 0, 0 + for _, s := range p.Streams { + opens += s.Opens + closes += s.Closes + } + sp := "absent" + if p.StructParents != nil { + sp = strconv.Itoa(*p.StructParents) + } + streams := fmt.Sprintf("%d streams", len(p.Streams)) + if len(p.Streams) == 1 { + streams = "1 stream" + } + fmt.Fprintf(b, "Page %d /StructParents %s %s BDC/BMC %d, EMC %d\n", p.PageNum, sp, streams, opens, closes) + if p.Err != "" { + b.WriteString(" error: " + asciiSafe(p.Err) + "\n") + } + if p.ParentTreeErr != "" { + b.WriteString(" ParentTree error: " + asciiSafe(p.ParentTreeErr) + "\n") + } + for _, s := range p.ParentTree { + if s.Err != "" { + fmt.Fprintf(b, " ParentTree slot %d: error: %s\n", s.MCID, asciiSafe(s.Err)) + } + } + for _, s := range p.Streams { + switch { + case s.Err != "": + b.WriteString(" stream " + s.Ref + ": error: " + asciiSafe(s.Err) + "\n") + case s.MinDepth < 0 || s.DepthAtEnd != 0: + fmt.Fprintf(b, " stream %s: BDC/BMC %d, EMC %d, depth at end %d, min depth %d\n", s.Ref, s.Opens, s.Closes, s.DepthAtEnd, s.MinDepth) + } + } + for _, c := range p.ContentMCIDs { + if c.Err != "" { + ref := strconv.Itoa(c.Stream) + if c.Stream < len(p.Streams) { + ref = p.Streams[c.Stream].Ref + } + fmt.Fprintf(b, " stream %s line %d col %d: %s %s\n", ref, c.Line, c.Col, asciiSafe(orText(c.Tag, "?")), asciiSafe(c.Err)) + } + } + if len(p.Alignment) == 0 { + continue + } + t := newTable("MCID", "CONTENT", "PARENTTREE", "TREE", "") + starred := false + for _, a := range p.Alignment { + mark := "" + if !a.Consistent { + mark, starred = "*", true + } + t.AddRow(strconv.Itoa(a.MCID), strconv.Itoa(a.ContentCount), + orText(joinNonEmpty(a.ParentTreeElement, a.ParentTreeS), "-"), + orText(strings.Join(a.TreeElements, ", "), "-"), mark) + } + var tb strings.Builder + _ = t.Render(&tb) + for line := range strings.SplitSeq(strings.TrimSuffix(tb.String(), "\n"), "\n") { + b.WriteString(" " + asciiSafe(strings.TrimRight(line, " ")) + "\n") + } + if starred && !legend { + b.WriteString(" * = content count, ParentTree slot and tree claimants do not line up on one element\n") + legend = true + } + } + if len(empty) > 0 { + b.WriteString("Pages with no /StructParents and no marked content: " + pageRanges(empty) + "\n") + } +} + +// pageRanges renders ascending page numbers as "1-3, 5". +func pageRanges(nums []int) string { + var parts []string + for i := 0; i < len(nums); { + j := i + for j+1 < len(nums) && nums[j+1] == nums[j]+1 { + j++ + } + if i == j { + parts = append(parts, strconv.Itoa(nums[i])) + } else { + parts = append(parts, fmt.Sprintf("%d-%d", nums[i], nums[j])) + } + i = j + 1 + } + return strings.Join(parts, ", ") +} + +// orText returns s, or alt when s is empty. +func orText(s, alt string) string { + if s == "" { + return alt + } + return s +} + +// joinNonEmpty joins the non-empty values with a space. +func joinNonEmpty(vals ...string) string { + var out []string + for _, v := range vals { + if v != "" { + out = append(out, v) + } + } + return strings.Join(out, " ") +} diff --git a/cmd/cli/cmd_structure_plain_test.go b/cmd/cli/cmd_structure_plain_test.go new file mode 100644 index 0000000..c1c19cd --- /dev/null +++ b/cmd/cli/cmd_structure_plain_test.go @@ -0,0 +1,360 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "unidoc-pdf-debugger/internal/pdfcore" +) + +func strPtr(s string) *string { return &s } + +func intPtr(n int) *int { return &n } + +// lineOf returns the first line of out containing sub, or fails. +func lineOf(t *testing.T, out, sub string) string { + t.Helper() + for line := range strings.SplitSeq(out, "\n") { + if strings.Contains(line, sub) { + return line + } + } + t.Fatalf("no line contains %q:\n%s", sub, out) + return "" +} + +func TestStructurePlain_HeaderVariants(t *testing.T) { + marked := false + r := &pdfcore.StructureReport{ + HasStructTree: true, + Marked: &marked, + RoleMap: []pdfcore.RoleMapEntry{{From: "Bad", To: ""}, {From: "Heading1", To: "H1"}}, + } + out := structurePlain(r) + for _, want := range []string{ + "Structure tree: (direct)\n", + "ParentTree: absent\n", + "MarkInfo /Marked: false\n", + "Document /Lang: absent\n", + "RoleMap: Bad -> ?, Heading1 -> H1\n", + "Elements (0; kids: 0 MCID, 0 MCR, 0 OBJR, 0 error):\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + + next := 4 + r = &pdfcore.StructureReport{ + HasStructTree: true, RootRef: "12 0 R", ParentTreeNextKey: &next, + HasParentTree: true, ParentTreeRef: "31 0 R", DocLang: "de-DE", + } + out = structurePlain(r) + for _, want := range []string{ + "Structure tree: 12 0 R (ParentTreeNextKey 4)\n", + "ParentTree: 31 0 R\n", + "MarkInfo /Marked: absent\n", + "Document /Lang: de-DE\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "RoleMap:") { + t.Errorf("empty RoleMap printed:\n%s", out) + } +} + +func TestStructurePlain_UntaggedHasNoHeaderOrElements(t *testing.T) { + r := &pdfcore.StructureReport{Pages: []*pdfcore.StructPage{{PageNum: 1}, {PageNum: 2}, {PageNum: 4}}} + out := structurePlain(r) + if !strings.HasPrefix(out, "No structure tree (/StructTreeRoot absent)\n") { + t.Errorf("output does not open with the no-tree line:\n%s", out) + } + if strings.Contains(out, "Elements (") || strings.Contains(out, "Structure tree:") { + t.Errorf("untagged output has a header or element section:\n%s", out) + } + if !strings.Contains(out, "no marked content: 1-2, 4\n") { + t.Errorf("empty pages not collapsed into ranges:\n%s", out) + } +} + +func TestStructurePlain_ElementTree(t *testing.T) { + long := strings.Repeat("a", 100) + r := &pdfcore.StructureReport{ + HasStructTree: true, + Counts: pdfcore.StructureCounts{Elements: 3, MCIDKids: 2, MCRKids: 2, OBJRKids: 2, ErrorKids: 1}, + Elements: []*pdfcore.StructKid{ + {Kind: "element", Element: &pdfcore.StructElem{ + S: "Heading1", MappedS: "H1", Ref: "14 0 R", Page: 1, Title: "Intro", Lang: "en", + Kids: []*pdfcore.StructKid{ + {Kind: "mcid", MCID: intPtr(0), Page: 1}, + {Kind: "mcid", MCID: intPtr(1), Err: "no page"}, + {Kind: "element", Element: &pdfcore.StructElem{ + S: "P", Page: 2, PageInherited: true, ActualText: strPtr(long), Alt: strPtr("caf\u00e9"), E: strPtr(""), + Kids: []*pdfcore.StructKid{ + {Kind: "mcr", MCID: intPtr(3), Page: 2, StmRef: "40 0 R"}, + {Kind: "mcr", Err: "MCR has no /MCID"}, + {Kind: "objr", ObjRef: "22 0 R", ObjType: "Annot/Link"}, + {Kind: "objr", Err: "OBJR has no /Obj"}, + }, + }}, + }, + }}, + {Kind: "element", Element: &pdfcore.StructElem{PgRef: "9 0 R", Err: "no /S; /Pg 9 0 R is a dictionary that is not a page in the page tree"}}, + {Kind: "error", Err: "element 14 0 R already listed; not walked again"}, + }, + } + out := structurePlain(r) + + if !strings.Contains(out, "Elements (3; kids: 2 MCID, 2 MCR, 2 OBJR, 1 error):\n") { + t.Errorf("counts line missing:\n%s", out) + } + lines := []string{ + `Heading1 (H1) 14 0 R page 1 T "Intro" Lang "en"`, + ` MCID 0`, + ` MCID 1 error: no page`, + ` P (direct) page 2 (inherited) Alt "caf\u00e9" ActualText "` + strings.Repeat("a", 80) + ` [truncated: 80 of 100]" E ""`, + ` MCR MCID 3 page 2 stream 40 0 R`, + ` MCR error: MCR has no /MCID`, + ` OBJR 22 0 R Annot/Link`, + ` OBJR - error: OBJR has no /Obj`, + `(no /S) (direct) /Pg 9 0 R error: no /S; /Pg 9 0 R is a dictionary that is not a page in the page tree`, + `error: element 14 0 R already listed; not walked again`, + } + for _, want := range lines { + if !strings.Contains(out, "\n"+want+"\n") { + t.Errorf("output missing line %q:\n%s", want, out) + } + } + for line := range strings.SplitSeq(out, "\n") { + for i := 0; i < len(line); i++ { + if line[i] < 0x20 || line[i] > 0x7e { + t.Fatalf("non-ASCII byte in line %q", line) + } + } + } +} + +func TestStructurePlain_PageErrorLines(t *testing.T) { + sp := 0 + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, StructParents: &sp, Err: "contents array element 0 is not an indirect reference"}, + {PageNum: 2, ParentTreeErr: "ParentTree has no key 5"}, + {PageNum: 3, Streams: []pdfcore.MarkedStream{ + {Ref: "20 0 R", Err: "failed to decode stream: bad"}, + {Ref: "21 0 R", Opens: 1, Closes: 2, DepthAtEnd: 0, MinDepth: -1}, + {Ref: "22 0 R", Opens: 1, Closes: 0, DepthAtEnd: 1, MinDepth: 0}, + {Ref: "23 0 R", Opens: 1, Closes: 1}, + }, ContentMCIDs: []pdfcore.ContentMCID{ + {MCID: -1, Tag: "Span", Stream: 1, Line: 2, Col: 3, Err: "property list /MC9 not found"}, + {MCID: -1, Stream: 7, Line: 1, Col: 1, Err: "BDC has 0 of its 2 operands"}, + {MCID: 0, Stream: 3}, + }}, + }, + } + out := structurePlain(r) + for _, want := range []string{ + "Page 1 /StructParents 0 0 streams BDC/BMC 0, EMC 0\n error: contents array element 0 is not an indirect reference\n", + "Page 2 /StructParents absent 0 streams BDC/BMC 0, EMC 0\n ParentTree error: ParentTree has no key 5\n", + "Page 3 /StructParents absent 4 streams BDC/BMC 3, EMC 3\n", + " stream 20 0 R: error: failed to decode stream: bad\n", + " stream 21 0 R: BDC/BMC 1, EMC 2, depth at end 0, min depth -1\n", + " stream 22 0 R: BDC/BMC 1, EMC 0, depth at end 1, min depth 0\n", + " stream 21 0 R line 2 col 3: Span property list /MC9 not found\n", + " stream 7 line 1 col 1: ? BDC has 0 of its 2 operands\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "stream 23 0 R") { + t.Errorf("balanced stream printed a line:\n%s", out) + } + if strings.Contains(out, "Pages with no /StructParents") { + t.Errorf("pages with errors collapsed:\n%s", out) + } +} + +func TestStructurePlain_LegendOnlyUnderFirstStarredTable(t *testing.T) { + inconsistent := []pdfcore.MCIDAlignment{{MCID: 0, ContentCount: 2, TreeElements: []string{}}} + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, Alignment: []pdfcore.MCIDAlignment{{MCID: 0, ContentCount: 1, ParentTreeElement: "10 0 R", ParentTreeS: "P", TreeElements: []string{"10 0 R"}, Consistent: true}}}, + {PageNum: 2, Alignment: inconsistent}, + {PageNum: 3, Alignment: inconsistent}, + }, + } + out := structurePlain(r) + if n := strings.Count(out, "* = content count"); n != 1 { + t.Errorf("legend printed %d times, want once:\n%s", n, out) + } + if strings.Index(out, "* = content count") < strings.Index(out, "Page 2 ") { + t.Errorf("legend before the first starred table:\n%s", out) + } + if got := lineOf(t, out, "10 0 R P"); strings.HasSuffix(got, "*") { + t.Errorf("consistent row starred: %q", got) + } + after := out[strings.Index(out, "Page 2 "):] + row := after[strings.Index(after, "\n 0 ")+1:] + row = row[:strings.Index(row, "\n")] + if got := strings.Fields(row); strings.Join(got, " ") != "0 2 - - *" { + t.Errorf("inconsistent row with no claimants = %q, want fields 0 2 - - *", row) + } +} + +func TestStructurePlain_OtherParentTreeAndProblems(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + OtherParentTree: []pdfcore.ParentTreeEntry{ + {Key: 2, ElementRef: "17 0 R", S: "Link"}, + {Key: 7, Err: "value is an array, but no page's /StructParents names this key"}, + }, + Problems: []pdfcore.StructureProblem{{Where: "ParentTree 31 0 R, node 33 0 R", Message: "key 1 outside /Limits [0 0]"}}, + } + out := structurePlain(r) + for _, want := range []string{ + "\nOther ParentTree keys:\n 2 -> 17 0 R Link\n 7 -> error: value is an array, but no page's /StructParents names this key\n", + "\nProblems (1):\n ParentTree 31 0 R, node 33 0 R: key 1 outside /Limits [0 0]\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } +} + +func TestStructurePlain_NoOptionalSectionsWhenEmpty(t *testing.T) { + out := structurePlain(&pdfcore.StructureReport{HasStructTree: true}) + for _, absent := range []string{"Other ParentTree keys", "Problems (", "Pages with no"} { + if strings.Contains(out, absent) { + t.Errorf("output has %q with nothing to list:\n%s", absent, out) + } + } +} + +func TestStructurePlain_PageWithOnlyUnmarkedSequencesGetsABlock(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, Streams: []pdfcore.MarkedStream{{Ref: "5 0 R", Opens: 1, Closes: 1}}}, + {PageNum: 2, Streams: []pdfcore.MarkedStream{{Ref: "6 0 R", Closes: 1, MinDepth: -1}}}, + {PageNum: 3, Streams: []pdfcore.MarkedStream{{Ref: "7 0 R"}}}, + }, + } + out := structurePlain(r) + for _, want := range []string{ + "Page 1 /StructParents absent 1 stream BDC/BMC 1, EMC 1\n", + "Page 2 /StructParents absent 1 stream BDC/BMC 0, EMC 1\n", + "no marked content: 3\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } +} + +func TestPageRanges(t *testing.T) { + cases := []struct { + in []int + want string + }{ + {[]int{1}, "1"}, + {[]int{1, 2, 3}, "1-3"}, + {[]int{1, 3, 5}, "1, 3, 5"}, + {[]int{2, 3, 7, 9, 10, 11}, "2-3, 7, 9-11"}, + {nil, ""}, + } + for _, tc := range cases { + if got := pageRanges(tc.in); got != tc.want { + t.Errorf("pageRanges(%v) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestStructureDumpWritesJSONOrPlain(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, RootRef: "12 0 R", + RoleMap: []pdfcore.RoleMapEntry{}, Elements: []*pdfcore.StructKid{}, Pages: []*pdfcore.StructPage{}, + OtherParentTree: []pdfcore.ParentTreeEntry{}, Problems: []pdfcore.StructureProblem{}, + } + var stdout, stderr bytes.Buffer + if code := writeStructureDump(&stdout, &stderr, r, docViewFlags{json: true}); code != 0 { + t.Fatalf("json exit %d, want 0", code) + } + var got pdfcore.StructureReport + if err := json.Unmarshal(stdout.Bytes(), &got); err != nil || !got.HasStructTree || got.RootRef != "12 0 R" { + t.Errorf("json stdout = %q (%v), want the report", stdout.String(), err) + } + if strings.Contains(stdout.String(), "\n ") { + t.Errorf("compact json is indented: %q", stdout.String()) + } + + stdout.Reset() + if code := writeStructureDump(&stdout, &stderr, r, docViewFlags{json: true, pretty: true}); code != 0 { + t.Fatalf("pretty exit %d, want 0", code) + } + if !strings.Contains(stdout.String(), "\n ") { + t.Errorf("--pretty json is not indented: %q", stdout.String()) + } + + stdout.Reset() + if code := writeStructureDump(&stdout, &stderr, r, docViewFlags{}); code != 0 { + t.Fatalf("plain exit %d, want 0", code) + } + if !strings.HasPrefix(stdout.String(), "Structure tree: 12 0 R\n") { + t.Errorf("plain stdout = %q, want the text view", stdout.String()) + } + if stderr.Len() != 0 { + t.Errorf("stderr = %q, want empty", stderr.String()) + } +} + +func TestStructureDumpWriteFailureExits2(t *testing.T) { + for _, jsonMode := range []bool{true, false} { + var stderr bytes.Buffer + code := writeStructureDump(failingWriter{}, &stderr, &pdfcore.StructureReport{}, docViewFlags{json: jsonMode}) + if code != 2 { + t.Errorf("json=%v: exit %d, want 2", jsonMode, code) + } + var obj map[string]string + if err := json.Unmarshal(bytes.TrimSpace(stderr.Bytes()), &obj); err != nil || !strings.HasPrefix(obj["error"], "failed to write output") { + t.Errorf("json=%v: stderr = %q, want one error object", jsonMode, stderr.String()) + } + } +} + +func TestStructureDumpOnFixtures(t *testing.T) { + cases := []struct { + name string + check func(t *testing.T, out string) + }{ + {"structure.pdf", func(t *testing.T, out string) { + if !strings.Contains(out, `ActualText "Hello world"`) || !strings.Contains(out, `Alt "Company logo"`) { + t.Errorf("element text fields missing:\n%s", out) + } + if strings.Contains(out, "* = content count") { + t.Errorf("clean file has an inconsistent row:\n%s", out) + } + }}, + {"structure-broken.pdf", func(t *testing.T, out string) { + if !strings.Contains(out, "already listed; not walked again") || !strings.Contains(out, "* = content count") { + t.Errorf("broken-file findings missing:\n%s", out) + } + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var code int + out := captureStdout(t, func() { code = execStructureDump(fixture(tc.name), docViewFlags{}) }) + if code != 0 { + t.Fatalf("exit %d, want 0", code) + } + tc.check(t, out) + }) + } +} diff --git a/cmd/cli/cmd_structure_test.go b/cmd/cli/cmd_structure_test.go new file mode 100644 index 0000000..d49a187 --- /dev/null +++ b/cmd/cli/cmd_structure_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "strings" + "testing" + + "unidoc-pdf-debugger/internal/pdfcore" +) + +func TestStructurePlain_TreeClaimOnUnmarkedPageIsShown(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, Streams: []pdfcore.MarkedStream{{Ref: "5 0 R"}}, Alignment: []pdfcore.MCIDAlignment{ + {MCID: 0, TreeElements: []string{"10 0 R"}}, + }}, + {PageNum: 2, Streams: []pdfcore.MarkedStream{{Ref: "6 0 R"}}, Alignment: []pdfcore.MCIDAlignment{}}, + }, + } + out := structurePlain(r) + if !strings.Contains(out, "Page 1 ") { + t.Errorf("page with a tree claim but no /StructParents or marked content has no block:\n%s", out) + } + if !strings.Contains(out, "10 0 R *") { + t.Errorf("tree-only alignment row is not shown as inconsistent:\n%s", out) + } + if !strings.Contains(out, "no marked content: 2\n") { + t.Errorf("empty page 2 is not collapsed:\n%s", out) + } +} + +func TestStructurePlain_ParentTreeSlotErrorIsShown(t *testing.T) { + sp := 0 + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, StructParents: &sp, Streams: []pdfcore.MarkedStream{{Ref: "5 0 R"}}, + ParentTree: []pdfcore.ParentTreeSlot{ + {MCID: 0, ElementRef: "10 0 R", S: "P"}, + {MCID: 1, ElementRef: "11 0 R", Err: "11 0 R is an integer, not an element"}, + }, + Alignment: []pdfcore.MCIDAlignment{ + {MCID: 0, ParentTreeElement: "10 0 R", ParentTreeS: "P", TreeElements: []string{}}, + {MCID: 1, TreeElements: []string{}}, + }}, + }, + } + out := structurePlain(r) + if !strings.Contains(out, "ParentTree slot 1: error: 11 0 R is an integer, not an element\n") { + t.Errorf("ParentTree slot error is not shown:\n%s", out) + } + if strings.Contains(out, "ParentTree slot 0:") { + t.Errorf("readable ParentTree slot printed as an error:\n%s", out) + } +} diff --git a/cmd/cli/doc.go b/cmd/cli/doc.go index 20e6097..9a6612f 100644 --- a/cmd/cli/doc.go +++ b/cmd/cli/doc.go @@ -2,8 +2,8 @@ // // The CLI consumes internal/pdfcore directly (zero Wails dependency) and // exposes these dump subcommands: tree, object, stream, page, font, image, -// source, reverserefs, xref, objects, pages, images, bytes, embedded, metadata, -// and signatures, plus the top-level `validate` command (a named subset of +// source, reverserefs, xref, objects, pages, structure, images, bytes, embedded, +// metadata, and signatures, plus the top-level `validate` command (a named subset of // structural PDF/A-1b and PDF/UA-1 checks per profile). // // Output is human-readable PLAIN TEXT on stdout by default; pass --json to emit diff --git a/cmd/cli/main.go b/cmd/cli/main.go index f84455c..7b55454 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -85,6 +85,8 @@ func dispatch(args []string) int { return runObjectsDump(remaining) case "pages": return runPagesDump(remaining) + case "structure": + return runStructureDump(remaining) case "bytes": return runBytesDump(remaining) case "plaintext": @@ -157,6 +159,7 @@ func printUsage(w io.Writer) { _, _ = fmt.Fprintln(w, " pdfdebug dump xref file.pdf") _, _ = fmt.Fprintln(w, " pdfdebug dump objects file.pdf") _, _ = fmt.Fprintln(w, " pdfdebug dump pages file.pdf") + _, _ = fmt.Fprintln(w, " pdfdebug dump structure file.pdf") _, _ = fmt.Fprintln(w, " pdfdebug dump images file.pdf") _, _ = fmt.Fprintln(w, " pdfdebug dump bytes file.pdf") _, _ = fmt.Fprintln(w, " pdfdebug dump embedded file.pdf") diff --git a/cmd/cli/usage.go b/cmd/cli/usage.go index 2cdf1ef..46ea2d2 100644 --- a/cmd/cli/usage.go +++ b/cmd/cli/usage.go @@ -35,6 +35,8 @@ var commandSpecs = []commandSpec{ "Dump the object index (plural: every object)"}, {"dump pages", `[--json] [--pretty] `, "Dump the page index (plural: every page leaf, with MediaBox, inheritance and errors)"}, + {"dump structure", `[--json] [--pretty] `, + "Dump the structure tree (S, page, MCID/MCR/OBJR, Alt, ActualText, Lang) and per-page ParentTree vs content MCIDs"}, {"dump images", `[--json] [--pretty] `, "Dump the image index (plural: every image XObject referenced from page resources, deduplicated; inline BI/ID/EI images are not listed)"}, {"dump bytes", `[--json] [--pretty] `, diff --git a/docs/cli-usage.md b/docs/cli-usage.md index 9a63150..6fc908b 100644 --- a/docs/cli-usage.md +++ b/docs/cli-usage.md @@ -86,6 +86,7 @@ Each description links to the command's section on its reference page. | `dump source` | [The reserialized object source (PDF syntax)](cli/structure.md#dump-source) | `--ref`, `--raw` | | `dump reverserefs` | [Inbound references (who points at this object)](cli/structure.md#dump-reverserefs) | `--ref` | | `dump xref` | [The cross-reference table](cli/structure.md#dump-xref) | - | +| `dump structure` | [The structure tree and per-page ParentTree vs content MCIDs](cli/structure.md#dump-structure) | - | | `dump pages` | [The page index (every page leaf of the page tree, in document order)](cli/pages-and-content.md#dump-pages) | - | | `dump page` | [Assembled per-page render info **(EXPERIMENTAL)**](cli/pages-and-content.md#dump-page) | `--info N`, `--section`, `--forms-recursive` | | `dump stream` | [A decoded content stream (page, object, or XObject)](cli/pages-and-content.md#dump-stream) | `--page`/`--ref`/`--xobject`, `--raw`, `--ops` | @@ -143,7 +144,7 @@ synopsis on its reference page shows which of them it takes. ## Reference pages - [Structure](cli/structure.md): `dump tree`, `dump object`, `dump objects`, - `dump source`, `dump reverserefs` and `dump xref`. + `dump source`, `dump reverserefs`, `dump xref` and `dump structure`. - [Pages and content](cli/pages-and-content.md): `dump pages`, `dump page`, `dump stream` and `dump bytes`. - [Images and fonts](cli/images-and-fonts.md): `dump images`, `dump image` and diff --git a/docs/cli/structure.md b/docs/cli/structure.md index c8fff39..ad9a14d 100644 --- a/docs/cli/structure.md +++ b/docs/cli/structure.md @@ -1,7 +1,8 @@ # Structure commands The commands that walk the object model: the tree, single objects, the object -index, an object's source, inbound references and the cross-reference table. +index, an object's source, inbound references, the cross-reference table and +the tagged-PDF structure tree. Back to the [CLI usage guide](../cli-usage.md). @@ -54,3 +55,52 @@ Lists the objects that point at the given one. `pdfdebug dump xref [--json] [--pretty] ` Prints the cross-reference table. + +### `dump structure` + +`pdfdebug dump structure [--json] [--pretty] ` + +Prints the tagged-PDF structure tree and, for each page, how its marked content +lines up with the tree. It reports what is in the file; it does not judge it. + +The header gives the `/StructTreeRoot` reference, the ParentTree (or `absent`), +`/ParentTreeNextKey`, the catalog's `/MarkInfo /Marked` and `/Lang`, and the +`/RoleMap`. A file with no structure tree prints +`No structure tree (/StructTreeRoot absent)` and still gets the page section. + +The element list follows `/K` from the root, one element per line with its +`/S` (and the role it maps to, in parentheses), reference, page (`(inherited)` +when it comes from an ancestor's `/Pg`), `/T`, `/Lang`, `/Alt`, `/ActualText` +and `/E`. Text longer than 80 runes is cut and marked `[truncated: N of M]`, as +in `dump tree`. Under each element are its kids: `MCID n`, `MCR MCID n page p` +(with `stream N G R` when the content is in another stream), and `OBJR N G R` +with the target's `/Type/Subtype`. Anything that cannot be read is an `error:` +line in its place, and the walk carries on. An element, or an indirect `/K` +array, reached a second time is listed once, with `already listed; not walked again` where it comes back, and a +tree deeper than 256 levels says where it stops. + +Each page with a `/StructParents` entry, any marked content, an MCID the tree +attributes to it, or an error gets a block. The first line has the +`/StructParents` key, the number of content streams and the BDC/BMC and EMC +counts. A stream that fails to decode, ends with sequences still open, or +closes one it never opened gets its own line, and so does each BDC and each +ParentTree slot that could not be read. The table has one row per MCID: + +- `CONTENT` - how many BDCs in the page's content carry that MCID. +- `PARENTTREE` - the element in that slot of the page's ParentTree array. +- `TREE` - every element whose `/K` names that MCID on that page. + +A row is marked `*` unless the content marks the MCID once, the ParentTree slot +names an element, and that element is the only one in `TREE`. The pages that get +no block are collapsed into one line. ParentTree keys +no page names (an annotation's `/StructParent`, or an orphan) are listed under +"Other ParentTree keys", and malformed ParentTree nodes under "Problems". + +MCIDs inside Form XObjects are not aligned: an MCR with `/Stm` is listed in the +tree but left out of the page table, and `Do` is not followed. + +`--json` carries the whole report, uncapped. One alignment row: + +```json +{"mcid":0,"contentCount":2,"parentTreeElement":"13 0 R","parentTreeS":"P","treeElements":["13 0 R"],"consistent":false} +``` diff --git a/internal/pdfcore/markedcontent.go b/internal/pdfcore/markedcontent.go new file mode 100644 index 0000000..a344c73 --- /dev/null +++ b/internal/pdfcore/markedcontent.go @@ -0,0 +1,217 @@ +package pdfcore + +import ( + "fmt" + "strconv" + "strings" + + pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" +) + +// mcOpen is one open marked-content sequence: its MCID, when it has one. +type mcOpen struct { + mcid int + hasMCID bool +} + +// mcState is the stack of open marked-content sequences, carried from one +// content stream of a page to the next. +type mcState struct { + stack []mcOpen +} + +// propertyLookup resolves a named property list (BDC with a name operand) in +// the page's /Resources /Properties and returns its /MCID value, nil when the +// list has none. An error means the name was not found, the entry is not a +// dictionary, or its /MCID does not resolve. +type propertyLookup func(name string) (mcid pdfcpu_types.Object, err error) + +// scanMarkedContent counts BMC/BDC/EMC in one stream's tokens into ms and +// returns a row for every BDC whose property list carries an /MCID. st is +// updated in place so nesting carries into the next stream. +// +// The tokenizer types delimiters and the keywords true, false and null as +// operators, so operands are collected by value: <<, >>, [, ], true, false and +// null are operands, and any other operator ends the operand list. A stray << +// or [ therefore cannot swallow the rest of the stream. +func scanMarkedContent(tokens []Token, streamIdx int, props propertyLookup, st *mcState, ms *MarkedStream) []ContentMCID { + var rows []ContentMCID + ms.MinDepth = len(st.stack) + var operands []Token + for _, tok := range tokens { + if tok.Type == "comment" { + continue + } + if tok.Type != "operator" { + operands = append(operands, tok) + continue + } + switch tok.Value { + case "<<", ">>", "[", "]", "true", "false", "null": + operands = append(operands, tok) + continue + } + switch tok.Value { + case "BMC": + ms.Opens++ + st.stack = append(st.stack, mcOpen{}) + case "BDC": + ms.Opens++ + row, ok := bdcRow(operands, tok, props) + parent := innermostMCID(st.stack) + open := mcOpen{} + if ok { + row.Stream = streamIdx + row.ParentMCID = parent + if row.Err == "" { + open = mcOpen{mcid: row.MCID, hasMCID: true} + } + } + st.stack = append(st.stack, open) + if ok { + row.Depth = len(st.stack) + rows = append(rows, row) + } + case "EMC": + ms.Closes++ + if len(st.stack) == 0 { + ms.MinDepth = min(ms.MinDepth, -1) + } else { + st.stack = st.stack[:len(st.stack)-1] + ms.MinDepth = min(ms.MinDepth, len(st.stack)) + } + } + operands = operands[:0] + } + ms.DepthAtEnd = len(st.stack) + return rows +} + +// innermostMCID returns the MCID of the innermost open sequence that has +// one, or nil. +func innermostMCID(stack []mcOpen) *int { + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].hasMCID { + v := stack[i].mcid + return &v + } + } + return nil +} + +// bdcRow reads a BDC's operands (tag, then an inline dictionary or a property +// list name). ok is false when the property list has no /MCID, so the BDC +// gets no row. Operands that are not a tag name followed by a dictionary or a +// name, a property list that cannot be read, or an /MCID that is not a +// non-negative integer give a row with MCID -1 and Err set. +func bdcRow(operands []Token, op Token, props propertyLookup) (row ContentMCID, ok bool) { + pos := op + if len(operands) > 0 { + pos = operands[0] + } + row = ContentMCID{MCID: -1, Line: pos.Line, Col: pos.Col} + if len(operands) > 0 && operands[0].Type == "name" { + row.Tag = strings.TrimPrefix(operands[0].Value, "/") + } + if len(operands) < 2 { + row.Err = fmt.Sprintf("BDC has %d of its 2 operands (a tag and a property list)", len(operands)) + return row, true + } + if operands[0].Type != "name" { + row.Err = fmt.Sprintf("BDC operands start with %q, not a tag name; property list not read", operands[0].Value) + return row, true + } + prop := operands[1] + switch { + case prop.Type == "operator" && prop.Value == "<<": + val, found := inlineMCID(operands[1:]) + if !found { + return row, false + } + if val.Type == "number" { + if n, err := strconv.Atoi(val.Value); err == nil { + setMCID(&row, n) + return row, true + } + } + row.Err = fmt.Sprintf("/MCID %s is not an integer", val.Value) + return row, true + case prop.Type == "name": + name := strings.TrimPrefix(prop.Value, "/") + mcid, err := props(name) + if err != nil { + row.Err = err.Error() + return row, true + } + if mcid == nil { + return row, false + } + if n, isInt := mcid.(pdfcpu_types.Integer); isInt { + setMCID(&row, int(n)) + } else { + row.Err = fmt.Sprintf("/MCID in property list /%s is %s, not an integer", name, objKind(mcid)) + } + return row, true + } + row.Err = fmt.Sprintf("BDC property list operand %q is not a dictionary or a name", prop.Value) + return row, true +} + +// setMCID stores a non-negative n as the row's MCID, or an error for a +// negative one. +func setMCID(row *ContentMCID, n int) { + if n < 0 { + row.Err = fmt.Sprintf("/MCID %d is negative", n) + return + } + row.MCID = n +} + +// inlineMCID finds the value token of the top-level /MCID key in an inline +// dictionary's tokens, which start at its "<<". A container value yields its +// opening token. +func inlineMCID(toks []Token) (Token, bool) { + depth := 0 + expectKey := true + key := "" + for _, t := range toks { + isOp := t.Type == "operator" + opening := isOp && (t.Value == "<<" || t.Value == "[") + closing := isOp && (t.Value == ">>" || t.Value == "]") + switch depth { + case 0: + if opening { + depth = 1 + } + case 1: + switch { + case closing: + return Token{}, false + case expectKey: + if t.Type == "name" { + key = t.Value + expectKey = false + } + default: + if key == "/MCID" { + return t, true + } + if opening { + depth++ + } else { + expectKey = true + } + } + default: + if opening { + depth++ + } else if closing { + depth-- + if depth == 1 { + expectKey = true + } + } + } + } + return Token{}, false +} diff --git a/internal/pdfcore/markedcontent_test.go b/internal/pdfcore/markedcontent_test.go new file mode 100644 index 0000000..059b167 --- /dev/null +++ b/internal/pdfcore/markedcontent_test.go @@ -0,0 +1,243 @@ +package pdfcore + +import ( + "strings" + "testing" +) + +// Marked-content scanning, read through GetStructure on one-page documents +// with no structure tree, so only the content side of the report is filled. + +// scanPage returns page 1 of the structure report for a markedPDF document. +func scanPage(t *testing.T, resources string, streams []string, extra ...rawObj) *StructPage { + t.Helper() + r := rawStructure(t, markedPDF(resources, streams, extra...)) + if r.HasStructTree { + t.Error("HasStructTree = true for a document with no /StructTreeRoot") + } + if len(r.Pages) != 1 { + t.Fatalf("got %d pages, want 1", len(r.Pages)) + } + p := r.Pages[0] + if len(p.Streams) != len(streams) { + t.Fatalf("got %d stream rows, want %d", len(p.Streams), len(streams)) + } + return p +} + +// mcidRow is the part of a ContentMCID the scan tests compare. +type mcidRow struct { + mcid int + tag string + stream int + depth int + parent int // -1 for no enclosing MCID + err string +} + +func checkContentMCIDs(t *testing.T, p *StructPage, want []mcidRow) { + t.Helper() + if len(p.ContentMCIDs) != len(want) { + t.Fatalf("ContentMCIDs = %+v, want %d rows", p.ContentMCIDs, len(want)) + } + for i, w := range want { + c := p.ContentMCIDs[i] + parent := -1 + if c.ParentMCID != nil { + parent = *c.ParentMCID + } + if c.MCID != w.mcid || c.Stream != w.stream || c.Depth != w.depth || parent != w.parent { + t.Errorf("row %d = MCID %d stream %d depth %d parent %d, want MCID %d stream %d depth %d parent %d", + i, c.MCID, c.Stream, c.Depth, parent, w.mcid, w.stream, w.depth, w.parent) + } + if w.tag != "" && c.Tag != w.tag { + t.Errorf("row %d tag = %q, want %q", i, c.Tag, w.tag) + } + switch { + case w.err == "" && c.Err != "": + t.Errorf("row %d: unexpected error %q", i, c.Err) + case w.err != "" && !strings.Contains(c.Err, w.err): + t.Errorf("row %d: error %q, want it to contain %q", i, c.Err, w.err) + } + } +} + +func checkStream(t *testing.T, s MarkedStream, opens, closes, depthAtEnd, minDepth int) { + t.Helper() + if s.Opens != opens || s.Closes != closes || s.DepthAtEnd != depthAtEnd || s.MinDepth != minDepth { + t.Errorf("stream = opens %d closes %d depthAtEnd %d minDepth %d, want %d %d %d %d", + s.Opens, s.Closes, s.DepthAtEnd, s.MinDepth, opens, closes, depthAtEnd, minDepth) + } +} + +func TestMarkedContentScan_PropertyLists(t *testing.T) { + cases := []struct { + name string + content string + want []mcidRow + }{ + {"inline dict with spaces", "/P << /MCID 0 >> BDC (x) Tj EMC", + []mcidRow{{0, "P", 0, 1, -1, ""}}}, + {"inline dict without spaces", "/P<>BDC EMC", + []mcidRow{{0, "P", 0, 1, -1, ""}}}, + {"booleans, null and an array in the property list", "/Span <> BDC EMC", + []mcidRow{{2, "Span", 0, 1, -1, ""}}}, + {"nested dict in the property list", "/Span <> /MCID 4>> BDC EMC", + []mcidRow{{4, "Span", 0, 1, -1, ""}}}, + {"property list with no /MCID", "/Artifact <> BDC EMC", + nil}, + {"/MCID not an integer", "/P <> BDC EMC", + []mcidRow{{-1, "P", 0, 1, -1, "MCID"}}}, + {"negative /MCID", "/P <> BDC EMC", + []mcidRow{{-1, "P", 0, 1, -1, "MCID"}}}, + {"comment ignored", "% /P <> BDC\n/P <> BDC EMC", + []mcidRow{{0, "P", 0, 1, -1, ""}}}, + {"BDC inside an inline image payload", "BI /W 1 /H 1 /CS /G /BPC 8 ID /P <> BDC\nEI /P <> BDC EMC", + []mcidRow{{0, "P", 0, 1, -1, ""}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := scanPage(t, "", []string{rawStream(tc.content)}) + checkContentMCIDs(t, p, tc.want) + checkStream(t, p.Streams[0], 1, 1, 0, 0) + }) + } +} + +func TestMarkedContentScan_NamedPropertyLists(t *testing.T) { + t.Run("found in /Resources /Properties", func(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 3 >> >> >>", []string{rawStream("/Span /MC0 BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{3, "Span", 0, 1, -1, ""}}) + }) + t.Run("found through indirect /Properties and entry", func(t *testing.T) { + p := scanPage(t, "<< /Properties 6 0 R >>", []string{rawStream("/Span /MC0 BDC EMC")}, + rawObj{6, "<< /MC0 7 0 R >>"}, rawObj{7, "<< /MCID 3 >>"}) + checkContentMCIDs(t, p, []mcidRow{{3, "Span", 0, 1, -1, ""}}) + }) + t.Run("missing", func(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 3 >> >> >>", []string{rawStream("/Span /MC9 BDC EMC")}) + if len(p.ContentMCIDs) != 1 || !strings.Contains(p.ContentMCIDs[0].Err, "MC9") { + t.Errorf("ContentMCIDs = %+v, want one row whose error names MC9", p.ContentMCIDs) + } + checkStream(t, p.Streams[0], 1, 1, 0, 0) + }) +} + +func TestMarkedContentScan_Nesting(t *testing.T) { + t.Run("BMC around an MCID sequence", func(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/Artifact BMC /P <> BDC EMC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 2, -1, ""}}) + checkStream(t, p.Streams[0], 2, 2, 0, 0) + }) + t.Run("MCID inside MCID", func(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/P <> BDC /Span <> BDC EMC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}, {1, "Span", 0, 2, 0, ""}}) + }) + t.Run("innermost enclosing MCID through a BMC", func(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/P <> BDC /X BMC /Span <> BDC EMC EMC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}, {1, "Span", 0, 3, 0, ""}}) + checkStream(t, p.Streams[0], 3, 3, 0, 0) + }) + t.Run("EMC with nothing open", func(t *testing.T) { + p := scanPage(t, "", []string{rawStream("EMC /P <> BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}}) + checkStream(t, p.Streams[0], 1, 2, 0, -1) + }) + t.Run("BDC with no EMC", func(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/P <> BDC (x) Tj")}) + checkStream(t, p.Streams[0], 1, 0, 1, 0) + }) +} + +func TestMarkedContentScan_StackCarriedAcrossStreams(t *testing.T) { + p := scanPage(t, "", []string{ + rawStream("/P <> BDC (x) Tj"), + rawStream("/Span <> BDC EMC EMC"), + }) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}, {1, "Span", 1, 2, 0, ""}}) + checkStream(t, p.Streams[0], 1, 0, 1, 0) + checkStream(t, p.Streams[1], 1, 2, 0, 0) +} + +func TestMarkedContentScan_UndecodableStreamIsSkipped(t *testing.T) { + p := scanPage(t, "", []string{ + rawStream("/P <> BDC"), + undecodableStream, + rawStream("EMC /Span <> BDC EMC"), + }) + if p.Streams[1].Err == "" { + t.Errorf("undecodable stream row = %+v, want an error", p.Streams[1]) + } + if p.Streams[1].Opens != 0 || p.Streams[1].Closes != 0 { + t.Errorf("undecodable stream row = %+v, want nothing counted", p.Streams[1]) + } + if p.Streams[0].Err != "" || p.Streams[2].Err != "" { + t.Errorf("decodable streams carry errors: %+v, %+v", p.Streams[0], p.Streams[2]) + } + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}, {1, "Span", 2, 1, -1, ""}}) + checkStream(t, p.Streams[2], 1, 2, 0, 0) +} + +func TestMarkedContentScan_StreamRefsInContentsOrder(t *testing.T) { + p := scanPage(t, "", []string{rawStream("q Q"), rawStream("q Q")}) + if p.Streams[0].Ref != "20 0 R" || p.Streams[1].Ref != "21 0 R" { + t.Errorf("stream refs = %q, %q, want 20 0 R, 21 0 R", p.Streams[0].Ref, p.Streams[1].Ref) + } + if len(p.ContentMCIDs) != 0 || len(p.Alignment) != 0 { + t.Errorf("page with no marked content has ContentMCIDs %+v, Alignment %+v", p.ContentMCIDs, p.Alignment) + } +} + +func TestMarkedContentScan_AlignmentWithoutStructureTree(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/P <> BDC EMC /P <> BDC EMC /P <> BDC EMC")}) + if p.StructParents != nil || len(p.ParentTree) != 0 || p.ParentTreeErr != "" { + t.Errorf("page with no /StructParents: StructParents %v ParentTree %+v ParentTreeErr %q", p.StructParents, p.ParentTree, p.ParentTreeErr) + } + if got := alignmentMCIDs(p); len(got) != 2 || got[0] != 0 || got[1] != 1 { + t.Fatalf("alignment MCIDs = %v, want [0 1] ascending", got) + } + if a := p.Alignment[1]; a.ContentCount != 2 || a.ParentTreeElement != "" || len(a.TreeElements) != 0 || a.Consistent { + t.Errorf("MCID 1 row = %+v, want content 2, no ParentTree element, no tree claimant, not consistent", a) + } +} + +func TestMarkedContentScan_StrayOpenSpanEndsAtNextOperator(t *testing.T) { + p := scanPage(t, "", []string{rawStream("[ (x) Tj /P <> BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}}) + checkStream(t, p.Streams[0], 1, 1, 0, 0) +} + +func TestMarkedContentScan_BDCWithoutTagIsAnErrorRow(t *testing.T) { + p := scanPage(t, "<< /Properties << /MCID << /MCID 5 >> >> >>", []string{rawStream("<> BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{-1, "", 0, 1, -1, "not a tag name"}}) + checkStream(t, p.Streams[0], 1, 1, 0, 0) +} + +func TestMarkedContentScan_DanglingMCIDInNamedPropertyListIsAnErrorRow(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 99 0 R >> >> >>", []string{rawStream("/Span /MC0 BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{-1, "Span", 0, 1, -1, "does not resolve"}}) +} + +func TestMarkedContentScan_BDCWithUnreadableOperandsIsAnErrorRow(t *testing.T) { + cases := []struct { + name string + content string + err string + }{ + {"tag only", "/P BDC EMC", "1 of its 2 operands"}, + {"no operands", "BDC EMC", "0 of its 2 operands"}, + {"number as property list", "/P 5 BDC EMC", "not a dictionary or a name"}, + {"array as property list", "/P [1] BDC EMC", "not a dictionary or a name"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := scanPage(t, "", []string{rawStream(tc.content)}) + tag := "" + if strings.HasPrefix(tc.content, "/P") { + tag = "P" + } + checkContentMCIDs(t, p, []mcidRow{{-1, tag, 0, 1, -1, tc.err}}) + checkStream(t, p.Streams[0], 1, 1, 0, 0) + }) + } +} diff --git a/internal/pdfcore/markedcontent_tokens_test.go b/internal/pdfcore/markedcontent_tokens_test.go new file mode 100644 index 0000000..1638b39 --- /dev/null +++ b/internal/pdfcore/markedcontent_tokens_test.go @@ -0,0 +1,250 @@ +package pdfcore + +import ( + "errors" + "strings" + "testing" + + pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" +) + +// scanMarkedContent driven directly on tokenizer output, without a document. + +// noProps is a property lookup for streams that name no property list. +func noProps(t *testing.T) propertyLookup { + return func(name string) (pdfcpu_types.Object, error) { + t.Errorf("unexpected property list lookup for /%s", name) + return nil, nil + } +} + +// scanTokens scans each content in order with one carried stack and returns +// every row and the per-stream counters. +func scanTokens(props propertyLookup, contents ...string) ([]ContentMCID, []MarkedStream) { + st := &mcState{} + var rows []ContentMCID + streams := make([]MarkedStream, len(contents)) + for i, c := range contents { + rows = append(rows, scanMarkedContent(tokenizeContentStream(c), i, props, st, &streams[i])...) + } + return rows, streams +} + +func rowParent(c ContentMCID) int { + if c.ParentMCID == nil { + return -1 + } + return *c.ParentMCID +} + +func TestScanMarkedContentTokens_InlinePropertyLists(t *testing.T) { + cases := []struct { + name string + content string + mcid int + tag string + }{ + {"no spaces", "/P<>BDC EMC", 0, "P"}, + {"spaces", "/P << /MCID 7 >> BDC EMC", 7, "P"}, + {"booleans, null and an array before the MCID", "/Span <> BDC EMC", 2, "Span"}, + {"nested dict before the MCID", "/P <> >> /MCID 3>> BDC EMC", 3, "P"}, + {"array of dicts before the MCID", "/P <> <>] /MCID 4>> BDC EMC", 4, "P"}, + {"comment inside the operands", "/P % note\n<> BDC EMC", 5, "P"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, streams := scanTokens(noProps(t), tc.content) + if len(rows) != 1 { + t.Fatalf("rows = %+v, want one", rows) + } + r := rows[0] + if r.MCID != tc.mcid || r.Tag != tc.tag || r.Err != "" || r.Depth != 1 || r.ParentMCID != nil { + t.Errorf("row = %+v, want MCID %d tag %s depth 1, no parent, no error", r, tc.mcid, tc.tag) + } + checkStream(t, streams[0], 1, 1, 0, 0) + }) + } +} + +func TestScanMarkedContentTokens_InlineDictWithoutTopLevelMCIDHasNoRow(t *testing.T) { + cases := []struct { + name string + content string + depth int + }{ + {"no MCID key", "/P <> BDC EMC", 0}, + {"MCID only inside a nested dict", "/P <> >> BDC EMC", 0}, + {"empty dict", "/P <<>> BDC EMC", 0}, + {"dict never closed", "/P << /X 1 BDC", 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, streams := scanTokens(noProps(t), tc.content) + if len(rows) != 0 { + t.Errorf("rows = %+v, want none", rows) + } + if streams[0].Opens != 1 || streams[0].DepthAtEnd != tc.depth { + t.Errorf("stream = %+v, want 1 open and depth %d at end", streams[0], tc.depth) + } + }) + } +} + +func TestScanMarkedContentTokens_BadInlineMCIDIsAnErrorRow(t *testing.T) { + cases := []struct { + name string + content string + err string + }{ + {"name", "/P <> BDC EMC", "/MCID /X is not an integer"}, + {"real", "/P <> BDC EMC", "/MCID 1.5 is not an integer"}, + {"array", "/P <> BDC EMC", "/MCID [ is not an integer"}, + {"negative", "/P <> BDC EMC", "/MCID -2 is negative"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, _ := scanTokens(noProps(t), tc.content) + if len(rows) != 1 || rows[0].MCID != -1 || rows[0].Err != tc.err { + t.Errorf("rows = %+v, want one row with MCID -1 and error %q", rows, tc.err) + } + }) + } +} + +func TestScanMarkedContentTokens_NamedPropertyLists(t *testing.T) { + lists := map[string]pdfcpu_types.Object{ + "Int": pdfcpu_types.Integer(4), + "Neg": pdfcpu_types.Integer(-1), + "Real": pdfcpu_types.Float(2.5), + "Str": pdfcpu_types.StringLiteral("x"), + "None": nil, + } + var asked []string + props := func(name string) (pdfcpu_types.Object, error) { + asked = append(asked, name) + v, ok := lists[name] + if !ok { + return nil, errors.New("property list /" + name + " not found") + } + return v, nil + } + cases := []struct { + name string + mcid int + err string + row bool + }{ + {"Int", 4, "", true}, + {"Neg", -1, "/MCID -1 is negative", true}, + {"Real", -1, "/MCID in property list /Real is a real, not an integer", true}, + {"Str", -1, "/MCID in property list /Str is a string, not an integer", true}, + {"None", 0, "", false}, + {"Missing", -1, "property list /Missing not found", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + asked = nil + rows, streams := scanTokens(props, "/Span /"+tc.name+" BDC EMC") + if len(asked) != 1 || asked[0] != tc.name { + t.Errorf("lookups = %v, want [%s] (leading slash stripped)", asked, tc.name) + } + checkStream(t, streams[0], 1, 1, 0, 0) + if !tc.row { + if len(rows) != 0 { + t.Errorf("rows = %+v, want none for a property list with no /MCID", rows) + } + return + } + if len(rows) != 1 { + t.Fatalf("rows = %+v, want one", rows) + } + if rows[0].MCID != tc.mcid || rows[0].Err != tc.err || rows[0].Tag != "Span" { + t.Errorf("row = %+v, want MCID %d tag Span error %q", rows[0], tc.mcid, tc.err) + } + }) + } +} + +func TestScanMarkedContentTokens_Nesting(t *testing.T) { + cases := []struct { + name string + content string + depths []int + parents []int + }{ + {"MCID inside BMC has no parent MCID", "/A BMC /P <> BDC EMC EMC", []int{2}, []int{-1}}, + {"MCID inside MCID", "/P <> BDC /Span <> BDC EMC EMC", []int{1, 2}, []int{-1, 0}}, + {"innermost MCID wins over an outer one", "/P <> BDC /Q <> BDC /A BMC /S <> BDC EMC EMC EMC EMC", []int{1, 2, 4}, []int{-1, 0, 1}}, + {"error row is no parent", "/P <> BDC /Span <> BDC EMC EMC", []int{1, 2}, []int{-1, -1}}, + {"BDC with no MCID is no parent", "/P <> BDC /Span <> BDC EMC EMC", []int{2}, []int{-1}}, + {"sibling after close has no parent", "/P <> BDC EMC /P <> BDC EMC", []int{1, 1}, []int{-1, -1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, streams := scanTokens(noProps(t), tc.content) + if len(rows) != len(tc.depths) { + t.Fatalf("rows = %+v, want %d", rows, len(tc.depths)) + } + for i, r := range rows { + if r.Depth != tc.depths[i] || rowParent(r) != tc.parents[i] { + t.Errorf("row %d = depth %d parent %d, want depth %d parent %d", i, r.Depth, rowParent(r), tc.depths[i], tc.parents[i]) + } + } + if streams[0].DepthAtEnd != 0 || streams[0].MinDepth != 0 || streams[0].Opens != streams[0].Closes { + t.Errorf("stream = %+v, want balanced", streams[0]) + } + }) + } +} + +func TestScanMarkedContentTokens_EMCOnEmptyStack(t *testing.T) { + rows, streams := scanTokens(noProps(t), "EMC EMC /P <> BDC EMC") + checkStream(t, streams[0], 1, 3, 0, -1) + if len(rows) != 1 || rows[0].Depth != 1 { + t.Errorf("rows = %+v, want MCID 0 at depth 1 (an unmatched EMC pops nothing)", rows) + } +} + +func TestScanMarkedContentTokens_StackCarriedAcrossStreams(t *testing.T) { + rows, streams := scanTokens(noProps(t), + "/P <> BDC", + "/Span <> BDC EMC", + "EMC", + ) + checkStream(t, streams[0], 1, 0, 1, 0) + checkStream(t, streams[1], 1, 1, 1, 1) + checkStream(t, streams[2], 0, 1, 0, 0) + if len(rows) != 2 { + t.Fatalf("rows = %+v, want two", rows) + } + if rows[1].Stream != 1 || rows[1].Depth != 2 || rowParent(rows[1]) != 0 { + t.Errorf("second row = %+v, want stream 1, depth 2, parent MCID 0 from the earlier stream", rows[1]) + } +} + +func TestScanMarkedContentTokens_InlineImagePayloadIsNotScanned(t *testing.T) { + rows, streams := scanTokens(noProps(t), "BI /W 1 /H 1 /BPC 8 /CS /G ID /P <> BDC EMC EI /Q <> BDC EMC") + if len(rows) != 1 || rows[0].MCID != 1 { + t.Errorf("rows = %+v, want only MCID 1 from outside the image data", rows) + } + checkStream(t, streams[0], 1, 1, 0, 0) +} + +func TestScanMarkedContentTokens_RowPositionIsTheFirstOperand(t *testing.T) { + rows, _ := scanTokens(noProps(t), "q\n /P <> BDC EMC Q") + if len(rows) != 1 || rows[0].Line != 2 || rows[0].Col != 3 { + t.Errorf("rows = %+v, want line 2 col 3 (the tag)", rows) + } + rows, _ = scanTokens(noProps(t), " BDC") + if len(rows) != 1 || rows[0].Line != 1 || rows[0].Col != 3 || !strings.Contains(rows[0].Err, "0 of its 2 operands") { + t.Errorf("rows = %+v, want the BDC position when it has no operands", rows) + } +} + +func TestScanMarkedContentTokens_OtherOperatorsResetOperands(t *testing.T) { + rows, streams := scanTokens(noProps(t), "/P <> Tj BDC EMC") + if len(rows) != 1 || !strings.Contains(rows[0].Err, "0 of its 2 operands") { + t.Errorf("rows = %+v, want an operand-count error row after Tj cleared the operands", rows) + } + checkStream(t, streams[0], 1, 1, 0, 0) +} diff --git a/internal/pdfcore/model.go b/internal/pdfcore/model.go index 8060ef7..56b4355 100644 --- a/internal/pdfcore/model.go +++ b/internal/pdfcore/model.go @@ -763,3 +763,152 @@ type DocumentInfo struct { FileSize int64 `json:"fileSize"` Error string `json:"error"` } + +// StructureReport is the tagged-structure dump returned by +// Inspector.GetStructure: the structure tree from /StructTreeRoot, and per +// numbered page the ParentTree array next to the MCIDs found in the page's +// content streams. Anything malformed is reported on the row it belongs to; +// Problems holds findings with no row of their own. Refs are "N G R" strings. +type StructureReport struct { + HasStructTree bool `json:"hasStructTree"` + RootRef string `json:"rootRef"` // "" when direct or absent + HasParentTree bool `json:"hasParentTree"` // root /ParentTree present + ParentTreeRef string `json:"parentTreeRef"` // "" when direct or absent + Marked *bool `json:"marked"` // catalog /MarkInfo /Marked; nil when absent + DocLang string `json:"docLang"` // catalog /Lang, decoded + ParentTreeNextKey *int `json:"parentTreeNextKey"` // nil when absent + RoleMap []RoleMapEntry `json:"roleMap"` // in key order + Elements []*StructKid `json:"elements"` // the root's /K, in order + Pages []*StructPage `json:"pages"` // every numbered page + OtherParentTree []ParentTreeEntry `json:"otherParentTree"` // ParentTree keys no page's /StructParents names + Problems []StructureProblem `json:"problems"` // findings with no row of their own + Counts StructureCounts `json:"counts"` +} + +// RoleMapEntry is one /RoleMap mapping from a structure type to another. +type RoleMapEntry struct { + From string `json:"from"` + To string `json:"to"` +} + +// StructKid is one /K entry. Kind is "element", "mcid", "mcr", "objr" or +// "error"; an error kid carries only Err. +type StructKid struct { + Kind string `json:"kind"` + Element *StructElem `json:"element,omitempty"` // kind "element" + MCID *int `json:"mcid,omitempty"` // kinds "mcid", "mcr" + Page int `json:"page"` // effective page number; 0 when unknown + PageInherited bool `json:"pageInherited"` // page not from the kid's own /Pg + StmRef string `json:"stmRef,omitempty"` // kind "mcr" with /Stm + ObjRef string `json:"objRef,omitempty"` // kind "objr": /Obj + ObjType string `json:"objType,omitempty"` // kind "objr": "Annot/Link", "XObject/Form", ... + Err string `json:"error,omitempty"` +} + +// StructElem is one structure element. Text fields are decoded text strings; +// Alt, ActualText and E are nil when absent, so an empty value is visible. +type StructElem struct { + Ref string `json:"ref"` // "" for a direct dictionary + NodeID string `json:"nodeId"` + S string `json:"s"` // /S as written; "" when absent or not a name + MappedS string `json:"mappedS"` // /S followed through /RoleMap; "" when not mapped + PgRef string `json:"pgRef"` // own /Pg ref; "" when absent + Page int `json:"page"` // own /Pg, else nearest ancestor's; 0 when none or not a page + PageInherited bool `json:"pageInherited"` + Title string `json:"title,omitempty"` // /T + Lang string `json:"lang,omitempty"` + Alt *string `json:"alt,omitempty"` + ActualText *string `json:"actualText,omitempty"` + E *string `json:"e,omitempty"` + Kids []*StructKid `json:"kids"` + Err string `json:"error,omitempty"` +} + +// StructPage is one numbered page of a StructureReport: its ParentTree array +// (through /StructParents), its content streams' marked-content counts, every +// MCID marked in them, and the per-MCID alignment of those three sources. +type StructPage struct { + PageNum int `json:"pageNum"` + Ref string `json:"ref"` + StructParents *int `json:"structParents"` // nil when absent + ParentTree []ParentTreeSlot `json:"parentTree"` // the array at that key, index = MCID + ParentTreeErr string `json:"parentTreeError,omitempty"` // key missing, value not an array, ... + Streams []MarkedStream `json:"streams"` // one per /Contents stream, in order + ContentMCIDs []ContentMCID `json:"contentMcids"` // every BDC carrying an MCID, in content order + Alignment []MCIDAlignment `json:"alignment"` // ascending MCID + Err string `json:"error,omitempty"` // page unreadable, /Contents malformed +} + +// ParentTreeSlot is one entry of a page's ParentTree array: the element that +// owns the content item with that MCID. ElementRef is "" for a null slot. +type ParentTreeSlot struct { + MCID int `json:"mcid"` + ElementRef string `json:"elementRef"` + S string `json:"s"` + Err string `json:"error,omitempty"` +} + +// MarkedStream is the marked-content balance of one content stream. Depth is +// the number of open marked-content sequences, carried from one stream of a +// page to the next. +type MarkedStream struct { + Ref string `json:"ref"` + Opens int `json:"opens"` // BMC + BDC + Closes int `json:"closes"` // EMC + DepthAtEnd int `json:"depthAtEnd"` // depth after this stream + MinDepth int `json:"minDepth"` // lowest depth reached; negative when an EMC had nothing to close + Err string `json:"error,omitempty"` // decode failure: the stream is not scanned +} + +// ContentMCID is one BDC in a page's content whose property list carries an +// /MCID, or whose operands or property list could not be read. MCID is -1 +// when Err is set. +type ContentMCID struct { + MCID int `json:"mcid"` + Tag string `json:"tag"` // the BDC tag name, e.g. "P" + Stream int `json:"stream"` // index into StructPage.Streams + Line int `json:"line"` + Col int `json:"col"` + Depth int `json:"depth"` // marked-content depth of this BDC, 1 = outermost + ParentMCID *int `json:"parentMcid"` // innermost enclosing open sequence with an MCID; nil when none + Err string `json:"error,omitempty"` // property-list name not found, /MCID not an integer, ... +} + +// MCIDAlignment lines up one MCID of a page across the content streams, the +// page's ParentTree array and the structure tree. Consistent is true exactly +// when the content marks it once, the ParentTree slot names an element, and +// that element is the only tree element claiming it. +type MCIDAlignment struct { + MCID int `json:"mcid"` + ContentCount int `json:"contentCount"` + ParentTreeElement string `json:"parentTreeElement"` // "" for no slot, a null slot or a slot with an error + ParentTreeS string `json:"parentTreeS"` + TreeElements []string `json:"treeElements"` // refs of elements whose /K names this MCID on this page; "(direct)" for a direct element + Consistent bool `json:"consistent"` +} + +// ParentTreeEntry is a ParentTree key that no page's /StructParents names, +// typically an annotation's or XObject's /StructParent. +type ParentTreeEntry struct { + Key int `json:"key"` + ElementRef string `json:"elementRef"` + S string `json:"s"` + Err string `json:"error,omitempty"` +} + +// StructureProblem is a finding with no row of its own: Where names the +// object or number-tree node, Message what is wrong with it. +type StructureProblem struct { + Where string `json:"where"` + Message string `json:"message"` +} + +// StructureCounts totals the structure tree and the content MCIDs. +type StructureCounts struct { + Elements int `json:"elements"` + MCIDKids int `json:"mcidKids"` + MCRKids int `json:"mcrKids"` + OBJRKids int `json:"objrKids"` + ErrorKids int `json:"errorKids"` + ContentMCIDs int `json:"contentMcids"` +} diff --git a/internal/pdfcore/numbertree.go b/internal/pdfcore/numbertree.go new file mode 100644 index 0000000..1b84368 --- /dev/null +++ b/internal/pdfcore/numbertree.go @@ -0,0 +1,202 @@ +package pdfcore + +import ( + "fmt" + "sort" + "strings" + + pdfcpu_model "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" + pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" +) + +// maxNumberTreeDepth caps how many nodes one descent path of a number tree may +// hold. A deeper node is reported and not walked. +const maxNumberTreeDepth = 64 + +// numberTreeWalker carries the state of one number-tree walk. +type numberTreeWalker struct { + ctx *pdfcpu_model.Context + where string + entries map[int]pdfcpu_types.Object + keyNode map[int]string // key -> label of the node that first held it + visited map[int]bool // object numbers of nodes already walked + problems []StructureProblem +} + +// walkNumberTree reads the number tree (ISO 32000-1 7.9.7) rooted at root and +// returns every key with its value as stored (possibly an indirect reference; +// callers dereference), the keys in ascending order, and every malformed node +// it met. A malformed node is reported and the walk goes on: a node that is +// not a dictionary, has neither /Kids nor /Nums, an odd-length /Nums, a +// non-integer key, a bad /Limits or a key outside it, keys out of order +// within one leaf, a +// duplicate key (the first value is kept), a node reached twice, and a node +// deeper than maxNumberTreeDepth. where prefixes every problem location. +// Callers hold pdfMu. +func walkNumberTree(ctx *pdfcpu_model.Context, root pdfcpu_types.Object, where string) (entries map[int]pdfcpu_types.Object, order []int, problems []StructureProblem) { + w := &numberTreeWalker{ + ctx: ctx, + where: where, + entries: map[int]pdfcpu_types.Object{}, + keyNode: map[int]string{}, + visited: map[int]bool{}, + } + w.visit(root, true, 1) + order = make([]int, 0, len(w.entries)) + for k := range w.entries { + order = append(order, k) + } + sort.Ints(order) + return w.entries, order, w.problems +} + +// ntNodeLabel names a node for problem locations: its reference, or "(direct)". +func ntNodeLabel(o pdfcpu_types.Object) string { + if ref, ok := o.(pdfcpu_types.IndirectRef); ok { + return refString(ref) + } + return "(direct)" +} + +// problem records msg against the node labelled label. +func (w *numberTreeWalker) problem(label, msg string) { + w.problems = append(w.problems, StructureProblem{Where: w.where + ", node " + label, Message: msg}) +} + +// visit reads one node and its descendants. +func (w *numberTreeWalker) visit(node pdfcpu_types.Object, isRoot bool, depth int) { + label := ntNodeLabel(node) + if depth > maxNumberTreeDepth { + w.problem(label, fmt.Sprintf("number tree deeper than %d levels; node not walked", maxNumberTreeDepth)) + return + } + if ref, ok := node.(pdfcpu_types.IndirectRef); ok { + n := ref.ObjectNumber.Value() + if w.visited[n] { + w.problem(label, "node reached twice; not walked again") + return + } + w.visited[n] = true + } + guard(func() { w.read(node, label, isRoot, depth) }, func(msg string) { + w.problem(label, "node could not be read: "+msg) + }) +} + +// read classifies one node and walks its /Kids or /Nums. +func (w *numberTreeWalker) read(node pdfcpu_types.Object, label string, isRoot bool, depth int) { + obj := derefIn(w.ctx, node) + if obj == nil { + if _, ok := node.(pdfcpu_types.IndirectRef); ok { + w.problem(label, "node does not resolve to an object") + } else { + w.problem(label, "node is null") + } + return + } + d, ok := obj.(pdfcpu_types.Dict) + if !ok { + w.problem(label, fmt.Sprintf("node is %s, not a dictionary", objKind(obj))) + return + } + + lo, hi, hasLimits := w.limits(d, label, isRoot) + + kidsObj, hasKids := d.Find("Kids") + numsObj, hasNums := d.Find("Nums") + if !hasKids && !hasNums { + w.problem(label, "node has neither /Kids nor /Nums") + return + } + if hasKids { + kids, ok := derefIn(w.ctx, kidsObj).(pdfcpu_types.Array) + if !ok { + w.problem(label, fmt.Sprintf("/Kids is %s, not an array", objKind(derefIn(w.ctx, kidsObj)))) + } else { + for _, kid := range kids { + w.visit(kid, false, depth+1) + } + } + } + if hasNums { + nums, ok := derefIn(w.ctx, numsObj).(pdfcpu_types.Array) + if !ok { + w.problem(label, fmt.Sprintf("/Nums is %s, not an array", objKind(derefIn(w.ctx, numsObj)))) + return + } + w.readNums(nums, label, lo, hi, hasLimits) + } +} + +// limits reads a node's /Limits. On a non-root node it must be two ascending +// integers; on the root it is ignored. ok is false when the node has no usable +// /Limits. +func (w *numberTreeWalker) limits(d pdfcpu_types.Dict, label string, isRoot bool) (lo, hi int, ok bool) { + if isRoot { + return 0, 0, false + } + obj, found := d.Find("Limits") + if !found { + w.problem(label, "non-root node has no /Limits") + return 0, 0, false + } + arr, isArr := derefIn(w.ctx, obj).(pdfcpu_types.Array) + if !isArr || len(arr) != 2 { + w.problem(label, "/Limits is not an array of two integers") + return 0, 0, false + } + a, aOK := derefIn(w.ctx, arr[0]).(pdfcpu_types.Integer) + b, bOK := derefIn(w.ctx, arr[1]).(pdfcpu_types.Integer) + if !aOK || !bOK { + w.problem(label, "/Limits is not an array of two integers") + return 0, 0, false + } + if a > b { + w.problem(label, fmt.Sprintf("/Limits [%d %d] is not ascending", a, b)) + return 0, 0, false + } + return int(a), int(b), true +} + +// readNums records the key/value pairs of one leaf. +func (w *numberTreeWalker) readNums(nums pdfcpu_types.Array, label string, lo, hi int, hasLimits bool) { + if len(nums)%2 != 0 { + w.problem(label, fmt.Sprintf("/Nums has odd length %d; the last entry has no value", len(nums))) + } + prev, havePrev := 0, false + for i := 0; i+1 < len(nums); i += 2 { + keyObj := derefIn(w.ctx, nums[i]) + k, ok := keyObj.(pdfcpu_types.Integer) + if !ok { + w.problem(label, fmt.Sprintf("/Nums entry %d: key is %s, not an integer", i, objKind(keyObj))) + continue + } + key := int(k) + if hasLimits && (key < lo || key > hi) { + w.problem(label, fmt.Sprintf("key %d outside /Limits [%d %d]", key, lo, hi)) + } + if havePrev && key <= prev { + w.problem(label, fmt.Sprintf("key %d after key %d: keys not in ascending order", key, prev)) + } + prev, havePrev = key, true + if first, dup := w.keyNode[key]; dup { + w.problem(label, fmt.Sprintf("key %d already in node %s; first value kept", key, first)) + continue + } + w.keyNode[key] = label + w.entries[key] = nums[i+1] + } +} + +// objKind names the PDF type of o for messages, with an article. +func objKind(o pdfcpu_types.Object) string { + l := typeLabel(o) + switch { + case o == nil: + return l + case strings.ContainsRune("aeiou", rune(l[0])): + return "an " + l + default: + return "a " + l + } +} diff --git a/internal/pdfcore/numbertree_shapes_test.go b/internal/pdfcore/numbertree_shapes_test.go new file mode 100644 index 0000000..28c604c --- /dev/null +++ b/internal/pdfcore/numbertree_shapes_test.go @@ -0,0 +1,88 @@ +package pdfcore + +import ( + "strings" + "testing" +) + +func TestWalkNumberTree_NodeShapes(t *testing.T) { + cases := []struct { + name string + objs []rawObj + // problems lists every expected problem as "node label: message part". + problems []string + want map[int]int + }{ + {"direct kid with /Limits", []rawObj{ + {10, "<< /Kids [<< /Limits [0 1] /Nums [0 100 1 101] >>] >>"}, + }, nil, map[int]int{0: 100, 1: 101}}, + {"direct kid without /Limits", []rawObj{ + {10, "<< /Kids [<< /Nums [0 100] >>] >>"}, + }, []string{"node (direct): non-root node has no /Limits"}, map[int]int{0: 100}}, + {"null kid", []rawObj{ + {10, "<< /Kids [null 11 0 R] >>"}, + {11, "<< /Limits [2 2] /Nums [2 102] >>"}, + }, []string{"node (direct): node is null"}, map[int]int{2: 102}}, + {"/Kids not an array", []rawObj{ + {10, "<< /Kids 5 >>"}, + }, []string{"node 10 0 R: /Kids is an integer, not an array"}, map[int]int{}}, + {"/Kids dangling", []rawObj{ + {10, "<< /Kids 99 0 R >>"}, + }, []string{"node 10 0 R: /Kids is null, not an array"}, map[int]int{}}, + {"/Kids and /Nums indirect", []rawObj{ + {10, "<< /Kids 11 0 R >>"}, + {11, "[12 0 R]"}, + {12, "<< /Limits [0 0] /Nums 13 0 R >>"}, + {13, "[0 100]"}, + }, nil, map[int]int{0: 100}}, + {"/Nums not an array", []rawObj{ + {10, "<< /Nums (x) >>"}, + }, []string{"node 10 0 R: /Nums is a string, not an array"}, map[int]int{}}, + {"/Limits of one entry", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits [0] /Nums [0 100] >>"}, + }, []string{"node 11 0 R: /Limits is not an array of two integers"}, map[int]int{0: 100}}, + {"/Limits not an array", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits 3 /Nums [0 100] >>"}, + }, []string{"node 11 0 R: /Limits is not an array of two integers"}, map[int]int{0: 100}}, + {"/Limits of non-integers", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits [(a) 1] /Nums [0 100] >>"}, + }, []string{"node 11 0 R: /Limits is not an array of two integers"}, map[int]int{0: 100}}, + {"keys on both /Limits bounds", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits [3 5] /Nums [3 103 5 105] >>"}, + }, nil, map[int]int{3: 103, 5: 105}}, + {"root /Limits ignored", []rawObj{ + {10, "<< /Limits [9 1] /Nums [0 100] >>"}, + }, nil, map[int]int{0: 100}}, + {"node with both /Kids and /Nums", []rawObj{ + {10, "<< /Kids [11 0 R] /Nums [0 100] >>"}, + {11, "<< /Limits [1 1] /Nums [1 101] >>"}, + }, nil, map[int]int{0: 100, 1: 101}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries, order, problems := walkRawNumberTree(t, tc.objs...) + if len(problems) != len(tc.problems) { + t.Errorf("problems:\n%s\nwant %d: %v", problemsText(problems), len(tc.problems), tc.problems) + } + for _, want := range tc.problems { + found := false + for _, p := range problems { + if strings.Contains(problemText(p), want) { + found = true + } + } + if !found { + t.Errorf("no problem matching %q in:\n%s", want, problemsText(problems)) + } + } + checkIntEntries(t, entries, tc.want) + if len(order) != len(tc.want) { + t.Errorf("order = %v, want %d keys", order, len(tc.want)) + } + }) + } +} diff --git a/internal/pdfcore/numbertree_test.go b/internal/pdfcore/numbertree_test.go new file mode 100644 index 0000000..26037a6 --- /dev/null +++ b/internal/pdfcore/numbertree_test.go @@ -0,0 +1,244 @@ +package pdfcore + +import ( + "fmt" + "slices" + "strings" + "testing" + + pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" +) + +// Number trees are read without pdfcpu validation: most malformed shapes here +// are rejected at Inspector.Open, and a /Kids cycle makes pdfcpu's own +// validator recurse until the process dies. + +const numberTreeWhere = "NT" + +// walkRawNumberTree reads objs as a document and walks the number tree rooted +// at object 10. +func walkRawNumberTree(t *testing.T, objs ...rawObj) (map[int]pdfcpu_types.Object, []int, []StructureProblem) { + t.Helper() + _, doc := openUnvalidated(t, numberTreePDF(objs...)) + entries, order, problems := walkNumberTree(doc.PDFContext, *pdfcpu_types.NewIndirectRef(10, 0), numberTreeWhere) + for _, p := range problems { + if !strings.Contains(p.Where, numberTreeWhere) { + t.Errorf("problem location %q does not carry the caller's prefix %q", p.Where, numberTreeWhere) + } + } + return entries, order, problems +} + +// problemText is a problem's location and message on one line. +func problemText(p StructureProblem) string { return p.Where + ": " + p.Message } + +func problemsText(ps []StructureProblem) string { + lines := make([]string, len(ps)) + for i, p := range ps { + lines[i] = problemText(p) + } + return strings.Join(lines, "\n") +} + +// checkIntEntries requires entries to hold exactly want, each value an integer. +func checkIntEntries(t *testing.T, entries map[int]pdfcpu_types.Object, want map[int]int) { + t.Helper() + if len(entries) != len(want) { + t.Errorf("got %d entries %v, want %d %v", len(entries), entries, len(want), want) + } + for k, v := range want { + got, ok := entries[k] + if !ok { + t.Errorf("key %d missing", k) + continue + } + if n, ok := got.(pdfcpu_types.Integer); !ok || int(n) != v { + t.Errorf("key %d = %v, want %d", k, got, v) + } + } +} + +func TestWalkNumberTree_WellFormed(t *testing.T) { + cases := []struct { + name string + objs []rawObj + want map[int]int + }{ + {"flat /Nums", []rawObj{ + {10, "<< /Nums [0 100 1 101 2 102] >>"}, + }, map[int]int{0: 100, 1: 101, 2: 102}}, + {"two-level /Kids with /Limits", []rawObj{ + {10, "<< /Kids [11 0 R 12 0 R] >>"}, + {11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, + {12, "<< /Limits [2 3] /Nums [2 102 3 103] >>"}, + }, map[int]int{0: 100, 1: 101, 2: 102, 3: 103}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries, order, problems := walkRawNumberTree(t, tc.objs...) + if len(problems) != 0 { + t.Errorf("unexpected problems:\n%s", problemsText(problems)) + } + checkIntEntries(t, entries, tc.want) + want := make([]int, 0, len(tc.want)) + for k := range tc.want { + want = append(want, k) + } + slices.Sort(want) + if !slices.Equal(order, want) { + t.Errorf("order = %v, want %v", order, want) + } + }) + } +} + +func TestWalkNumberTree_ReportsMalformedNodes(t *testing.T) { + cases := []struct { + name string + objs []rawObj + // node is the ref every problem list must name; key, when set, must be + // named by the same problem. + node string + key string + want map[int]int + }{ + {"odd-length /Nums", []rawObj{ + {10, "<< /Nums [0 100 1] >>"}, + }, "10 0 R", "", map[int]int{0: 100}}, + {"non-integer key", []rawObj{ + {10, "<< /Nums [0 100 (x) 101 2 102] >>"}, + }, "10 0 R", "", map[int]int{0: 100, 2: 102}}, + {"key outside /Limits", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits [0 1] /Nums [0 100 5 105] >>"}, + }, "11 0 R", "5", nil}, + {"/Limits reversed", []rawObj{ + {10, "<< /Kids [11 0 R] >>"}, + {11, "<< /Limits [3 0] /Nums [0 100 3 103] >>"}, + }, "11 0 R", "", nil}, + {"keys out of order in a leaf", []rawObj{ + {10, "<< /Nums [2 102 0 100] >>"}, + }, "10 0 R", "", nil}, + {"node not a dict", []rawObj{ + {10, "<< /Kids [11 0 R 12 0 R] >>"}, + {11, "42"}, + {12, "<< /Limits [0 0] /Nums [0 100] >>"}, + }, "11 0 R", "", map[int]int{0: 100}}, + {"node with neither /Kids nor /Nums", []rawObj{ + {10, "<< /Kids [11 0 R 12 0 R] >>"}, + {11, "<< /Limits [0 1] >>"}, + {12, "<< /Limits [2 2] /Nums [2 102] >>"}, + }, "11 0 R", "", map[int]int{2: 102}}, + {"dangling kid ref", []rawObj{ + {10, "<< /Kids [11 0 R 99 0 R] >>"}, + {11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, + }, "99 0 R", "", map[int]int{0: 100, 1: 101}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries, _, problems := walkRawNumberTree(t, tc.objs...) + if len(problems) == 0 { + t.Fatalf("no problem reported") + } + found := false + for _, p := range problems { + text := problemText(p) + if strings.Contains(text, tc.node) && (tc.key == "" || strings.Contains(p.Message, tc.key)) { + found = true + } + } + if !found { + t.Errorf("no problem names node %s (key %q):\n%s", tc.node, tc.key, problemsText(problems)) + } + if tc.want != nil { + checkIntEntries(t, entries, tc.want) + } + }) + } +} + +func TestWalkNumberTree_DuplicateKeyKeepsFirstValueAndNamesBothNodes(t *testing.T) { + entries, order, problems := walkRawNumberTree(t, + rawObj{10, "<< /Kids [11 0 R 12 0 R] >>"}, + rawObj{11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, + rawObj{12, "<< /Limits [1 2] /Nums [1 201 2 202] >>"}, + ) + checkIntEntries(t, entries, map[int]int{0: 100, 1: 101, 2: 202}) + if !slices.Equal(order, []int{0, 1, 2}) { + t.Errorf("order = %v, want every key once: [0 1 2]", order) + } + found := false + for _, p := range problems { + text := problemText(p) + if strings.Contains(text, "11 0 R") && strings.Contains(text, "12 0 R") && strings.Contains(p.Message, "1") { + found = true + } + } + if !found { + t.Errorf("no problem names key 1 and both nodes 11 0 R and 12 0 R:\n%s", problemsText(problems)) + } +} + +func TestWalkNumberTree_KidPointingBackAtRootIsReportedOnce(t *testing.T) { + entries, order, problems := walkRawNumberTree(t, + rawObj{10, "<< /Kids [11 0 R 10 0 R] >>"}, + rawObj{11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, + ) + checkIntEntries(t, entries, map[int]int{0: 100, 1: 101}) + if !slices.Equal(order, []int{0, 1}) { + t.Errorf("order = %v, want every key once: [0 1]", order) + } + n := 0 + for _, p := range problems { + if strings.Contains(p.Message, "reached twice") { + n++ + if !strings.Contains(problemText(p), "10 0 R") { + t.Errorf("the reached-twice problem does not name 10 0 R: %s", problemText(p)) + } + } + } + if n != 1 { + t.Errorf("got %d \"reached twice\" problems, want 1:\n%s", n, problemsText(problems)) + } +} + +func TestWalkNumberTree_DepthPastCapIsReported(t *testing.T) { + const levels = 65 + objs := make([]rawObj, 0, levels) + for i := 0; i < levels; i++ { + num := 10 + i + switch { + case i == 0: + objs = append(objs, rawObj{num, fmt.Sprintf("<< /Kids [%d 0 R] >>", num+1)}) + case i < levels-1: + objs = append(objs, rawObj{num, fmt.Sprintf("<< /Limits [0 0] /Kids [%d 0 R] >>", num+1)}) + default: + objs = append(objs, rawObj{num, "<< /Limits [0 0] /Nums [0 100] >>"}) + } + } + _, _, problems := walkRawNumberTree(t, objs...) + found := false + for _, p := range problems { + if strings.Contains(p.Message, "64") && strings.Contains(p.Message, "deep") { + found = true + } + } + if !found { + t.Errorf("no depth problem naming the 64-level cap:\n%s", problemsText(problems)) + } +} + +func TestWalkNumberTree_ReturnsValuesAsStored(t *testing.T) { + entries, _, problems := walkRawNumberTree(t, + rawObj{10, "<< /Nums [0 11 0 R] >>"}, + rawObj{11, "[12 0 R null]"}, + rawObj{12, "<< /S /P >>"}, + ) + if len(problems) != 0 { + t.Errorf("unexpected problems:\n%s", problemsText(problems)) + } + ref, ok := entries[0].(pdfcpu_types.IndirectRef) + if !ok || ref.ObjectNumber != 11 { + t.Errorf("key 0 = %#v, want the reference 11 0 R as stored", entries[0]) + } +} diff --git a/internal/pdfcore/stream.go b/internal/pdfcore/stream.go index 398cc59..c97402a 100644 --- a/internal/pdfcore/stream.go +++ b/internal/pdfcore/stream.go @@ -33,8 +33,13 @@ func (ins *Inspector) pageContentStreamNodeIDs(tabID string, pageNum int) ([]str if leaf == nil { return nil, fmt.Errorf("page %d not found", pageNum) } - pageDict := leaf.dict + return contentStreamIDs(leaf.dict) +} +// contentStreamIDs returns the node IDs of pageDict's /Contents stream(s) in +// concatenation order, with the contract pageContentStreamNodeIDs documents. +// Callers that dereference through pdfcpu hold pdfMu. +func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { contents, found := pageDict.Find("Contents") if !found || contents == nil { return nil, nil @@ -219,17 +224,47 @@ func (ins *Inspector) GetContentStream(tabID, nodeID string) (*ContentStreamData return cached, nil } + raw, decodeErr, err := doc.decodeContentStreamLocked(nodeID) + if err != nil { + return nil, err + } + if decodeErr != "" { + result := &ContentStreamData{ + NodeID: nodeID, + Error: decodeErr, + } + doc.streamCache[nodeID] = result + return result, nil + } + + result := &ContentStreamData{ + NodeID: nodeID, + Raw: raw, + } + if raw != "" { + result.Tokenized = tokenizeContentStream(raw) + result.Formatted = Format(result.Tokenized) + } + doc.streamCache[nodeID] = result + return result, nil +} + +// decodeContentStreamLocked resolves nodeID and returns its decoded stream +// content. decodeErr carries a failure that belongs in the result (the node is +// not a stream, or decoding failed); err a failure to resolve the node. It +// neither reads nor writes streamCache and does not take streamMu. Callers +// hold pdfMu. +func (d *DocumentState) decodeContentStreamLocked(nodeID string) (raw string, decodeErr string, err error) { var obj pdfcpu_types.Object err = safeCall(func() error { var e error - obj, e = resolveNodeObject(doc, nodeID) + obj, e = resolveNodeObject(d, nodeID) return e }) if err != nil { - return nil, wrapPDFError(err) + return "", "", wrapPDFError(err) } - // Extract the StreamDict from the resolved object. var sd pdfcpu_types.StreamDict switch v := obj.(type) { case pdfcpu_types.StreamDict: @@ -239,44 +274,20 @@ func (ins *Inspector) GetContentStream(tabID, nodeID string) (*ContentStreamData case pdfcpu_types.XRefStreamDict: sd = v.StreamDict default: - result := &ContentStreamData{ - NodeID: nodeID, - Error: "node is not a stream object", - } - doc.streamCache[nodeID] = result - return result, nil + return "", "node is not a stream object", nil } // Decode the stream inside safeCall (pdfcpu can panic). - err = safeCall(func() error { + if err := safeCall(func() error { return sd.Decode() - }) - if err != nil { - result := &ContentStreamData{ - NodeID: nodeID, - Error: fmt.Sprintf("failed to decode stream: %v", err), - } - doc.streamCache[nodeID] = result - return result, nil + }); err != nil { + return "", fmt.Sprintf("failed to decode stream: %v", err), nil } - var raw string if sd.Content != nil { - raw = string(sd.Content) - } else { - raw = string(sd.Raw) + return string(sd.Content), "", nil } - - result := &ContentStreamData{ - NodeID: nodeID, - Raw: raw, - } - if raw != "" { - result.Tokenized = tokenizeContentStream(raw) - result.Formatted = Format(result.Tokenized) - } - doc.streamCache[nodeID] = result - return result, nil + return string(sd.Raw), "", nil } // tokenizeContentStream performs lexical tokenization of a PDF content stream, diff --git a/internal/pdfcore/structure.go b/internal/pdfcore/structure.go new file mode 100644 index 0000000..44ea686 --- /dev/null +++ b/internal/pdfcore/structure.go @@ -0,0 +1,837 @@ +package pdfcore + +import ( + "context" + "fmt" + "sort" + "strings" + "unicode/utf8" + + pdfcpu_model "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" + pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" +) + +// maxStructTreeDepth caps how many elements one descent path of the +// structure tree may hold. A deeper element becomes one error kid, so a +// pathological chain cannot exhaust the goroutine stack. +const maxStructTreeDepth = 256 + +// noPageMsg is the error on an MCID or MCR kid with no page to attribute it to. +const noPageMsg = "no page: the nearest /Pg (on the kid, its element or an ancestor) is absent or not a page; MCID not attributed to a page" + +// rootMCIDMsg is the error on an MCID or MCR kid listed directly in the +// root's /K, where no element owns it. +const rootMCIDMsg = "MCID directly under the structure tree root, with no element to own it; not aligned" + +// directElement stands for a direct-dictionary element where a ref is shown. +const directElement = "(direct)" + +// errStructureCanceled is returned when the document is closed during +// GetStructure. It satisfies errors.Is(err, context.Canceled). +var errStructureCanceled = fmt.Errorf("structure walk canceled: %w", context.Canceled) + +// GetStructure returns the tagged-structure report for the document in tabID: +// the structure tree from the catalog's /StructTreeRoot and, for every +// numbered page, its ParentTree array, the marked content of its content +// streams and the per-MCID alignment of the two with the tree. Anything +// malformed in the document is a finding in the report; the call fails only +// for an unknown tab, a close during the walk (errors.Is context.Canceled) or +// a pdfcpu failure outside every per-node guard. Not cached. Every content +// stream of every page is decoded, without writing the stream cache. +func (ins *Inspector) GetStructure(tabID string) (*StructureReport, error) { + doc, err := ins.GetDocument(tabID) + if err != nil { + return nil, err + } + return doc.buildStructure(newStructureWalker(doc.PDFContext)) +} + +// structureWalker carries the state of one GetStructure walk. +type structureWalker struct { + ctx *pdfcpu_model.Context + // beforeLock, when set, runs just before pdfMu is taken: with 0 before + // the tree walk, then with each page number. Tests use it to close the + // document at a known point. + beforeLock func(page int) + + report *StructureReport + // pageNums maps a page object number to the first page number it got in + // the page-tree walk. + pageNums map[int]int + roleMap map[string]string + // visited holds the object numbers of every element and indirect /K + // array already walked. + visited map[int]bool + // claims lists, per page number and MCID, the elements whose /K names + // it, in walk order. + claims map[int]map[int][]string + // parentTree holds the ParentTree entries by key. + parentTree map[int]pdfcpu_types.Object +} + +func newStructureWalker(ctx *pdfcpu_model.Context) *structureWalker { + return &structureWalker{ + ctx: ctx, + report: &StructureReport{ + RoleMap: []RoleMapEntry{}, + Elements: []*StructKid{}, + Pages: []*StructPage{}, + OtherParentTree: []ParentTreeEntry{}, + Problems: []StructureProblem{}, + }, + pageNums: map[int]int{}, + roleMap: map[string]string{}, + visited: map[int]bool{}, + claims: map[int]map[int][]string{}, + parentTree: map[int]pdfcpu_types.Object{}, + } +} + +// buildStructure runs w over d. Phase 1 holds pdfMu once to read the page +// walk, the structure tree, the ParentTree and every page's /StructParents. +// Phase 2 takes pdfMu per page to decode and scan its content streams, and +// checks for a close of d before every page. +func (d *DocumentState) buildStructure(w *structureWalker) (*StructureReport, error) { + if w.beforeLock != nil { + w.beforeLock(0) + } + if docClosed(d) { + return nil, errStructureCanceled + } + leaves, err := d.structurePhase1(w) + if err != nil { + return nil, err + } + for i, leaf := range leaves { + page := i + 1 + if w.beforeLock != nil { + w.beforeLock(page) + } + if docClosed(d) { + return nil, errStructureCanceled + } + sp := w.report.Pages[i] + d.scanPageLocked(w, leaf, sp) + sp.Alignment = w.align(sp) + } + return w.report, nil +} + +// structurePhase1 reads everything but the content streams under one pdfMu +// hold and returns the numbered pages. +func (d *DocumentState) structurePhase1(w *structureWalker) ([]*pageLeaf, error) { + d.pdfMu.Lock() + defer d.pdfMu.Unlock() + var leaves []*pageLeaf + err := safeCall(func() error { + pt, ptErr := d.pageTree() + leaves = pt.leaves + w.readTree(leaves) + if ptErr != nil { + w.problem("page tree", fmt.Sprintf("page-tree walk stopped after %d pages: %v; later pages not listed", len(leaves), ptErr)) + } + return nil + }) + if err != nil { + return nil, wrapPDFError(err) + } + return leaves, nil +} + +// scanPageLocked decodes and scans one page's content streams with pdfMu +// held. +func (d *DocumentState) scanPageLocked(w *structureWalker, leaf *pageLeaf, sp *StructPage) { + d.pdfMu.Lock() + defer d.pdfMu.Unlock() + guard(func() { w.scanPage(d, leaf, sp) }, func(msg string) { + sp.Err = joinErr(sp.Err, "page could not be read: "+msg) + }) +} + +// problem records a finding with no row of its own. +func (w *structureWalker) problem(where, msg string) { + w.report.Problems = append(w.report.Problems, StructureProblem{Where: where, Message: msg}) +} + +// deref resolves o, returning nil for a dangling reference. +func (w *structureWalker) deref(o pdfcpu_types.Object) pdfcpu_types.Object { + return derefIn(w.ctx, o) +} + +// readTree fills the report from the catalog, the structure tree root and the +// page leaves. +func (w *structureWalker) readTree(leaves []*pageLeaf) { + for i, leaf := range leaves { + if leaf.ref != nil { + n := leaf.ref.ObjectNumber.Value() + if _, seen := w.pageNums[n]; !seen { + w.pageNums[n] = i + 1 + } + } + } + + catalog, err := w.ctx.Catalog() + if err != nil || catalog == nil { + w.problem("catalog", "the catalog could not be read") + } else { + w.readCatalog(catalog) + } + + named := map[int]bool{} + for i, leaf := range leaves { + sp := &StructPage{ + PageNum: i + 1, + ParentTree: []ParentTreeSlot{}, + Streams: []MarkedStream{}, + ContentMCIDs: []ContentMCID{}, + Alignment: []MCIDAlignment{}, + } + if leaf.ref != nil { + sp.Ref = refString(*leaf.ref) + } + w.report.Pages = append(w.report.Pages, sp) + if leaf.err != nil { + sp.Err = leaf.err.Error() + continue + } + guard(func() { w.readPageParentTree(leaf, sp) }, func(msg string) { + sp.ParentTreeErr = "the ParentTree entry could not be read: " + msg + }) + if sp.StructParents != nil { + named[*sp.StructParents] = true + } + } + w.readOtherParentTree(named) +} + +// readCatalog reads the document-level entries and walks the structure tree. +func (w *structureWalker) readCatalog(catalog pdfcpu_types.Dict) { + r := w.report + if o, found := catalog.Find("Lang"); found { + v := w.deref(o) + s, ok := decodeTextStringOK(v) + if !ok || !utf8.ValidString(s) { + s = textStringOrRaw(v) + w.problem("catalog", fmt.Sprintf("/Lang is %s that does not decode as a text string", objKind(v))) + } + r.DocLang = s + } + if o, found := catalog.Find("MarkInfo"); found { + mi, ok := w.deref(o).(pdfcpu_types.Dict) + if !ok { + w.problem("catalog", fmt.Sprintf("/MarkInfo is %s, not a dictionary", objKind(w.deref(o)))) + } else if mo, found := mi.Find("Marked"); found { + if b, ok := w.deref(mo).(pdfcpu_types.Boolean); ok { + v := b.Value() + r.Marked = &v + } else { + w.problem("catalog", fmt.Sprintf("/MarkInfo /Marked is %s, not a boolean", objKind(w.deref(mo)))) + } + } + } + + rootObj, found := catalog.Find("StructTreeRoot") + if !found || rootObj == nil { + return + } + r.HasStructTree = true + where := "StructTreeRoot" + if ref, ok := rootObj.(pdfcpu_types.IndirectRef); ok { + r.RootRef = refString(ref) + where += " " + r.RootRef + } + root, ok := w.deref(rootObj).(pdfcpu_types.Dict) + if !ok { + w.problem(where, fmt.Sprintf("/StructTreeRoot is %s, not a dictionary", objKind(w.deref(rootObj)))) + return + } + guard(func() { w.readRoot(root, where) }, func(msg string) { + w.problem(where, "the structure tree root could not be read: "+msg) + }) +} + +// readRoot reads /RoleMap, /ParentTreeNextKey and /ParentTree, then walks +// the root's /K. +func (w *structureWalker) readRoot(root pdfcpu_types.Dict, where string) { + r := w.report + if o, found := root.Find("RoleMap"); found { + if rm, ok := w.deref(o).(pdfcpu_types.Dict); ok { + keys := make([]string, 0, len(rm)) + for k := range rm { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := w.deref(rm[k]) + to := nameOf(v) + if _, isName := v.(pdfcpu_types.Name); !isName { + w.problem(where, fmt.Sprintf("/RoleMap /%s maps to %s, not a name", k, objKind(v))) + } else { + w.roleMap[k] = to + } + r.RoleMap = append(r.RoleMap, RoleMapEntry{From: k, To: to}) + } + } else { + w.problem(where, fmt.Sprintf("/RoleMap is %s, not a dictionary", objKind(w.deref(o)))) + } + } + if o, found := root.Find("ParentTreeNextKey"); found { + if n, ok := w.deref(o).(pdfcpu_types.Integer); ok { + v := int(n) + r.ParentTreeNextKey = &v + } else { + w.problem(where, fmt.Sprintf("/ParentTreeNextKey is %s, not an integer", objKind(w.deref(o)))) + } + } + if o, found := root.Find("ParentTree"); found && o != nil { + r.HasParentTree = true + ptWhere := "ParentTree" + if ref, ok := o.(pdfcpu_types.IndirectRef); ok { + r.ParentTreeRef = refString(ref) + ptWhere += " " + r.ParentTreeRef + } + entries, _, problems := walkNumberTree(w.ctx, o, ptWhere) + w.parentTree = entries + r.Problems = append(r.Problems, problems...) + } + r.Elements = w.walkK(root["K"], "", 0, 0) +} + +// walkK returns the kids of one /K value. owner is the claiming element's ref +// ("" for the root), page its effective page, depth its depth (0 for the +// root). +func (w *structureWalker) walkK(k pdfcpu_types.Object, owner string, page, depth int) []*StructKid { + kids := []*StructKid{} + if k == nil { + return kids + } + items := []pdfcpu_types.Object{k} + if arr, ok := k.(pdfcpu_types.Array); ok { + items = arr + } else if ref, ok := k.(pdfcpu_types.IndirectRef); ok { + if arr, ok := w.deref(ref).(pdfcpu_types.Array); ok { + // An indirect /K array shared by several elements is walked once, + // so its kids cannot be listed once per element that names it. + n := ref.ObjectNumber.Value() + if w.visited[n] { + return append(kids, w.errorKid(fmt.Sprintf("/K array %s already listed; not walked again", refString(ref)))) + } + w.visited[n] = true + items = arr + } + } + for _, item := range items { + if item == nil { + continue + } + kids = append(kids, w.kid(item, owner, page, depth)) + } + return kids +} + +// kid classifies one /K entry. +func (w *structureWalker) kid(raw pdfcpu_types.Object, owner string, page, depth int) *StructKid { + c := &w.report.Counts + var ref *pdfcpu_types.IndirectRef + if r, ok := raw.(pdfcpu_types.IndirectRef); ok { + ref = &r + } + obj := w.deref(raw) + if obj == nil && ref != nil { + return w.errorKid(refString(*ref) + " does not resolve to an object") + } + switch v := obj.(type) { + case pdfcpu_types.Integer: + if v < 0 { + return w.errorKid(fmt.Sprintf("MCID %d is negative", v)) + } + c.MCIDKids++ + mcid := int(v) + w.claim(owner, page, mcid) + k := &StructKid{Kind: "mcid", MCID: &mcid, Page: page, PageInherited: page != 0} + switch { + case owner == "": + k.Err = rootMCIDMsg + case page == 0: + k.Err = noPageMsg + } + return k + case pdfcpu_types.Dict: + switch nameOf(w.deref(v["Type"])) { + case "MCR": + c.MCRKids++ + return w.mcrKid(v, owner, page) + case "OBJR": + c.OBJRKids++ + return w.objrKid(v, page) + } + return w.elementKid(v, ref, page, depth+1) + } + return w.errorKid(fmt.Sprintf("kid is %s, not an element, MCID, MCR or OBJR", objKind(obj))) +} + +// errorKid returns a kind "error" kid carrying msg. +func (w *structureWalker) errorKid(msg string) *StructKid { + w.report.Counts.ErrorKids++ + return &StructKid{Kind: "error", Err: msg} +} + +// claim records that owner's /K names mcid on page. +func (w *structureWalker) claim(owner string, page, mcid int) { + if owner == "" || page == 0 { + return + } + byMCID := w.claims[page] + if byMCID == nil { + byMCID = map[int][]string{} + w.claims[page] = byMCID + } + byMCID[mcid] = append(byMCID[mcid], owner) +} + +// kidPage returns the page of an MCR or OBJR kid: its own /Pg when present, +// else the owning element's page. errMsg is set when an own /Pg is not a page. +func (w *structureWalker) kidPage(d pdfcpu_types.Dict, page int) (int, bool, string) { + pg, found := d.Find("Pg") + if !found { + return page, page != 0, "" + } + n, _, msg := w.pageOf(pg) + return n, false, msg +} + +// mcrKid reads a marked-content reference dictionary. +func (w *structureWalker) mcrKid(d pdfcpu_types.Dict, owner string, page int) *StructKid { + k := &StructKid{Kind: "mcr"} + var errs []string + var msg string + k.Page, k.PageInherited, msg = w.kidPage(d, page) + errs = appendErr(errs, msg) + if stm, found := d.Find("Stm"); found { + ref, isRef := stm.(pdfcpu_types.IndirectRef) + switch { + case !isRef: + errs = append(errs, fmt.Sprintf("/Stm is %s, not a reference to a stream", objKind(stm))) + k.StmRef = directElement + default: + k.StmRef = refString(ref) + if _, isStream := w.deref(ref).(pdfcpu_types.StreamDict); !isStream { + errs = append(errs, fmt.Sprintf("/Stm %s is %s, not a stream", k.StmRef, objKind(w.deref(ref)))) + } + } + } + mcidObj, found := d.Find("MCID") + switch n, isInt := w.deref(mcidObj).(pdfcpu_types.Integer); { + case !found: + errs = append(errs, "MCR has no /MCID") + case !isInt: + errs = append(errs, fmt.Sprintf("MCR /MCID is %s, not an integer", objKind(w.deref(mcidObj)))) + case n < 0: + errs = append(errs, fmt.Sprintf("MCR /MCID %d is negative", n)) + default: + mcid := int(n) + k.MCID = &mcid + switch { + case k.StmRef != "": + case owner == "": + errs = append(errs, rootMCIDMsg) + default: + w.claim(owner, k.Page, mcid) + if k.Page == 0 && msg == "" { + errs = append(errs, noPageMsg) + } + } + } + k.Err = strings.Join(errs, "; ") + return k +} + +// objrKid reads an object reference dictionary. +func (w *structureWalker) objrKid(d pdfcpu_types.Dict, page int) *StructKid { + k := &StructKid{Kind: "objr"} + var errs []string + var msg string + k.Page, k.PageInherited, msg = w.kidPage(d, page) + errs = appendErr(errs, msg) + objRaw, found := d.Find("Obj") + ref, isRef := objRaw.(pdfcpu_types.IndirectRef) + switch { + case !found: + errs = append(errs, "OBJR has no /Obj") + case !isRef: + errs = append(errs, fmt.Sprintf("OBJR /Obj is %s, not a reference", objKind(objRaw))) + default: + k.ObjRef = refString(ref) + target := w.deref(ref) + var td pdfcpu_types.Dict + switch t := target.(type) { + case pdfcpu_types.Dict: + td = t + case pdfcpu_types.StreamDict: + td = t.Dict + case nil: + errs = append(errs, fmt.Sprintf("OBJR /Obj %s does not resolve to an object", k.ObjRef)) + default: + errs = append(errs, fmt.Sprintf("OBJR /Obj %s is %s, not a dictionary or stream", k.ObjRef, objKind(target))) + } + if td != nil { + var parts []string + for _, key := range []string{"Type", "Subtype"} { + if s := nameOf(w.deref(td[key])); s != "" { + parts = append(parts, s) + } + } + k.ObjType = strings.Join(parts, "/") + } + } + k.Err = strings.Join(errs, "; ") + return k +} + +// elementKid walks one structure element at depth (1 = a kid of the root). +// parentPage is the effective page of the nearest ancestor. +func (w *structureWalker) elementKid(d pdfcpu_types.Dict, ref *pdfcpu_types.IndirectRef, parentPage, depth int) *StructKid { + if depth > maxStructTreeDepth { + return w.errorKid(fmt.Sprintf("structure tree deeper than %d levels; subtree not walked", maxStructTreeDepth)) + } + e := &StructElem{Kids: []*StructKid{}} + id := directElement + if ref != nil { + n := ref.ObjectNumber.Value() + if w.visited[n] { + return w.errorKid(fmt.Sprintf("element %s already listed; not walked again", refString(*ref))) + } + w.visited[n] = true + e.Ref = refString(*ref) + e.NodeID = nodeIDForRef(*ref) + id = e.Ref + } + w.report.Counts.Elements++ + guard(func() { w.readElement(d, e, id, parentPage, depth) }, func(msg string) { + e.Err = joinErr(e.Err, "element could not be read: "+msg) + }) + return &StructKid{Kind: "element", Element: e, Page: e.Page, PageInherited: e.PageInherited} +} + +// readElement fills e from d and walks its kids. +func (w *structureWalker) readElement(d pdfcpu_types.Dict, e *StructElem, id string, parentPage, depth int) { + var errs []string + sObj, hasS := d.Find("S") + sv := w.deref(sObj) + switch s, isName := sv.(pdfcpu_types.Name); { + case !hasS || sv == nil: + errs = append(errs, "no /S") + case !isName: + errs = append(errs, fmt.Sprintf("/S is %s, not a name", objKind(sv))) + case s.Value() == "": + errs = append(errs, "/S is an empty name") + default: + e.S = s.Value() + } + if typ := nameOf(w.deref(d["Type"])); typ != "" && typ != "StructElem" && e.S == "" { + errs = append(errs, fmt.Sprintf("/Type /%s is not StructElem", typ)) + } + if e.S != "" { + mapped, msg := w.mapRole(e.S) + e.MappedS = mapped + errs = appendErr(errs, msg) + } + + if pg, found := d.Find("Pg"); found { + var msg string + e.Page, e.PgRef, msg = w.pageOf(pg) + errs = appendErr(errs, msg) + } else { + e.Page = parentPage + e.PageInherited = parentPage != 0 + } + + for _, f := range []struct { + key string + dst **string + }{{"Alt", &e.Alt}, {"ActualText", &e.ActualText}, {"E", &e.E}} { + if o, found := d.Find(f.key); found { + s, msg := w.textField(f.key, o) + *f.dst = &s + errs = appendErr(errs, msg) + } + } + for _, f := range []struct { + key string + dst *string + }{{"T", &e.Title}, {"Lang", &e.Lang}} { + if o, found := d.Find(f.key); found { + s, msg := w.textField(f.key, o) + *f.dst = s + errs = appendErr(errs, msg) + } + } + e.Err = strings.Join(errs, "; ") + + e.Kids = w.walkK(d["K"], id, e.Page, depth) +} + +// mapRole follows s through /RoleMap until a name with no mapping. A cycle or +// a mapping to the empty name stops the walk and is returned as msg with +// mapped "". +func (w *structureWalker) mapRole(s string) (mapped, msg string) { + seen := map[string]bool{s: true} + path := []string{s} + cur := s + for { + to, ok := w.roleMap[cur] + if !ok { + break + } + if to == "" { + return "", fmt.Sprintf("/RoleMap maps /%s to an empty name", cur) + } + path = append(path, to) + if seen[to] { + return "", "/RoleMap cycle: " + strings.Join(path, " -> ") + } + seen[to] = true + cur = to + } + if cur == s { + return "", "" + } + return cur, "" +} + +// pageOf resolves a /Pg value to a page number. pgRef is its "N G R" (or +// "(direct)"); msg is set when it does not name a numbered page. +func (w *structureWalker) pageOf(pg pdfcpu_types.Object) (page int, pgRef, msg string) { + ref, ok := pg.(pdfcpu_types.IndirectRef) + if !ok { + return 0, directElement, fmt.Sprintf("/Pg is %s, not a reference to a page", objKind(pg)) + } + pgRef = refString(ref) + if n, found := w.pageNums[ref.ObjectNumber.Value()]; found { + return n, pgRef, "" + } + target := w.deref(ref) + if target == nil { + return 0, pgRef, fmt.Sprintf("/Pg %s does not resolve to an object", pgRef) + } + return 0, pgRef, fmt.Sprintf("/Pg %s is %s that is not a page in the page tree", pgRef, objKind(target)) +} + +// textField decodes a text-string entry; a value that does not decode is +// shown raw with msg set. +func (w *structureWalker) textField(key string, o pdfcpu_types.Object) (string, string) { + v := w.deref(o) + if s, ok := decodeTextStringOK(v); ok && utf8.ValidString(s) { + return s, "" + } + return textStringOrRaw(v), fmt.Sprintf("/%s is %s that does not decode as a text string", key, objKind(v)) +} + +// readPageParentTree reads a page's /StructParents and its ParentTree array. +func (w *structureWalker) readPageParentTree(leaf *pageLeaf, sp *StructPage) { + o, found := leaf.dict.Find("StructParents") + if !found { + return + } + n, ok := w.deref(o).(pdfcpu_types.Integer) + if !ok { + sp.ParentTreeErr = fmt.Sprintf("/StructParents is %s, not an integer", objKind(w.deref(o))) + return + } + key := int(n) + sp.StructParents = &key + if !w.report.HasParentTree { + sp.ParentTreeErr = fmt.Sprintf("/StructParents %d but the document has no ParentTree", key) + return + } + v, ok := w.parentTree[key] + if !ok { + sp.ParentTreeErr = fmt.Sprintf("ParentTree has no key %d", key) + return + } + arr, ok := w.deref(v).(pdfcpu_types.Array) + if !ok { + sp.ParentTreeErr = fmt.Sprintf("ParentTree key %d is %s, not an array", key, objKind(w.deref(v))) + return + } + for i, slot := range arr { + s := ParentTreeSlot{MCID: i} + if slot != nil { + s.ElementRef, s.S, s.Err = w.elementRefOf(slot) + } + sp.ParentTree = append(sp.ParentTree, s) + } +} + +// elementRefOf reads a ParentTree value that should be a reference to an +// element: its ref, its /S and an error when it is not one. +func (w *structureWalker) elementRefOf(o pdfcpu_types.Object) (ref, s, msg string) { + r, ok := o.(pdfcpu_types.IndirectRef) + if !ok { + return "", "", fmt.Sprintf("value is %s, not a reference to an element", objKind(o)) + } + ref = refString(r) + target := w.deref(r) + d, ok := target.(pdfcpu_types.Dict) + if !ok { + if target == nil { + return ref, "", ref + " does not resolve to an object" + } + return ref, "", fmt.Sprintf("%s is %s, not an element", ref, objKind(target)) + } + return ref, nameOf(w.deref(d["S"])), "" +} + +// readOtherParentTree lists the ParentTree keys no page names. +func (w *structureWalker) readOtherParentTree(named map[int]bool) { + keys := make([]int, 0, len(w.parentTree)) + for k := range w.parentTree { + if !named[k] { + keys = append(keys, k) + } + } + sort.Ints(keys) + for _, k := range keys { + e := ParentTreeEntry{Key: k} + v := w.parentTree[k] + if _, isArr := w.deref(v).(pdfcpu_types.Array); isArr { + e.Err = "value is an array, but no page's /StructParents names this key" + } else { + e.ElementRef, e.S, e.Err = w.elementRefOf(v) + } + w.report.OtherParentTree = append(w.report.OtherParentTree, e) + } +} + +// scanPage decodes the page's content streams in /Contents order and scans +// their marked content. Callers hold pdfMu. +func (w *structureWalker) scanPage(d *DocumentState, leaf *pageLeaf, sp *StructPage) { + if leaf.err != nil { + return + } + ids, err := contentStreamIDs(leaf.dict) + if err != nil { + sp.Err = joinErr(sp.Err, err.Error()) + return + } + props := w.propertyLookup(leaf) + st := &mcState{} + for i, id := range ids { + ms := MarkedStream{Ref: nodeIDToRef(id)} + raw, decodeErr, err := d.decodeContentStreamLocked(id) + switch { + case err != nil: + ms.Err = err.Error() + case decodeErr != "": + ms.Err = decodeErr + } + if ms.Err != "" { + ms.DepthAtEnd, ms.MinDepth = len(st.stack), len(st.stack) + } else { + rows := scanMarkedContent(tokenizeContentStream(raw), i, props, st, &ms) + sp.ContentMCIDs = append(sp.ContentMCIDs, rows...) + } + sp.Streams = append(sp.Streams, ms) + } + for _, c := range sp.ContentMCIDs { + if c.Err == "" { + w.report.Counts.ContentMCIDs++ + } + } +} + +// propertyLookup returns the named-property-list lookup for a page: its +// effective /Resources /Properties, each level dereferenced. +func (w *structureWalker) propertyLookup(leaf *pageLeaf) propertyLookup { + return func(name string) (pdfcpu_types.Object, error) { + notFound := fmt.Errorf("property list /%s not found in /Resources /Properties", name) + res, ok := w.deref(leaf.attrs.resources).(pdfcpu_types.Dict) + if !ok { + return nil, notFound + } + props, ok := w.deref(res["Properties"]).(pdfcpu_types.Dict) + if !ok { + return nil, notFound + } + entry, found := props.Find(name) + if !found { + return nil, notFound + } + pl, ok := w.deref(entry).(pdfcpu_types.Dict) + if !ok { + return nil, fmt.Errorf("property list /%s is %s, not a dictionary", name, objKind(w.deref(entry))) + } + mcid, found := pl.Find("MCID") + if !found { + return nil, nil + } + v := w.deref(mcid) + if v == nil { + return nil, fmt.Errorf("/MCID in property list /%s does not resolve to an object", name) + } + return v, nil + } +} + +// align builds a page's per-MCID alignment rows in ascending MCID order. +func (w *structureWalker) align(sp *StructPage) []MCIDAlignment { + rows := map[int]*MCIDAlignment{} + row := func(mcid int) *MCIDAlignment { + if a, ok := rows[mcid]; ok { + return a + } + a := &MCIDAlignment{MCID: mcid, TreeElements: []string{}} + rows[mcid] = a + return a + } + for _, c := range sp.ContentMCIDs { + if c.Err == "" { + row(c.MCID).ContentCount++ + } + } + for _, s := range sp.ParentTree { + if s.ElementRef == "" && s.Err == "" { + continue + } + a := row(s.MCID) + if s.Err == "" { + a.ParentTreeElement, a.ParentTreeS = s.ElementRef, s.S + } + } + for mcid, owners := range w.claims[sp.PageNum] { + row(mcid).TreeElements = append([]string{}, owners...) + } + out := make([]MCIDAlignment, 0, len(rows)) + for _, a := range rows { + a.Consistent = a.ContentCount == 1 && a.ParentTreeElement != "" && + len(a.TreeElements) == 1 && a.TreeElements[0] == a.ParentTreeElement + out = append(out, *a) + } + sort.Slice(out, func(i, j int) bool { return out[i].MCID < out[j].MCID }) + return out +} + +// nodeIDToRef renders an "obj::" node ID as "N G R". +func nodeIDToRef(id string) string { + var gen, num int + if _, err := fmt.Sscanf(id, "obj:%d:%d", &gen, &num); err != nil { + return id + } + return fmt.Sprintf("%d %d R", num, gen) +} + +// appendErr appends msg to errs when it is not empty. +func appendErr(errs []string, msg string) []string { + if msg == "" { + return errs + } + return append(errs, msg) +} + +// joinErr joins two error texts with "; ", skipping an empty one. +func joinErr(a, b string) string { + if a == "" { + return b + } + return a + "; " + b +} diff --git a/internal/pdfcore/structure_bench_test.go b/internal/pdfcore/structure_bench_test.go new file mode 100644 index 0000000..15019a0 --- /dev/null +++ b/internal/pdfcore/structure_bench_test.go @@ -0,0 +1,130 @@ +package pdfcore + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// Benchmarks for the document-wide walks on a generated tagged document: +// 2000 pages, one image XObject per 10 pages, and 20 000 structure elements +// with one MCID each, 10 per page. Each iteration opens a fresh tab so no +// cache hides the walk; Open (parse and pdfcpu validation) is not timed. +// +// Run: go test -bench . -run '^$' ./internal/pdfcore/ + +const ( + benchPages = 2000 + benchMCIDsPerPage = 10 + benchImageEvery = 10 +) + +// benchTaggedPDF writes the benchmark document to a temp dir and returns its +// path. Pages are objects 10.., content streams 3000.., images 6000.., +// elements 10000.. +func benchTaggedPDF(b *testing.B) string { + b.Helper() + const ( + pageBase = 10 + contentBase = 3000 + imageBase = 6000 + elemBase = 10000 + ) + nElems := benchPages * benchMCIDsPerPage + objs := make([]rawObj, 0, 2*benchPages+nElems+10) + + kids := make([]string, benchPages) + nums := make([]string, 0, benchPages) + for p := 0; p < benchPages; p++ { + kids[p] = fmt.Sprintf("%d 0 R", pageBase+p) + slots := make([]string, benchMCIDsPerPage) + var content strings.Builder + for m := 0; m < benchMCIDsPerPage; m++ { + slots[m] = fmt.Sprintf("%d 0 R", elemBase+p*benchMCIDsPerPage+m) + fmt.Fprintf(&content, "/P <> BDC BT /F1 12 Tf 72 %d Td (Line) Tj ET EMC\n", m, 700-m*20) + } + nums = append(nums, fmt.Sprintf("%d [%s]", p, strings.Join(slots, " "))) + res := "" + if p%benchImageEvery == 0 { + img := imageBase + p/benchImageEvery + res = fmt.Sprintf("/Resources << /XObject << /Im0 %d 0 R >> >>", img) + content.WriteString("q 10 0 0 10 0 0 cm /Im0 Do Q\n") + objs = append(objs, rawObj{img, "<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>\nstream\n\x80\nendstream"}) + } + objs = append(objs, + rawObj{pageBase + p, fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R /StructParents %d %s >>", contentBase+p, p, res)}, + rawObj{contentBase + p, rawStream(content.String())}, + ) + } + elemRefs := make([]string, nElems) + for e := 0; e < nElems; e++ { + elemRefs[e] = fmt.Sprintf("%d 0 R", elemBase+e) + objs = append(objs, rawObj{elemBase + e, fmt.Sprintf("<< /Type /StructElem /S /P /P 6 0 R /Pg %d 0 R /K %d >>", pageBase+e/benchMCIDsPerPage, e%benchMCIDsPerPage)}) + } + objs = append(objs, + rawObj{1, "<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 5 0 R >>"}, + rawObj{2, fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", strings.Join(kids, " "), benchPages)}, + rawObj{5, fmt.Sprintf("<< /Type /StructTreeRoot /K 6 0 R /ParentTree 7 0 R /ParentTreeNextKey %d >>", benchPages)}, + rawObj{6, fmt.Sprintf("<< /Type /StructElem /S /Document /P 5 0 R /K [%s] >>", strings.Join(elemRefs, " "))}, + rawObj{7, fmt.Sprintf("<< /Nums [%s] >>", strings.Join(nums, " "))}, + ) + path := filepath.Join(b.TempDir(), "bench-tagged.pdf") + if err := os.WriteFile(path, rawPDF(objs...), 0o644); err != nil { + b.Fatal(err) + } + return path +} + +// benchWalk opens path in a fresh tab per iteration, untimed, and times run. +func benchWalk(b *testing.B, run func(ins *Inspector) error) { + path := benchTaggedPDF(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + ins := NewInspector() + if _, err := ins.Open("b", path); err != nil { + b.Fatalf("Open: %v", err) + } + b.StartTimer() + err := run(ins) + b.StopTimer() + _ = ins.Close("b") + if err != nil { + b.Fatal(err) + } + b.StartTimer() + } +} + +func BenchmarkGetPageIndex(b *testing.B) { + benchWalk(b, func(ins *Inspector) error { + entries, err := ins.GetPageIndex("b") + if err == nil && len(entries) != benchPages { + err = fmt.Errorf("got %d page rows, want %d", len(entries), benchPages) + } + return err + }) +} + +func BenchmarkGetImageIndex(b *testing.B) { + benchWalk(b, func(ins *Inspector) error { + entries, err := ins.GetImageIndex("b") + if err == nil && len(entries) != benchPages/benchImageEvery { + err = fmt.Errorf("got %d image rows, want %d", len(entries), benchPages/benchImageEvery) + } + return err + }) +} + +func BenchmarkGetStructure(b *testing.B) { + benchWalk(b, func(ins *Inspector) error { + r, err := ins.GetStructure("b") + if err == nil && r.Counts.ContentMCIDs != benchPages*benchMCIDsPerPage { + err = fmt.Errorf("got %d content MCIDs, want %d", r.Counts.ContentMCIDs, benchPages*benchMCIDsPerPage) + } + return err + }) +} diff --git a/internal/pdfcore/structure_raw_test.go b/internal/pdfcore/structure_raw_test.go new file mode 100644 index 0000000..dfe4e8f --- /dev/null +++ b/internal/pdfcore/structure_raw_test.go @@ -0,0 +1,154 @@ +package pdfcore + +import ( + "fmt" + "strings" +) + +// In-memory documents for the structure walk. Most of them carry shapes +// pdfcpu's open-time validation rejects (or recurses on), so tests read them +// with openUnvalidated, never through Inspector.Open or writeTempPDF. + +// structPagesObjs returns a page tree of n pages. Page k (1-based) is object +// 2+k and its /Contents is the single stream object 9000+k holding contents[k-1] +// (an empty stream when contents is shorter). pageExtra[k-1], when present, is +// added to page k's dictionary. +func structPagesObjs(n int, contents []string, pageExtra []string) []rawObj { + kids := make([]string, n) + for k := 1; k <= n; k++ { + kids[k-1] = fmt.Sprintf("%d 0 R", 2+k) + } + objs := []rawObj{{2, fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", strings.Join(kids, " "), n)}} + for k := 1; k <= n; k++ { + extra := "" + if k-1 < len(pageExtra) { + extra = pageExtra[k-1] + } + data := "" + if k-1 < len(contents) { + data = contents[k-1] + } + objs = append(objs, + rawObj{2 + k, fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R %s >>", 9000+k, extra)}, + rawObj{9000 + k, rawStream(data)}, + ) + } + return objs +} + +// structTreePDF is a one-page document (page object 3, content stream 9001) +// whose catalog points at a /StructTreeRoot object 5 with the given body. +// Element objects start at 10 by convention in the tests. +func structTreePDF(rootBody string, objs ...rawObj) []byte { + all := []rawObj{ + {1, "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>"}, + {5, rootBody}, + } + all = append(all, structPagesObjs(1, nil, nil)...) + all = append(all, objs...) + return rawPDF(all...) +} + +// structChainPDF is a structure tree whose root /K is a chain of depth nested +// /Div elements, objects 10 .. 10+depth-1, each the only kid of the previous. +func structChainPDF(depth int) []byte { + objs := make([]rawObj, 0, depth) + for i := 0; i < depth; i++ { + k := "[]" + if i < depth-1 { + k = fmt.Sprintf("[%d 0 R]", 11+i) + } + objs = append(objs, rawObj{10 + i, fmt.Sprintf("<< /Type /StructElem /S /Div /K %s >>", k)}) + } + return structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", objs...) +} + +// structSharedChildPDF is a structure tree whose root /K lists n /Div elements +// (objects 10 .. 10+n-1) that all name one /Span element (object 5000) as +// their only kid. +func structSharedChildPDF(n int) []byte { + refs := make([]string, n) + objs := make([]rawObj, 0, n+1) + for i := 0; i < n; i++ { + refs[i] = fmt.Sprintf("%d 0 R", 10+i) + objs = append(objs, rawObj{10 + i, "<< /Type /StructElem /S /Div /K [5000 0 R] >>"}) + } + objs = append(objs, rawObj{5000, "<< /Type /StructElem /S /Span >>"}) + return structTreePDF(fmt.Sprintf("<< /Type /StructTreeRoot /K [%s] >>", strings.Join(refs, " ")), objs...) +} + +// structSingleBackRefPDF has a single-object /K (not an array) leading back +// to an ancestor: root -> 10 (/Div) -> 11 (/P) -> 10. +func structSingleBackRefPDF() []byte { + return structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /Div /K 11 0 R >>"}, + rawObj{11, "<< /Type /StructElem /S /P /K 10 0 R >>"}, + ) +} + +// structRejectedKidsPDF has one /Div element (object 10, on page 1) whose /K +// array holds, in order: a string, a name, a real, a dangling ref (99 0 R), a +// dict with /Type /Foo and no /S, an MCR with no /MCID, a negative integer, a +// direct null, the MCID 0, and a /P element (object 11) with MCID 1. +func structRejectedKidsPDF() []byte { + return structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /Div /Pg 3 0 R /K [(str) /Nm 1.5 99 0 R << /Type /Foo >> << /Type /MCR /Pg 3 0 R >> -3 null 0 11 0 R] >>"}, + rawObj{11, "<< /Type /StructElem /S /P /Pg 3 0 R /K 1 >>"}, + ) +} + +// structRoleMapPDF has a /RoleMap with a cycle (/A -> /B -> /A) and a plain +// mapping (/Heading1 -> /H1); element 10 has /S /A, element 11 /S /Heading1, +// element 12 an empty /S name, element 13 no /S at all. +func structRoleMapPDF() []byte { + return structTreePDF("<< /Type /StructTreeRoot /RoleMap << /Heading1 /H1 /B /A /A /B >> /K [10 0 R 11 0 R 12 0 R 13 0 R] >>", + rawObj{10, "<< /Type /StructElem /S /A >>"}, + rawObj{11, "<< /Type /StructElem /S /Heading1 >>"}, + rawObj{12, "<< /Type /StructElem /S / >>"}, + rawObj{13, "<< /Type /StructElem /T (untitled) >>"}, + ) +} + +// structDirectElementPDF has a direct-dictionary /P element on page 1 that +// claims MCID 0, a ParentTree whose page array slot 0 is null, and page +// content that marks MCID 0. +func structDirectElementPDF() []byte { + all := []rawObj{ + {1, "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>"}, + {5, "<< /Type /StructTreeRoot /K [<< /Type /StructElem /S /P /Pg 3 0 R /K 0 >>] /ParentTree 6 0 R >>"}, + {6, "<< /Nums [0 [null]] >>"}, + } + all = append(all, structPagesObjs(1, []string{"/P <> BDC (x) Tj EMC"}, []string{"/StructParents 0"})...) + return rawPDF(all...) +} + +// markedPDF is a one-page document with no structure tree. The page /Contents +// is an array of the given stream object bodies (objects 20, 21, ...), and +// resources, when not empty, is the page /Resources entry. extra objects are +// added as given. +func markedPDF(resources string, streams []string, extra ...rawObj) []byte { + refs := make([]string, len(streams)) + all := []rawObj{{1, rawCatalog}, {2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"}} + for i, s := range streams { + refs[i] = fmt.Sprintf("%d 0 R", 20+i) + all = append(all, rawObj{20 + i, s}) + } + res := "" + if resources != "" { + res = "/Resources " + resources + } + all = append(all, rawObj{3, fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [%s] %s >>", strings.Join(refs, " "), res)}) + all = append(all, extra...) + return rawPDF(all...) +} + +// undecodableStream is a stream object body whose /FlateDecode data is not +// zlib, so decoding it fails. +const undecodableStream = "<< /Length 9 /Filter /FlateDecode >>\nstream\nnot-zlib!\nendstream" + +// numberTreePDF is a document with an empty page tree plus objs, for reading +// number trees directly. +func numberTreePDF(objs ...rawObj) []byte { + all := []rawObj{{1, rawCatalog}, {2, "<< /Type /Pages /Kids [] /Count 0 >>"}} + return rawPDF(append(all, objs...)...) +} diff --git a/internal/pdfcore/structure_shapes_test.go b/internal/pdfcore/structure_shapes_test.go new file mode 100644 index 0000000..c8ae734 --- /dev/null +++ b/internal/pdfcore/structure_shapes_test.go @@ -0,0 +1,450 @@ +package pdfcore + +import ( + "context" + "errors" + "slices" + "strings" + "testing" +) + +// Structure-walk branches for malformed entries, read from in-memory +// documents through openUnvalidated. + +// problemsContain reports whether some problem has where as its location +// prefix and msg in its message. +func problemsContain(r *StructureReport, where, msg string) bool { + for _, p := range r.Problems { + if strings.HasPrefix(p.Where, where) && strings.Contains(p.Message, msg) { + return true + } + } + return false +} + +// onlyKid returns the single kid of the single element under the root. +func onlyKid(t *testing.T, r *StructureReport) *StructKid { + t.Helper() + if len(r.Elements) != 1 || r.Elements[0].Element == nil || len(r.Elements[0].Element.Kids) != 1 { + t.Fatalf("root kids = %v, want one element with one kid", kindsOf(r.Elements)) + } + return r.Elements[0].Element.Kids[0] +} + +func TestGetStructure_CleanFileActualTextValue(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "structure.pdf"), "s") + p := mustElement(t, r, "P") + if p.ActualText == nil || *p.ActualText != "Hello world" { + t.Errorf("P ActualText = %v, want \"Hello world\" decoded from UTF-16BE", p.ActualText) + } + if p.Alt != nil || p.E != nil { + t.Errorf("P Alt %v E %v, want both absent (nil)", p.Alt, p.E) + } +} + +func TestGetStructure_MalformedRootEntriesAreProblems(t *testing.T) { + cases := []struct { + name string + root string + msg string + }{ + {"RoleMap not a dictionary", "<< /Type /StructTreeRoot /RoleMap 7 >>", "/RoleMap is an integer, not a dictionary"}, + {"RoleMap value not a name", "<< /Type /StructTreeRoot /RoleMap << /X 7 >> >>", "/RoleMap /X maps to an integer, not a name"}, + {"ParentTreeNextKey not an integer", "<< /Type /StructTreeRoot /ParentTreeNextKey (x) >>", "/ParentTreeNextKey is a string, not an integer"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, structTreePDF(tc.root)) + if !problemsContain(r, "StructTreeRoot 5 0 R", tc.msg) { + t.Errorf("Problems = %+v, want one at StructTreeRoot 5 0 R with %q", r.Problems, tc.msg) + } + if r.ParentTreeNextKey != nil { + t.Errorf("ParentTreeNextKey = %v, want nil", *r.ParentTreeNextKey) + } + }) + } +} + +func TestGetStructure_RoleMapValueNotANameIsListedButNotFollowed(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /RoleMap << /X 7 >> /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /X >>"}, + )) + if !slices.Equal(r.RoleMap, []RoleMapEntry{{From: "X", To: ""}}) { + t.Errorf("RoleMap = %+v, want [{X \"\"}]", r.RoleMap) + } + if e := mustElement(t, r, "X"); e.MappedS != "" || e.Err != "" { + t.Errorf("element X MappedS %q Err %q, want unmapped with no error", e.MappedS, e.Err) + } +} + +func TestGetStructure_StructTreeRootNotADictionary(t *testing.T) { + r := rawStructure(t, structTreePDF("42")) + if !r.HasStructTree || r.RootRef != "5 0 R" { + t.Errorf("HasStructTree %v RootRef %q, want true and 5 0 R", r.HasStructTree, r.RootRef) + } + if !problemsContain(r, "StructTreeRoot 5 0 R", "/StructTreeRoot is an integer, not a dictionary") { + t.Errorf("Problems = %+v, want the root type named", r.Problems) + } + if r.Elements == nil || len(r.Elements) != 0 || len(r.Pages) != 1 { + t.Errorf("Elements %v, %d pages; want an empty non-nil slice and page 1 still listed", r.Elements, len(r.Pages)) + } +} + +func TestGetStructure_CatalogLangThatDoesNotDecodeIsAProblem(t *testing.T) { + all := []rawObj{{1, "<< /Type /Catalog /Pages 2 0 R /Lang 5 >>"}} + all = append(all, structPagesObjs(1, nil, nil)...) + r := rawStructure(t, rawPDF(all...)) + if !problemsContain(r, "catalog", "/Lang is an integer that does not decode as a text string") { + t.Errorf("Problems = %+v, want the /Lang problem", r.Problems) + } + if r.HasStructTree { + t.Error("HasStructTree = true with no /StructTreeRoot") + } +} + +func TestGetStructure_MCRKidShapes(t *testing.T) { + cases := []struct { + name string + mcr string + stmRef string + mcid int // -1 for no MCID + err string + }{ + {"direct /Stm", "<< /Type /MCR /MCID 0 /Stm << /X 1 >> >>", "(direct)", 0, "/Stm is a dictionary, not a reference to a stream"}, + {"/Stm to a non-stream", "<< /Type /MCR /MCID 0 /Stm 3 0 R >>", "3 0 R", 0, "/Stm 3 0 R is a dictionary, not a stream"}, + {"/Stm to a stream", "<< /Type /MCR /MCID 0 /Stm 9001 0 R >>", "9001 0 R", 0, ""}, + {"/MCID not an integer", "<< /Type /MCR /MCID (x) >>", "", -1, "MCR /MCID is a string, not an integer"}, + {"negative /MCID", "<< /Type /MCR /MCID -1 >>", "", -1, "MCR /MCID -1 is negative"}, + {"own /Pg not a page", "<< /Type /MCR /MCID 0 /Pg 5 0 R >>", "", 0, "/Pg 5 0 R is a dictionary that is not a page"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /P /Pg 3 0 R /K " + tc.mcr + " >>"}, + )) + k := onlyKid(t, r) + if k.Kind != "mcr" || k.StmRef != tc.stmRef { + t.Errorf("kid = %+v, want mcr with StmRef %q", *k, tc.stmRef) + } + if (tc.mcid < 0 && k.MCID != nil) || (tc.mcid >= 0 && !intPtrIs(k.MCID, tc.mcid)) { + t.Errorf("MCID = %v, want %d (-1 for none)", k.MCID, tc.mcid) + } + if tc.err == "" && k.Err != "" || !strings.Contains(k.Err, tc.err) { + t.Errorf("Err = %q, want %q", k.Err, tc.err) + } + if tc.stmRef != "" && len(mustPage(t, r, 1).Alignment) != 0 { + t.Errorf("an MCR with /Stm is aligned on page 1: %+v", mustPage(t, r, 1).Alignment) + } + }) + } +} + +func TestGetStructure_OBJRKidShapes(t *testing.T) { + form := rawObj{30, "<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 0 >>\nstream\n\nendstream"} + subtypeOnly := rawObj{31, "<< /Subtype /Widget >>"} + cases := []struct { + name string + objr string + objRef string + objType string + err string + }{ + {"no /Obj", "<< /Type /OBJR >>", "", "", "OBJR has no /Obj"}, + {"direct /Obj", "<< /Type /OBJR /Obj << /Type /Annot >> >>", "", "", "OBJR /Obj is a dictionary, not a reference"}, + {"/Obj dangling", "<< /Type /OBJR /Obj 99 0 R >>", "99 0 R", "", "OBJR /Obj 99 0 R does not resolve to an object"}, + {"/Obj to a stream", "<< /Type /OBJR /Obj 30 0 R >>", "30 0 R", "XObject/Form", ""}, + {"/Obj with only /Subtype", "<< /Type /OBJR /Obj 31 0 R >>", "31 0 R", "Widget", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /Link /Pg 3 0 R /K " + tc.objr + " >>"}, form, subtypeOnly, + )) + k := onlyKid(t, r) + if k.Kind != "objr" || k.ObjRef != tc.objRef || k.ObjType != tc.objType || k.Page != 1 { + t.Errorf("kid = %+v, want objr ObjRef %q ObjType %q on page 1", *k, tc.objRef, tc.objType) + } + if tc.err == "" && k.Err != "" || !strings.Contains(k.Err, tc.err) { + t.Errorf("Err = %q, want %q", k.Err, tc.err) + } + }) + } +} + +func TestGetStructure_ElementEntryShapes(t *testing.T) { + cases := []struct { + name string + elem string + check func(t *testing.T, e *StructElem) + err string + }{ + {"/S not a name", "<< /Type /StructElem /S (P) >>", func(t *testing.T, e *StructElem) { + if e.S != "" { + t.Errorf("S = %q, want empty", e.S) + } + }, "/S is a string, not a name"}, + {"direct /Pg", "<< /Type /StructElem /S /P /Pg << /Type /Page >> >>", func(t *testing.T, e *StructElem) { + if e.Page != 0 || e.PgRef != "(direct)" { + t.Errorf("Page %d PgRef %q, want 0 and (direct)", e.Page, e.PgRef) + } + }, "/Pg is a dictionary, not a reference to a page"}, + {"dangling /Pg", "<< /Type /StructElem /S /P /Pg 99 0 R >>", func(t *testing.T, e *StructElem) { + if e.Page != 0 || e.PgRef != "99 0 R" { + t.Errorf("Page %d PgRef %q, want 0 and 99 0 R", e.Page, e.PgRef) + } + }, "/Pg 99 0 R does not resolve to an object"}, + {"/Alt not a text string", "<< /Type /StructElem /S /Figure /Alt 5 >>", func(t *testing.T, e *StructElem) { + if e.Alt == nil { + t.Error("Alt = nil, want the raw value shown") + } + }, "/Alt is an integer that does not decode as a text string"}, + {"empty /E and /T and /Lang read", "<< /Type /StructElem /S /Span /E () /T (Intro) /Lang (fr) >>", func(t *testing.T, e *StructElem) { + if e.E == nil || *e.E != "" || e.Title != "Intro" || e.Lang != "fr" { + t.Errorf("E %v T %q Lang %q, want present-but-empty E, T Intro, Lang fr", e.E, e.Title, e.Lang) + } + if e.Alt != nil || e.ActualText != nil { + t.Errorf("Alt %v ActualText %v, want nil when absent", e.Alt, e.ActualText) + } + }, ""}, + {"/Type other than StructElem with an /S", "<< /Type /Foo /S /P >>", func(t *testing.T, e *StructElem) { + if e.S != "P" { + t.Errorf("S = %q, want P", e.S) + } + }, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", rawObj{10, tc.elem})) + if len(r.Elements) != 1 || r.Elements[0].Element == nil { + t.Fatalf("root kids = %v, want one element", kindsOf(r.Elements)) + } + e := r.Elements[0].Element + tc.check(t, e) + if tc.err == "" && e.Err != "" || !strings.Contains(e.Err, tc.err) { + t.Errorf("Err = %q, want %q", e.Err, tc.err) + } + }) + } +} + +// pageParentTreePDF is a one-page tagged document whose page carries +// pageExtra and whose root carries rootExtra; element 10 is a /P on page 1 +// claiming MCID 4, element 11 a /Span with no page. +func pageParentTreePDF(rootExtra, pageExtra string, objs ...rawObj) []byte { + all := []rawObj{ + {1, "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>"}, + {5, "<< /Type /StructTreeRoot /K [10 0 R 11 0 R] " + rootExtra + " >>"}, + {10, "<< /Type /StructElem /S /P /Pg 3 0 R /K 4 >>"}, + {11, "<< /Type /StructElem /S /Span >>"}, + } + all = append(all, structPagesObjs(1, nil, []string{pageExtra})...) + return rawPDF(append(all, objs...)...) +} + +func TestGetStructure_PageStructParentsErrors(t *testing.T) { + cases := []struct { + name string + rootExtra string + pageExtra string + objs []rawObj + err string + }{ + {"not an integer", "/ParentTree 6 0 R", "/StructParents (x)", []rawObj{{6, "<< /Nums [] >>"}}, "/StructParents is a string, not an integer"}, + {"no ParentTree", "", "/StructParents 0", nil, "/StructParents 0 but the document has no ParentTree"}, + {"value not an array", "/ParentTree 6 0 R", "/StructParents 0", []rawObj{{6, "<< /Nums [0 5] >>"}}, "ParentTree key 0 is an integer, not an array"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, pageParentTreePDF(tc.rootExtra, tc.pageExtra, tc.objs...)) + p := mustPage(t, r, 1) + if p.ParentTreeErr != tc.err { + t.Errorf("ParentTreeErr = %q, want %q", p.ParentTreeErr, tc.err) + } + if p.ParentTree == nil || len(p.ParentTree) != 0 { + t.Errorf("ParentTree = %v, want an empty non-nil slice", p.ParentTree) + } + }) + } +} + +func TestGetStructure_ParentTreeSlotShapes(t *testing.T) { + r := rawStructure(t, pageParentTreePDF("/ParentTree 6 0 R", "/StructParents 0", + rawObj{6, "<< /Nums [0 [5 99 0 R 9001 0 R null 10 0 R]] >>"}, + )) + p := mustPage(t, r, 1) + want := []ParentTreeSlot{ + {MCID: 0, Err: "value is an integer, not a reference to an element"}, + {MCID: 1, ElementRef: "99 0 R", Err: "99 0 R does not resolve to an object"}, + {MCID: 2, ElementRef: "9001 0 R", Err: "9001 0 R is a stream, not an element"}, + {MCID: 3}, + {MCID: 4, ElementRef: "10 0 R", S: "P"}, + } + if !slices.Equal(p.ParentTree, want) { + t.Errorf("ParentTree = %+v\nwant %+v", p.ParentTree, want) + } + if got := alignmentMCIDs(p); !slices.Equal(got, []int{0, 1, 2, 4}) { + t.Errorf("alignment MCIDs = %v, want [0 1 2 4] (error slots kept, null slot dropped)", got) + } + for _, mcid := range []int{0, 1, 2} { + if a := alignmentRow(t, p, mcid); a.ParentTreeElement != "" || a.Consistent { + t.Errorf("MCID %d row = %+v, want no ParentTree element and not consistent", mcid, a) + } + } + if a := alignmentRow(t, p, 4); a.ParentTreeElement != "10 0 R" || a.ContentCount != 0 || a.Consistent { + t.Errorf("MCID 4 row = %+v, want ParentTree 10 0 R, no content, not consistent", a) + } +} + +func TestGetStructure_OtherParentTreeValueShapes(t *testing.T) { + r := rawStructure(t, pageParentTreePDF("/ParentTree 6 0 R", "/StructParents 0", + rawObj{6, "<< /Nums [0 [] 7 [10 0 R] 8 11 0 R 9 5] >>"}, + )) + want := []ParentTreeEntry{ + {Key: 7, Err: "value is an array, but no page's /StructParents names this key"}, + {Key: 8, ElementRef: "11 0 R", S: "Span"}, + {Key: 9, Err: "value is an integer, not a reference to an element"}, + } + if !slices.Equal(r.OtherParentTree, want) { + t.Errorf("OtherParentTree = %+v\nwant %+v", r.OtherParentTree, want) + } +} + +func TestGetStructure_ContentsShapes(t *testing.T) { + cases := []struct { + name string + contents string + pageErr string + streams int + }{ + {"array element not a reference", "[5]", "contents array element 0 is not an indirect reference", 0}, + {"not a reference or array", "5", "unexpected Contents type", 0}, + {"dangling stream ref", "[99 0 R 20 0 R]", "", 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := rawStructure(t, rawPDF( + rawObj{1, rawCatalog}, + rawObj{2, "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"}, + rawObj{3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents " + tc.contents + " >>"}, + rawObj{4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 21 0 R >>"}, + rawObj{20, rawStream("/P <> BDC EMC")}, + rawObj{21, rawStream("/P <> BDC EMC")}, + )) + p1 := mustPage(t, r, 1) + if !strings.Contains(p1.Err, tc.pageErr) || (tc.pageErr == "") != (p1.Err == "") { + t.Errorf("page 1 Err = %q, want %q", p1.Err, tc.pageErr) + } + if len(p1.Streams) != tc.streams { + t.Fatalf("page 1 streams = %+v, want %d", p1.Streams, tc.streams) + } + if tc.streams == 2 { + if p1.Streams[0].Ref != "99 0 R" || p1.Streams[0].Err == "" { + t.Errorf("stream 0 = %+v, want 99 0 R with an error", p1.Streams[0]) + } + if p1.Streams[1].Err != "" || len(p1.ContentMCIDs) != 1 { + t.Errorf("stream 1 = %+v, MCIDs %+v; want the next stream scanned", p1.Streams[1], p1.ContentMCIDs) + } + } + p2 := mustPage(t, r, 2) + if len(p2.ContentMCIDs) != 1 || p2.ContentMCIDs[0].MCID != 3 { + t.Errorf("page 2 ContentMCIDs = %+v, want MCID 3 (page 2 still scanned)", p2.ContentMCIDs) + } + }) + } +} + +func TestGetStructure_NamedPropertyListLookup(t *testing.T) { + cases := []struct { + name string + pagesRes string + resources string + mcid int // -1 for an error row, -2 for no row + err string + }{ + {"no /Resources", "", "", -1, "property list /MC0 not found in /Resources /Properties"}, + {"no /Properties", "", "<< /Font << >> >>", -1, "property list /MC0 not found"}, + {"name not in /Properties", "", "<< /Properties << /MC1 << /MCID 1 >> >> >>", -1, "property list /MC0 not found"}, + {"entry not a dictionary", "", "<< /Properties << /MC0 5 >> >>", -1, "property list /MC0 is an integer, not a dictionary"}, + {"entry without /MCID", "", "<< /Properties << /MC0 << /X 1 >> >> >>", -2, ""}, + {"indirect /Properties and entry", "", "<< /Properties 40 0 R >>", 6, ""}, + {"inherited from /Pages", "/Resources << /Properties << /MC0 << /MCID 8 >> >> >>", "", 8, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res := "" + if tc.resources != "" { + res = "/Resources " + tc.resources + } + r := rawStructure(t, rawPDF( + rawObj{1, rawCatalog}, + rawObj{2, "<< /Type /Pages /Kids [3 0 R] /Count 1 " + tc.pagesRes + " >>"}, + rawObj{3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 20 0 R " + res + " >>"}, + rawObj{20, rawStream("/Span /MC0 BDC EMC")}, + rawObj{40, "<< /MC0 41 0 R >>"}, + rawObj{41, "<< /MCID 6 >>"}, + )) + p := mustPage(t, r, 1) + switch { + case tc.mcid == -2: + if len(p.ContentMCIDs) != 0 { + t.Errorf("ContentMCIDs = %+v, want none", p.ContentMCIDs) + } + case len(p.ContentMCIDs) != 1: + t.Errorf("ContentMCIDs = %+v, want one row", p.ContentMCIDs) + default: + c := p.ContentMCIDs[0] + if c.MCID != tc.mcid || c.Tag != "Span" || !strings.Contains(c.Err, tc.err) || (tc.err == "") != (c.Err == "") { + t.Errorf("row = %+v, want MCID %d tag Span error %q", c, tc.mcid, tc.err) + } + } + }) + } +} + +func TestGetStructure_CloseBeforeTreeWalkCancels(t *testing.T) { + ins, doc := openUnvalidated(t, threePagePDF()) + w := newStructureWalker(doc.PDFContext) + var calls []int + w.beforeLock = func(page int) { + calls = append(calls, page) + if page == 0 { + if err := ins.Close("raw"); err != nil { + t.Errorf("Close: %v", err) + } + } + } + r, err := doc.buildStructure(w) + if !errors.Is(err, context.Canceled) || r != nil { + t.Errorf("report %v err %v, want nil and an error satisfying errors.Is(err, context.Canceled)", r, err) + } + if !slices.Equal(calls, []int{0}) { + t.Errorf("beforeLock calls = %v, want [0]", calls) + } +} + +func TestNodeIDToRef(t *testing.T) { + cases := map[string]string{ + "obj:0:12": "12 0 R", + "obj:3:7": "7 3 R", + "root": "root", + "": "", + } + for in, want := range cases { + if got := nodeIDToRef(in); got != want { + t.Errorf("nodeIDToRef(%q) = %q, want %q", in, got, want) + } + } +} + +func TestJoinErr(t *testing.T) { + cases := []struct{ a, b, want string }{ + {"", "b", "b"}, + {"a", "b", "a; b"}, + {"", "", ""}, + } + for _, tc := range cases { + if got := joinErr(tc.a, tc.b); got != tc.want { + t.Errorf("joinErr(%q, %q) = %q, want %q", tc.a, tc.b, got, tc.want) + } + } +} diff --git a/internal/pdfcore/structure_test.go b/internal/pdfcore/structure_test.go new file mode 100644 index 0000000..91e6f14 --- /dev/null +++ b/internal/pdfcore/structure_test.go @@ -0,0 +1,898 @@ +package pdfcore + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "unicode/utf8" +) + +// openStructureFixture opens a committed testdata file under tab "s". +func openStructureFixture(t *testing.T, name string) *Inspector { + t.Helper() + ins := NewInspector() + if _, err := ins.Open("s", filepath.Join(testdataDir(t), name)); err != nil { + t.Fatalf("Open %s: %v", name, err) + } + t.Cleanup(func() { _ = ins.Close("s") }) + return ins +} + +func getStructure(t *testing.T, ins *Inspector, tabID string) *StructureReport { + t.Helper() + r, err := ins.GetStructure(tabID) + if err != nil { + t.Fatalf("GetStructure: %v", err) + } + if r == nil { + t.Fatal("GetStructure returned a nil report") + } + return r +} + +// rawStructure runs GetStructure on an in-memory document read without +// pdfcpu validation. +func rawStructure(t *testing.T, pdf []byte) *StructureReport { + t.Helper() + ins, _ := openUnvalidated(t, pdf) + return getStructure(t, ins, "raw") +} + +// eachKid calls fn on every kid of kids and of their elements, depth first. +func eachKid(kids []*StructKid, fn func(k *StructKid)) { + for _, k := range kids { + fn(k) + if k.Element != nil { + eachKid(k.Element.Kids, fn) + } + } +} + +// findElement returns the first element whose /S is s. +func findElement(r *StructureReport, s string) *StructElem { + var found *StructElem + eachKid(r.Elements, func(k *StructKid) { + if found == nil && k.Element != nil && k.Element.S == s { + found = k.Element + } + }) + return found +} + +func mustElement(t *testing.T, r *StructureReport, s string) *StructElem { + t.Helper() + e := findElement(r, s) + if e == nil { + t.Fatalf("no element with /S %s in the report", s) + } + return e +} + +// kidsWithError returns every kid whose Kind is "error". +func kidsWithError(r *StructureReport) []*StructKid { + var out []*StructKid + eachKid(r.Elements, func(k *StructKid) { + if k.Kind == "error" { + out = append(out, k) + } + }) + return out +} + +// kidHasError reports whether a kid, or the element it carries, has an error. +func kidHasError(k *StructKid) bool { + return k.Kind == "error" || k.Err != "" || (k.Element != nil && k.Element.Err != "") +} + +func kindsOf(kids []*StructKid) []string { + out := make([]string, len(kids)) + for i, k := range kids { + out[i] = k.Kind + if k.Element != nil { + out[i] += ":" + k.Element.S + } + } + return out +} + +func alignmentRow(t *testing.T, p *StructPage, mcid int) MCIDAlignment { + t.Helper() + for _, a := range p.Alignment { + if a.MCID == mcid { + return a + } + } + t.Fatalf("page %d has no alignment row for MCID %d: %+v", p.PageNum, mcid, p.Alignment) + return MCIDAlignment{} +} + +func alignmentMCIDs(p *StructPage) []int { + out := make([]int, len(p.Alignment)) + for i, a := range p.Alignment { + out[i] = a.MCID + } + return out +} + +func mustPage(t *testing.T, r *StructureReport, n int) *StructPage { + t.Helper() + for _, p := range r.Pages { + if p.PageNum == n { + return p + } + } + t.Fatalf("page %d missing from the report (%d pages)", n, len(r.Pages)) + return nil +} + +func intPtrIs(p *int, v int) bool { return p != nil && *p == v } + +// --------------------------------------------------------------------------- +// Clean tagged fixture +// --------------------------------------------------------------------------- + +func TestGetStructure_CleanFileTree(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "structure.pdf"), "s") + + if !r.HasStructTree { + t.Error("HasStructTree = false, want true") + } + if r.Marked == nil || !*r.Marked { + t.Errorf("Marked = %v, want true", r.Marked) + } + if r.DocLang != "en-US" { + t.Errorf("DocLang = %q, want en-US", r.DocLang) + } + if !intPtrIs(r.ParentTreeNextKey, 3) { + t.Errorf("ParentTreeNextKey = %v, want 3", r.ParentTreeNextKey) + } + if !slices.Equal(r.RoleMap, []RoleMapEntry{{From: "Heading1", To: "H1"}}) { + t.Errorf("RoleMap = %+v, want [{Heading1 H1}]", r.RoleMap) + } + if len(r.Problems) != 0 { + t.Errorf("Problems = %+v, want none", r.Problems) + } + if !r.HasParentTree || r.ParentTreeRef == "" { + t.Errorf("HasParentTree = %v, ParentTreeRef = %q; want true and a ref", r.HasParentTree, r.ParentTreeRef) + } + if r.RootRef == "" { + t.Error("RootRef is empty for an indirect /StructTreeRoot") + } + + if len(r.Elements) != 1 || r.Elements[0].Kind != "element" || r.Elements[0].Element.S != "Document" { + t.Fatalf("top-level kids = %v, want one Document element", kindsOf(r.Elements)) + } + doc := r.Elements[0].Element + if got, want := kindsOf(doc.Kids), []string{"element:Heading1", "element:Figure", "element:Sect", "element:Link"}; !slices.Equal(got, want) { + t.Fatalf("Document kids = %v, want %v", got, want) + } + + h1 := doc.Kids[0].Element + if h1.MappedS != "H1" || h1.Page != 1 || h1.PageInherited || h1.Lang != "en" || h1.Ref == "" { + t.Errorf("Heading1 = %+v, want MappedS H1, own page 1, Lang en, a ref", *h1) + } + if len(h1.Kids) != 1 || h1.Kids[0].Kind != "mcid" || !intPtrIs(h1.Kids[0].MCID, 0) || h1.Kids[0].Page != 1 { + t.Errorf("Heading1 kids = %+v, want one mcid kid 0 on page 1", h1.Kids) + } + + fig := doc.Kids[1].Element + if fig.Page != 1 || fig.Alt == nil || *fig.Alt != "Company logo" { + t.Errorf("Figure page %d, Alt %v; want page 1, Alt \"Company logo\"", fig.Page, fig.Alt) + } + if len(fig.Kids) != 1 || fig.Kids[0].Kind != "mcid" || !intPtrIs(fig.Kids[0].MCID, 1) { + t.Errorf("Figure kids = %+v, want one mcid kid 1", fig.Kids) + } + + sect := doc.Kids[2].Element + if sect.Page != 2 { + t.Errorf("Sect page = %d, want 2", sect.Page) + } + if got := kindsOf(sect.Kids); !slices.Equal(got, []string{"element:P"}) { + t.Fatalf("Sect kids = %v, want [element:P]", got) + } + p := sect.Kids[0].Element + if p.Page != 2 || !p.PageInherited || p.PgRef != "" { + t.Errorf("P page %d inherited %v PgRef %q; want page 2 inherited from Sect, no own /Pg", p.Page, p.PageInherited, p.PgRef) + } + if p.ActualText == nil || *p.ActualText == "" { + t.Errorf("P ActualText = %v, want the decoded text", p.ActualText) + } else if at := *p.ActualText; !utf8.ValidString(at) || strings.ContainsRune(at, 0) || strings.ContainsRune(at, 0xFEFF) { + t.Errorf("P ActualText = %q, want UTF-16BE decoded (no NUL, no BOM)", at) + } + if len(p.Kids) != 1 || p.Kids[0].Kind != "mcr" || !intPtrIs(p.Kids[0].MCID, 0) || p.Kids[0].Page != 2 || p.Kids[0].StmRef != "" { + t.Errorf("P kids = %+v, want one mcr kid MCID 0 on page 2 with no /Stm", p.Kids) + } + + link := doc.Kids[3].Element + if link.Page != 2 { + t.Errorf("Link page = %d, want 2", link.Page) + } + if got := kindsOf(link.Kids); !slices.Equal(got, []string{"mcid", "objr"}) { + t.Fatalf("Link kids = %v, want [mcid objr]", got) + } + if !intPtrIs(link.Kids[0].MCID, 1) || link.Kids[0].Page != 2 { + t.Errorf("Link mcid kid = %+v, want MCID 1 on page 2", *link.Kids[0]) + } + if link.Kids[1].ObjType != "Annot/Link" || link.Kids[1].ObjRef == "" { + t.Errorf("Link objr kid ObjType %q ObjRef %q, want Annot/Link and a ref", link.Kids[1].ObjType, link.Kids[1].ObjRef) + } + + want := StructureCounts{Elements: 6, MCIDKids: 3, MCRKids: 1, OBJRKids: 1, ErrorKids: 0, ContentMCIDs: 4} + if r.Counts != want { + t.Errorf("Counts = %+v, want %+v", r.Counts, want) + } + + eachKid(r.Elements, func(k *StructKid) { + if kidHasError(k) { + t.Errorf("unexpected error on a clean file: kid %+v element %+v", *k, k.Element) + } + }) +} + +func TestGetStructure_CleanFilePagesAndAlignment(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "structure.pdf"), "s") + if len(r.Pages) != 2 { + t.Fatalf("got %d pages, want 2", len(r.Pages)) + } + h1 := mustElement(t, r, "Heading1") + fig := mustElement(t, r, "Figure") + para := mustElement(t, r, "P") + link := mustElement(t, r, "Link") + + p1 := mustPage(t, r, 1) + if !intPtrIs(p1.StructParents, 0) || p1.ParentTreeErr != "" || p1.Err != "" { + t.Errorf("page 1 StructParents %v ParentTreeErr %q Err %q; want 0, no errors", p1.StructParents, p1.ParentTreeErr, p1.Err) + } + if len(p1.Streams) != 1 { + t.Fatalf("page 1 has %d streams, want 1", len(p1.Streams)) + } + if s := p1.Streams[0]; s.Opens != 2 || s.Closes != 2 || s.DepthAtEnd != 0 || s.MinDepth != 0 || s.Err != "" || s.Ref == "" { + t.Errorf("page 1 stream = %+v, want 2 opens, 2 closes, depth 0 at end, min depth 0, a ref", s) + } + if len(p1.ContentMCIDs) != 2 { + t.Fatalf("page 1 ContentMCIDs = %+v, want 2", p1.ContentMCIDs) + } + for i, want := range []struct { + mcid int + tag string + }{{0, "Heading1"}, {1, "P"}} { + c := p1.ContentMCIDs[i] + if c.MCID != want.mcid || c.Tag != want.tag || c.Depth != 1 || c.ParentMCID != nil || c.Stream != 0 || c.Err != "" { + t.Errorf("page 1 content MCID %d = %+v, want MCID %d tag %s depth 1 no parent stream 0", i, c, want.mcid, want.tag) + } + } + if got := alignmentMCIDs(p1); !slices.Equal(got, []int{0, 1}) { + t.Errorf("page 1 alignment MCIDs = %v, want [0 1]", got) + } + for mcid, el := range map[int]*StructElem{0: h1, 1: fig} { + a := alignmentRow(t, p1, mcid) + if !a.Consistent || a.ContentCount != 1 || a.ParentTreeElement != el.Ref || !slices.Equal(a.TreeElements, []string{el.Ref}) { + t.Errorf("page 1 MCID %d row = %+v, want consistent and naming %s (%s)", mcid, a, el.Ref, el.S) + } + } + if a := alignmentRow(t, p1, 0); a.ParentTreeS != "Heading1" { + t.Errorf("page 1 MCID 0 ParentTreeS = %q, want Heading1", a.ParentTreeS) + } + if len(p1.ParentTree) != 2 || p1.ParentTree[0].ElementRef != h1.Ref || p1.ParentTree[1].ElementRef != fig.Ref { + t.Errorf("page 1 ParentTree = %+v, want slots 0 -> %s, 1 -> %s", p1.ParentTree, h1.Ref, fig.Ref) + } + + p2 := mustPage(t, r, 2) + if !intPtrIs(p2.StructParents, 1) || p2.ParentTreeErr != "" || p2.Err != "" { + t.Errorf("page 2 StructParents %v ParentTreeErr %q Err %q; want 1, no errors", p2.StructParents, p2.ParentTreeErr, p2.Err) + } + if len(p2.Streams) != 2 { + t.Fatalf("page 2 has %d streams, want 2", len(p2.Streams)) + } + var named *ContentMCID + for i := range p2.ContentMCIDs { + if p2.ContentMCIDs[i].MCID == 1 { + named = &p2.ContentMCIDs[i] + } + } + if named == nil || named.Stream != 1 || named.Err != "" { + t.Errorf("page 2 MCID 1 from /MC0 = %+v, want it recorded from stream 1 with no error", named) + } + if got := alignmentMCIDs(p2); !slices.Equal(got, []int{0, 1}) { + t.Errorf("page 2 alignment MCIDs = %v, want [0 1]", got) + } + for mcid, el := range map[int]*StructElem{0: para, 1: link} { + a := alignmentRow(t, p2, mcid) + if !a.Consistent || a.ParentTreeElement != el.Ref || !slices.Equal(a.TreeElements, []string{el.Ref}) { + t.Errorf("page 2 MCID %d row = %+v, want consistent and naming %s (%s)", mcid, a, el.Ref, el.S) + } + } + + if want := []ParentTreeEntry{{Key: 2, ElementRef: link.Ref, S: "Link"}}; !slices.Equal(r.OtherParentTree, want) { + t.Errorf("OtherParentTree = %+v, want %+v", r.OtherParentTree, want) + } + if len(r.Problems) != 0 { + t.Errorf("the two-level ParentTree produced problems: %+v", r.Problems) + } +} + +func TestGetStructure_CleanFileDoesNotFillStreamCache(t *testing.T) { + ins := openStructureFixture(t, "structure.pdf") + getStructure(t, ins, "s") + doc, err := ins.GetDocument("s") + if err != nil { + t.Fatal(err) + } + doc.streamMu.Lock() + n := len(doc.streamCache) + doc.streamMu.Unlock() + if n != 0 { + t.Errorf("streamCache holds %d entries after GetStructure, want 0", n) + } +} + +// --------------------------------------------------------------------------- +// Broken fixture: each planted finding in its place +// --------------------------------------------------------------------------- + +func TestGetStructure_BrokenFileReportsEachFinding(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "structure-broken.pdf"), "s") + + t.Run("cycle in a /K array", func(t *testing.T) { + var cycle *StructKid + for _, k := range kidsWithError(r) { + if strings.Contains(k.Err, "already listed") { + cycle = k + } + } + if cycle == nil { + t.Fatalf("no \"already listed\" error kid; error kids: %+v", kidsWithError(r)) + } + seen := map[string]int{} + eachKid(r.Elements, func(k *StructKid) { + if k.Element != nil && k.Element.Ref != "" { + seen[k.Element.Ref]++ + } + }) + for ref, n := range seen { + if n != 1 { + t.Errorf("element %s listed %d times, want once", ref, n) + } + } + }) + + t.Run("/Pg naming a non-page", func(t *testing.T) { + var found bool + eachKid(r.Elements, func(k *StructKid) { + if e := k.Element; e != nil && e.PgRef != "" && e.Page == 0 && e.Err != "" { + found = true + } + }) + if !found { + t.Error("no element with PgRef set, Page 0 and an error") + } + }) + + t.Run("missing /S", func(t *testing.T) { + var found bool + eachKid(r.Elements, func(k *StructKid) { + if e := k.Element; e != nil && e.S == "" && e.Err != "" { + found = true + } + }) + if !found { + t.Error("no element with an empty S and an error") + } + }) + + p1 := mustPage(t, r, 1) + t.Run("page 1 alignment", func(t *testing.T) { + if a := alignmentRow(t, p1, 0); a.ContentCount != 2 || a.Consistent { + t.Errorf("MCID 0 row = %+v, want ContentCount 2 and not consistent", a) + } + if a := alignmentRow(t, p1, 2); a.ParentTreeElement != "" || a.Consistent { + t.Errorf("MCID 2 row = %+v, want no ParentTree element and not consistent", a) + } + a := alignmentRow(t, p1, 1) + if a.ContentCount != 0 || a.ParentTreeElement == "" || slices.Contains(a.TreeElements, a.ParentTreeElement) || a.Consistent { + t.Errorf("MCID 1 row = %+v, want no content, a ParentTree element that no tree claimant matches, not consistent", a) + } + }) + + t.Run("page 1 marked-content balance", func(t *testing.T) { + if len(p1.Streams) != 2 { + t.Fatalf("page 1 has %d streams, want 2", len(p1.Streams)) + } + if p1.Streams[0].MinDepth != -1 { + t.Errorf("first stream MinDepth = %d, want -1 (EMC with nothing open)", p1.Streams[0].MinDepth) + } + if p1.Streams[1].DepthAtEnd != 1 { + t.Errorf("last stream DepthAtEnd = %d, want 1 (BDC with no EMC)", p1.Streams[1].DepthAtEnd) + } + }) + + t.Run("named property list missing", func(t *testing.T) { + var found bool + for _, p := range r.Pages { + for _, c := range p.ContentMCIDs { + if c.Err != "" && strings.Contains(c.Err, "MC9") { + found = true + } + } + } + if !found { + t.Error("no content MCID with an error naming /MC9") + } + }) + + t.Run("page 2 /StructParents names a missing key", func(t *testing.T) { + p2 := mustPage(t, r, 2) + if !intPtrIs(p2.StructParents, 5) || !strings.Contains(p2.ParentTreeErr, "5") { + t.Errorf("page 2 StructParents %v ParentTreeErr %q, want 5 and an error naming key 5", p2.StructParents, p2.ParentTreeErr) + } + }) + + t.Run("key outside /Limits", func(t *testing.T) { + var found bool + for _, p := range r.Problems { + if strings.Contains(p.Message, "key 1") && strings.Contains(p.Message, "/Limits") { + found = true + } + } + if !found { + t.Errorf("no problem for key 1 outside /Limits [0 0]: %+v", r.Problems) + } + }) + + t.Run("everything else still reported", func(t *testing.T) { + if len(r.Pages) != 2 { + t.Errorf("got %d pages, want 2", len(r.Pages)) + } + if r.Counts.Elements < 3 { + t.Errorf("Counts.Elements = %d, want the other elements walked", r.Counts.Elements) + } + if r.Counts.ErrorKids < 1 { + t.Errorf("Counts.ErrorKids = %d, want the cycle counted", r.Counts.ErrorKids) + } + }) +} + +// --------------------------------------------------------------------------- +// Untagged and empty roots +// --------------------------------------------------------------------------- + +// requireJSONArrays marshals v and requires every named key to be a JSON +// array, never null. +func requireJSONArrays(t *testing.T, where string, v any, keys ...string) { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + for _, k := range keys { + raw, ok := m[k] + if !ok { + t.Errorf("%s: JSON has no %q key", where, k) + continue + } + if !strings.HasPrefix(string(raw), "[") { + t.Errorf("%s: %q = %s, want an array", where, k, raw) + } + } +} + +func requireReportArrays(t *testing.T, r *StructureReport) { + t.Helper() + requireJSONArrays(t, "report", r, "roleMap", "elements", "pages", "otherParentTree", "problems") + for _, p := range r.Pages { + requireJSONArrays(t, fmt.Sprintf("page %d", p.PageNum), p, "parentTree", "streams", "contentMcids", "alignment") + } +} + +func TestGetStructure_UntaggedFile(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "untagged.pdf"), "s") + if r.HasStructTree { + t.Error("HasStructTree = true for a file with no /StructTreeRoot") + } + if len(r.Elements) != 0 { + t.Errorf("Elements = %v, want empty", kindsOf(r.Elements)) + } + if len(r.Pages) == 0 { + t.Error("no pages listed") + } + requireReportArrays(t, r) +} + +func TestGetStructure_EmptyRoot(t *testing.T) { + r := getStructure(t, openStructureFixture(t, "tagged.pdf"), "s") + if !r.HasStructTree || r.HasParentTree { + t.Errorf("HasStructTree %v HasParentTree %v, want true / false", r.HasStructTree, r.HasParentTree) + } + if len(r.Elements) != 0 || len(r.Problems) != 0 { + t.Errorf("Elements %v Problems %+v, want both empty (absent /K and /ParentTree are not problems)", kindsOf(r.Elements), r.Problems) + } + requireReportArrays(t, r) +} + +func TestGetStructure_UnknownTab(t *testing.T) { + _, err := NewInspector().GetStructure("missing") + if !errors.Is(err, ErrDocumentNotFound) { + t.Errorf("err = %v, want ErrDocumentNotFound", err) + } +} + +// --------------------------------------------------------------------------- +// Caps, cycles and shapes pdfcpu rejects at open (in-memory, unvalidated) +// --------------------------------------------------------------------------- + +// chainBottom follows the only kid of each element from the top and returns +// the element at the end of the chain and its depth (1 = top). +func chainBottom(t *testing.T, r *StructureReport) (*StructElem, int) { + t.Helper() + if len(r.Elements) != 1 || r.Elements[0].Element == nil { + t.Fatalf("top-level kids = %v, want one element", kindsOf(r.Elements)) + } + e, depth := r.Elements[0].Element, 1 + for len(e.Kids) == 1 && e.Kids[0].Element != nil { + e, depth = e.Kids[0].Element, depth+1 + } + return e, depth +} + +func TestGetStructure_ChainPastDepthCapGivesOneErrorKid(t *testing.T) { + r := rawStructure(t, structChainPDF(257)) + bottom, depth := chainBottom(t, r) + if depth != 256 { + t.Errorf("walked %d levels, want 256", depth) + } + if len(bottom.Kids) != 1 || bottom.Kids[0].Kind != "error" || + !strings.Contains(bottom.Kids[0].Err, "structure tree deeper than 256 levels; subtree not walked") { + t.Errorf("kids of the 256th element = %+v, want one error kid with the depth message", bottom.Kids) + } + if got := len(kidsWithError(r)); got != 1 { + t.Errorf("got %d error kids, want 1", got) + } + if r.Counts.Elements != 256 || r.Counts.ErrorKids != 1 { + t.Errorf("Counts = %+v, want 256 elements and 1 error kid", r.Counts) + } +} + +func TestGetStructure_ChainAtDepthCapHasNoError(t *testing.T) { + r := rawStructure(t, structChainPDF(256)) + bottom, depth := chainBottom(t, r) + if depth != 256 || len(bottom.Kids) != 0 { + t.Errorf("walked %d levels, bottom kids %+v; want 256 levels and no kid below", depth, bottom.Kids) + } + if n := len(kidsWithError(r)); n != 0 || r.Counts.ErrorKids != 0 { + t.Errorf("got %d error kids (Counts.ErrorKids %d), want 0", n, r.Counts.ErrorKids) + } +} + +func TestGetStructure_SharedChildListedOnce(t *testing.T) { + r := rawStructure(t, structSharedChildPDF(1000)) + spans, listed := 0, 0 + eachKid(r.Elements, func(k *StructKid) { + if k.Element != nil && k.Element.S == "Span" { + spans++ + } + if k.Kind == "error" && strings.Contains(k.Err, "5000 0 R already listed; not walked again") { + listed++ + } + }) + if spans != 1 || listed != 999 { + t.Errorf("shared Span listed %d times with %d already-listed kids, want 1 and 999", spans, listed) + } + if r.Counts.Elements != 1001 || r.Counts.ErrorKids != 999 { + t.Errorf("Counts = %+v, want 1001 elements and 999 error kids", r.Counts) + } +} + +func TestGetStructure_SharedIndirectKArrayWalkedOnce(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K [10 0 R 11 0 R 12 0 R] >>", + rawObj{7, "[0 1 << /Type /MCR /MCID 2 >>]"}, + rawObj{10, "<< /Type /StructElem /S /P /Pg 3 0 R /K 7 0 R >>"}, + rawObj{11, "<< /Type /StructElem /S /P /Pg 3 0 R /K 7 0 R >>"}, + rawObj{12, "<< /Type /StructElem /S /P /Pg 3 0 R /K 7 0 R >>"}, + )) + if len(r.Elements) != 3 { + t.Fatalf("root kids = %v, want 3 elements", kindsOf(r.Elements)) + } + if got := kindsOf(r.Elements[0].Element.Kids); !slices.Equal(got, []string{"mcid", "mcid", "mcr"}) { + t.Errorf("first element kids = %v, want the shared array walked once", got) + } + for _, k := range r.Elements[1:] { + kids := k.Element.Kids + if len(kids) != 1 || kids[0].Kind != "error" || !strings.Contains(kids[0].Err, "/K array 7 0 R already listed; not walked again") { + t.Errorf("element %s kids = %+v, want one error kid naming 7 0 R as already listed", k.Element.Ref, kids) + } + } + if c := r.Counts; c.MCIDKids != 2 || c.MCRKids != 1 || c.ErrorKids != 2 { + t.Errorf("Counts = %+v, want 2 MCID kids, 1 MCR kid, 2 error kids", c) + } + for _, a := range mustPage(t, r, 1).Alignment { + if !slices.Equal(a.TreeElements, []string{"10 0 R"}) { + t.Errorf("MCID %d tree elements = %v, want only 10 0 R", a.MCID, a.TreeElements) + } + } +} + +func TestGetStructure_SingleObjectKBackReference(t *testing.T) { + r := rawStructure(t, structSingleBackRefPDF()) + errs := kidsWithError(r) + if len(errs) != 1 || !strings.Contains(errs[0].Err, "element 10 0 R already listed; not walked again") { + t.Errorf("error kids = %+v, want one naming 10 0 R as already listed", errs) + } + if r.Counts.Elements != 2 { + t.Errorf("Counts.Elements = %d, want 2", r.Counts.Elements) + } +} + +func TestGetStructure_KidShapesRejectedByPdfcpuAreErrorKids(t *testing.T) { + r := rawStructure(t, structRejectedKidsPDF()) + div := mustElement(t, r, "Div") + if len(div.Kids) != 9 { + t.Fatalf("Div kids = %v, want 9 (the direct null skipped)", kindsOf(div.Kids)) + } + for i, what := range []string{"string", "name", "real", "dangling ref"} { + if div.Kids[i].Kind != "error" || div.Kids[i].Err == "" { + t.Errorf("kid %d (%s) = %+v, want an error kid", i, what, *div.Kids[i]) + } + } + if !strings.Contains(div.Kids[3].Err, "99 0 R") { + t.Errorf("dangling kid error %q does not name 99 0 R", div.Kids[3].Err) + } + if !kidHasError(div.Kids[4]) { + t.Errorf("/Type /Foo with no /S kid = %+v, want an error on it", *div.Kids[4]) + } + if !kidHasError(div.Kids[5]) { + t.Errorf("MCR with no /MCID kid = %+v, want an error on it", *div.Kids[5]) + } + if div.Kids[6].Kind != "error" || div.Kids[6].Err == "" { + t.Errorf("negative MCID kid = %+v, want an error kid", *div.Kids[6]) + } + if k := div.Kids[7]; k.Kind != "mcid" || !intPtrIs(k.MCID, 0) || k.Page != 1 { + t.Errorf("kid 7 = %+v, want mcid 0 on page 1 after the bad kids", *k) + } + if k := div.Kids[8]; k.Element == nil || k.Element.S != "P" || len(k.Element.Kids) != 1 || !intPtrIs(k.Element.Kids[0].MCID, 1) { + t.Errorf("kid 8 = %+v, want the P element with MCID 1", *k) + } +} + +func TestGetStructure_RoleMapAndSValues(t *testing.T) { + r := rawStructure(t, structRoleMapPDF()) + want := []RoleMapEntry{{"A", "B"}, {"B", "A"}, {"Heading1", "H1"}} + if !slices.Equal(r.RoleMap, want) { + t.Errorf("RoleMap = %+v, want %+v in key order", r.RoleMap, want) + } + if len(r.Elements) != 4 { + t.Fatalf("top-level kids = %v, want 4 elements", kindsOf(r.Elements)) + } + if e := r.Elements[0].Element; e == nil || e.S != "A" || e.Err == "" { + t.Errorf("element with a RoleMap cycle = %+v, want S A and an error", e) + } + if e := r.Elements[1].Element; e == nil || e.MappedS != "H1" || e.Err != "" { + t.Errorf("Heading1 element = %+v, want MappedS H1 and no error", e) + } + if e := r.Elements[2].Element; e == nil || e.S != "" || !strings.Contains(e.Err, "/S is an empty name") { + t.Errorf("empty /S element = %+v, want S \"\" and \"/S is an empty name\"", e) + } + if e := r.Elements[3].Element; e == nil || e.S != "" || e.Err == "" || e.Title != "untitled" { + t.Errorf("element with no /S = %+v, want S \"\", an error and Title untitled", e) + } +} + +func TestGetStructure_DirectElementNeverConsistent(t *testing.T) { + r := rawStructure(t, structDirectElementPDF()) + if len(r.Elements) != 1 || r.Elements[0].Element == nil || r.Elements[0].Element.Ref != "" { + t.Fatalf("top-level kids = %+v, want one direct element with no ref", r.Elements) + } + a := alignmentRow(t, mustPage(t, r, 1), 0) + if a.ContentCount != 1 || a.ParentTreeElement != "" || !slices.Equal(a.TreeElements, []string{"(direct)"}) || a.Consistent { + t.Errorf("MCID 0 row = %+v, want content 1, null slot, tree [(direct)], not consistent", a) + } +} + +// --------------------------------------------------------------------------- +// Locking and close +// --------------------------------------------------------------------------- + +func threePagePDF() []byte { + all := []rawObj{{1, rawCatalog}} + all = append(all, structPagesObjs(3, []string{ + "/P <> BDC EMC", "/P <> BDC EMC", "/P <> BDC EMC", + }, nil)...) + return rawPDF(all...) +} + +func TestGetStructure_ConcurrentWithOtherReads(t *testing.T) { + ins, _ := openUnvalidated(t, threePagePDF()) + var wg sync.WaitGroup + errs := make(chan error, 30) + for i := 0; i < 10; i++ { + wg.Add(3) + go func() { + defer wg.Done() + if _, err := ins.GetStructure("raw"); err != nil { + errs <- fmt.Errorf("GetStructure: %w", err) + } + }() + go func() { + defer wg.Done() + if _, err := ins.GetContentStream("raw", "obj:0:9001"); err != nil { + errs <- fmt.Errorf("GetContentStream: %w", err) + } + }() + go func() { + defer wg.Done() + if _, err := ins.GetPageIndex("raw"); err != nil { + errs <- fmt.Errorf("GetPageIndex: %w", err) + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + +func TestGetStructure_CloseDuringTheWalkCancels(t *testing.T) { + ins, doc := openUnvalidated(t, threePagePDF()) + w := newStructureWalker(doc.PDFContext) + var calls []int + w.beforeLock = func(page int) { + calls = append(calls, page) + if !doc.pdfMu.TryLock() { + t.Errorf("pdfMu held when beforeLock(%d) ran", page) + } else { + doc.pdfMu.Unlock() + } + if page == 2 { + if err := ins.Close("raw"); err != nil { + t.Errorf("Close: %v", err) + } + } + } + _, err := doc.buildStructure(w) + if !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want one satisfying errors.Is(err, context.Canceled)", err) + } + if len(calls) == 0 || calls[0] != 0 { + t.Errorf("beforeLock calls = %v, want 0 first (before phase 1)", calls) + } + if slices.Contains(calls, 3) { + t.Errorf("beforeLock calls = %v, want the walk stopped before page 3", calls) + } +} + +func TestGetStructure_BuildWithoutCloseVisitsEveryPage(t *testing.T) { + _, doc := openUnvalidated(t, threePagePDF()) + w := newStructureWalker(doc.PDFContext) + var calls []int + w.beforeLock = func(page int) { calls = append(calls, page) } + r, err := doc.buildStructure(w) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(calls, []int{0, 1, 2, 3}) { + t.Errorf("beforeLock calls = %v, want [0 1 2 3]", calls) + } + if len(r.Pages) != 3 || r.HasStructTree { + t.Errorf("got %d pages, HasStructTree %v; want 3 pages and no tree", len(r.Pages), r.HasStructTree) + } + for _, p := range r.Pages { + if len(p.ContentMCIDs) != 1 || p.ContentMCIDs[0].MCID != 0 { + t.Errorf("page %d ContentMCIDs = %+v, want MCID 0 with no structure tree", p.PageNum, p.ContentMCIDs) + } + } +} + +func TestGetStructure_MCIDKidWithNoPageCarriesError(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /P /K [0 << /Type /MCR /MCID 1 >>] >>"}, + )) + p := mustElement(t, r, "P") + if len(p.Kids) != 2 { + t.Fatalf("P kids = %v, want 2", kindsOf(p.Kids)) + } + for _, k := range p.Kids { + if k.Page != 0 || !strings.Contains(k.Err, "not attributed to a page") { + t.Errorf("kid %+v, want page 0 and an error saying the MCID is not attributed to a page", *k) + } + } +} + +func TestGetStructure_MCIDKidUnderNonPagePgSaysPgIsNotAPage(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K [10 0 R] >>", + rawObj{10, "<< /Type /StructElem /S /P /Pg 11 0 R /K 0 >>"}, + rawObj{11, "<< /Foo 1 >>"}, + )) + p := mustElement(t, r, "P") + if len(p.Kids) != 1 { + t.Fatalf("P kids = %v, want 1", kindsOf(p.Kids)) + } + if k := p.Kids[0]; k.Page != 0 || !strings.Contains(k.Err, "not a page") || !strings.Contains(k.Err, "not attributed to a page") { + t.Errorf("kid %+v, want page 0 and an error saying the nearest /Pg is not a page", *k) + } +} + +func TestGetStructure_MCIDKidsUnderRootCarryError(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K [0 << /Type /MCR /MCID 1 /Pg 3 0 R >>] >>")) + if len(r.Elements) != 2 { + t.Fatalf("root kids = %v, want 2", kindsOf(r.Elements)) + } + for _, k := range r.Elements { + if !strings.Contains(k.Err, "structure tree root") { + t.Errorf("kid %+v, want an error saying no element owns the MCID", *k) + } + } + if rows := mustPage(t, r, 1).Alignment; len(rows) != 0 { + t.Errorf("page 1 alignment = %+v, want none", rows) + } +} + +func TestGetStructure_OBJRToNonDictionaryCarriesError(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /Link /Pg 3 0 R /K [<< /Type /OBJR /Obj 11 0 R >>] >>"}, + rawObj{11, "42"}, + )) + link := mustElement(t, r, "Link") + if len(link.Kids) != 1 || link.Kids[0].Kind != "objr" || !strings.Contains(link.Kids[0].Err, "not a dictionary or stream") { + t.Fatalf("Link kids = %+v, want one objr kid whose error says /Obj is not a dictionary or stream", link.Kids) + } +} + +func TestGetStructure_RoleMapToEmptyNameCarriesError(t *testing.T) { + r := rawStructure(t, structTreePDF("<< /Type /StructTreeRoot /RoleMap << /Foo / >> /K 10 0 R >>", + rawObj{10, "<< /Type /StructElem /S /Foo >>"}, + )) + e := mustElement(t, r, "Foo") + if e.MappedS != "" || !strings.Contains(e.Err, "empty name") { + t.Errorf("Foo MappedS %q, Err %q; want no mapping and an error naming the empty name", e.MappedS, e.Err) + } +} + +func TestGetStructure_MalformedMarkInfoIsAProblem(t *testing.T) { + for _, tc := range []struct{ name, markInfo, want string }{ + {"Marked not a boolean", "<< /Marked /true >>", "/Marked is a name"}, + {"MarkInfo not a dictionary", "[true]", "/MarkInfo is an array"}, + } { + t.Run(tc.name, func(t *testing.T) { + objs := []rawObj{{1, "<< /Type /Catalog /Pages 2 0 R /MarkInfo " + tc.markInfo + " >>"}} + r := rawStructure(t, rawPDF(append(objs, structPagesObjs(1, nil, nil)...)...)) + if r.Marked != nil { + t.Errorf("Marked = %v, want nil", *r.Marked) + } + var found bool + for _, p := range r.Problems { + found = found || (p.Where == "catalog" && strings.Contains(p.Message, tc.want)) + } + if !found { + t.Errorf("Problems = %+v, want a catalog problem containing %q", r.Problems, tc.want) + } + }) + } +} + +func TestGetStructure_PageTreeWalkStoppedEarlyIsAProblem(t *testing.T) { + ins, doc := openUnvalidated(t, rawPDF(append([]rawObj{{1, rawCatalog}}, structPagesObjs(2, nil, nil)...)...)) + pw := newPageWalker(doc.PDFContext) + pw.walk() + if _, err := doc.pageIndex.get(func() (*pageTree, error) { + return &pageTree{entries: pw.entries[:1], leaves: pw.leaves[:1], err: errors.New("pdf parsing panic: boom")}, nil + }); err != nil { + t.Fatal(err) + } + r := getStructure(t, ins, "raw") + if len(r.Pages) != 1 { + t.Fatalf("got %d pages, want the 1 page reached before the failure", len(r.Pages)) + } + if len(r.Problems) != 1 || r.Problems[0].Where != "page tree" || !strings.Contains(r.Problems[0].Message, "boom") { + t.Errorf("Problems = %+v, want one page tree problem carrying the walk error", r.Problems) + } +} diff --git a/testdata/generate_test.go b/testdata/generate_test.go index 041df2a..bc9b04a 100644 --- a/testdata/generate_test.go +++ b/testdata/generate_test.go @@ -715,6 +715,61 @@ func taggedPDFContent() []byte { }, "") } +// structurePDFContent is a clean tagged two-page document for `dump +// structure`: a RoleMap, a two-level ParentTree, an inline and a named +// property list, an MCR kid and an OBJR kid for a Link annotation. Every +// ParentTree slot, content MCID and tree MCID line up. +func structurePDFContent() []byte { + return complAssemblePDF([]string{ + "1 0 obj\n<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /Lang (en-US) /StructTreeRoot 6 0 R >>\nendobj\n", + "2 0 obj\n<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>\nendobj\n", + "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 7 0 R /StructParents 0 /Resources << >> >>\nendobj\n", + "4 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [8 0 R 9 0 R] /StructParents 1 /Resources << /Properties << /MC0 << /MCID 1 >> >> >> /Annots [5 0 R] >>\nendobj\n", + "5 0 obj\n<< /Type /Annot /Subtype /Link /Rect [72 600 200 620] /Border [0 0 0] /StructParent 2 >>\nendobj\n", + "6 0 obj\n<< /Type /StructTreeRoot /K 13 0 R /ParentTree 10 0 R /ParentTreeNextKey 3 /RoleMap << /Heading1 /H1 >> >>\nendobj\n", + complStreamObj(7, "", "/Heading1 <> BDC 0 0 m EMC /P <> BDC 0 0 m EMC"), + complStreamObj(8, "", "/P <> BDC 0 0 m EMC"), + complStreamObj(9, "", "/Span /MC0 BDC 0 0 m EMC"), + "10 0 obj\n<< /Kids [11 0 R 12 0 R] >>\nendobj\n", + "11 0 obj\n<< /Limits [0 1] /Nums [0 [14 0 R 15 0 R] 1 [17 0 R 18 0 R]] >>\nendobj\n", + "12 0 obj\n<< /Limits [2 2] /Nums [2 18 0 R] >>\nendobj\n", + "13 0 obj\n<< /Type /StructElem /S /Document /P 6 0 R /K [14 0 R 15 0 R 16 0 R 18 0 R] >>\nendobj\n", + "14 0 obj\n<< /Type /StructElem /S /Heading1 /P 13 0 R /Pg 3 0 R /Lang (en) /K 0 >>\nendobj\n", + "15 0 obj\n<< /Type /StructElem /S /Figure /P 13 0 R /Pg 3 0 R /Alt (Company logo) /K 1 >>\nendobj\n", + "16 0 obj\n<< /Type /StructElem /S /Sect /P 13 0 R /Pg 4 0 R /K [17 0 R] >>\nendobj\n", + "17 0 obj\n<< /Type /StructElem /S /P /P 16 0 R /ActualText /K << /Type /MCR /MCID 0 /Pg 4 0 R >> >>\nendobj\n", + "18 0 obj\n<< /Type /StructElem /S /Link /P 13 0 R /Pg 4 0 R /K [1 << /Type /OBJR /Obj 5 0 R >>] >>\nendobj\n", + }, "") +} + +// structureBrokenPDFContent plants one finding of each kind `dump structure` +// reports, all of them shapes pdfcpu's relaxed open accepts: a /K array +// naming its parent, a /Pg on a non-page, an element with no /S, MCID 0 +// marked twice and MCID 2 with no ParentTree slot on page 1, a ParentTree +// slot 1 no element claims, an EMC with nothing open and a BDC never closed, +// a missing named property list, /StructParents naming a missing key, and a +// ParentTree key outside its leaf's /Limits. +func structureBrokenPDFContent() []byte { + return complAssemblePDF([]string{ + "1 0 obj\n<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /Lang (en-US) /StructTreeRoot 6 0 R >>\nendobj\n", + "2 0 obj\n<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>\nendobj\n", + "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [7 0 R 8 0 R] /StructParents 0 /Resources << /Properties << /MC0 << /MCID 3 >> >> >> >>\nendobj\n", + "4 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 9 0 R /StructParents 5 /Resources << >> >>\nendobj\n", + "5 0 obj\n<< /Note (not a page) >>\nendobj\n", + "6 0 obj\n<< /Type /StructTreeRoot /K 12 0 R /ParentTree 10 0 R /ParentTreeNextKey 2 >>\nendobj\n", + complStreamObj(7, "", "EMC /P <> BDC 0 0 m EMC /P <> BDC 0 0 m EMC"), + complStreamObj(8, "", "/P <> BDC 0 0 m EMC /Span /MC9 BDC 0 0 m EMC /Artifact <> BDC 0 0 m"), + complStreamObj(9, "", "/P <> BDC 0 0 m EMC"), + "10 0 obj\n<< /Kids [11 0 R] >>\nendobj\n", + "11 0 obj\n<< /Limits [0 0] /Nums [0 [13 0 R 15 0 R] 1 16 0 R] >>\nendobj\n", + "12 0 obj\n<< /Type /StructElem /S /Document /P 6 0 R /K [13 0 R 14 0 R 15 0 R 16 0 R] >>\nendobj\n", + "13 0 obj\n<< /Type /StructElem /S /P /P 12 0 R /Pg 3 0 R /K 0 >>\nendobj\n", + "14 0 obj\n<< /Type /StructElem /S /Sect /P 12 0 R /Pg 3 0 R /K [12 0 R] >>\nendobj\n", + "15 0 obj\n<< /Type /StructElem /S /Figure /P 12 0 R /Pg 5 0 R /Alt (Chart) >>\nendobj\n", + "16 0 obj\n<< /Type /StructElem /P 12 0 R /Pg 4 0 R /K 0 >>\nendobj\n", + }, "") +} + // pdfaCleanContent builds a minimal PDF/A-1b file that veraPDF --flavour 1b // passes: a PDF/A-identification XMP packet (pdfaid part 1 / conformance B), a // document /ID, no fonts, no device color, and a blank page. Provenance: @@ -920,6 +975,8 @@ func TestGenerateFixtures(t *testing.T) { {"untagged.pdf", untaggedPDFContent}, {"tagged.pdf", taggedPDFContent}, {"pdfa-1b-clean.pdf", pdfaCleanContent}, + {"structure.pdf", structurePDFContent}, + {"structure-broken.pdf", structureBrokenPDFContent}, } for _, fx := range complFixtures { t.Run(fx.name, func(t *testing.T) { diff --git a/testdata/structure-broken.pdf b/testdata/structure-broken.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fbf72e04beb202bd88967aa0903ead9afd020136 GIT binary patch literal 1856 zcmb_dO>f#j5WV|X%ms-YQhN;sEJa9(0V(362+|YkVGE1n7HqUE66v4pq5q)$Bc0jB zVDF~tr6@|mF#G21d-F_VJ{aAz4{fjU>-UeJ9)kdiFJHZf2k>9s4g&myE50jI@aH@g zM+mWD0sfTBZ_oL*0BWQ+VgpK^1oZphPk5d}Q{?xbmUrMUvA$L>Qiw%SDClT_e%}-M zh9(hUeWDrUkF(7YRxRD$rr)2DOmWH;a=0C*&)wt1Rtnu}=!#9opNccA0_^Xw5V~=w z@p6T2IH#-U^e%G@XM=IXn!gao;w0B709|-4i-V9VyS$wyfzcs(9zcsED1XXJ6950e zGS(z7ik9pmi=Ccb4vs2=XN3~b%+Ui52Q>4pA|ts{CurH`j{#NFVw%BZC@_)}*Wp&0 ziL)Aux0*+!Z3#XRd8%H~0vdi)Qt-VuoF)i?68ck8Fu)r;4H7&Qz#jhfA>H(XbJ-{| z(Ha*=m2s6(p*w1M;5n{z65AXe(eN!_E8=OQfs2~uTxCUGg>eayZ5E-h_dX%#?k<|v zyo56$VH8`;$1N7RQZ`MpeMUQug5u84PJ8|TikWON+7Ud=faz@3VBBo8d3axVsX2N^$Zh9^FIpJOxFU54}eH-JS%4OYnSujh1zWoJhW#DVq zuExfAZ2^w5^d#}=SRwprWfSb8Vhj&?OKe^TD4Y$$+2qfu6s6OZjBzP_StTTZdrJ}jyof6f#j5WV|X%mpbur1lzrUIs5TqxRg997a3D{^^OG*D^5B&%2AL-1l z0eekvwPb|HyYIc7c{9+Nk0#gbvuAaF|M~gLV&FjW{A%_4U_XBBIoNR`lWmcLJx@}8 z02dc5z@8=I?Mwbr04<~kz5yvp4ub*McS)YY70<7~EkA+1#Q0i13eFcrAt6uzgMr2K z4KWd5lTj1d(`<8qRaY(d=<^B16lE?^!+ATsZ*M2Hf~%!kF5hIyuy}`+gX;t4LKwLU zFPFIN0HLC(wCcSF4A~2QC`z%$gX2N*Y-eJU^dE>W_`m}HIyueefd<}b6O@XFrwG@fVXQ}Fam*sCnyOyvDT?iZpT)YF< zL3zVMY`D-_r02TaMp>xJO&-Gr_7s?|ZB;5V2rw!S+Dj*ZLihY#PWcCPQB$$yvt+MM ze9MzfmZzAY+v?m$KT#!*%y&Fb<#z}?s@p*dp6o39w6)*I1Y+VG%-C=g<30y=FrLN8 zk0kmfGhY%<1ywniDrhjey*lF=twq&FBPW<>4gJe~l4H+d=^LWkNGh2}DBVqr`g;NB ze`U9`FVu_U+PiF*k$W6qSmXP0r@pV4>Ox?ZsTi!7ZX2dN{PaX;u5zLnOgtB;=QN%R z)vo%cb~9_&jJJG;W&?XvtjitsHC9Lu67*417%=_f0(`q=4fvBa5dP%O6uqZRc^qpv z)20}y9k$(72Jj?HOM%1m670=Z!c|=Cu&f_|leSR!p4!~;~iHw#&AkFVTjN8J;Vp~&ck6e zRUNBoW-<9h^ dL^SI0pMsuM*aiog2&t;Y!d_@~I^&0l^%sb8L4N=M literal 0 HcmV?d00001 diff --git a/tests/dump-structure/dump_structure_test.go b/tests/dump-structure/dump_structure_test.go new file mode 100644 index 0000000..14d3764 --- /dev/null +++ b/tests/dump-structure/dump_structure_test.go @@ -0,0 +1,222 @@ +package dump_structure_test + +import ( + "encoding/json" + "slices" + "strings" + "testing" +) + +func TestDumpStructureJSON_CleanFile(t *testing.T) { + r, raw := dumpStructureJSON(t, "structure.pdf") + + var top map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &top); err != nil { + t.Fatal(err) + } + for _, k := range reportKeys { + if _, ok := top[k]; !ok { + t.Errorf("--json has no %q key", k) + } + } + + if !r.HasStructTree || !r.HasParentTree { + t.Errorf("hasStructTree %v hasParentTree %v, want both true", r.HasStructTree, r.HasParentTree) + } + if len(r.Elements) != 1 || r.Elements[0].Element == nil || r.Elements[0].Element.S != "Document" { + t.Fatalf("elements = %v, want one Document element", kidS(r.Elements)) + } + doc := r.Elements[0].Element + if got, want := kidS(doc.Kids), []string{"Heading1", "Figure", "Sect", "Link"}; !slices.Equal(got, want) { + t.Errorf("Document kids = %v, want %v", got, want) + } + pages := map[string]int{} + eachKid(r.Elements, func(k *structKid) { + if k.Element != nil { + pages[k.Element.S] = k.Element.Page + } + if k.Kind == "error" { + t.Errorf("error kid on a clean file: %+v", *k) + } + }) + for s, want := range map[string]int{"Heading1": 1, "Figure": 1, "Sect": 2, "P": 2, "Link": 2} { + if pages[s] != want { + t.Errorf("%s page = %d, want %d", s, pages[s], want) + } + } + + if len(r.Pages) != 2 { + t.Fatalf("pages = %d, want 2", len(r.Pages)) + } + for _, n := range []int{1, 2} { + p := page(t, r, n) + var mcids []int + for _, a := range p.Alignment { + mcids = append(mcids, a.MCID) + if !a.Consistent { + t.Errorf("page %d MCID %d not consistent: %+v", n, a.MCID, a) + } + } + if !slices.Equal(mcids, []int{0, 1}) { + t.Errorf("page %d alignment MCIDs = %v, want [0 1]", n, mcids) + } + } + if len(r.OtherParentTree) != 1 || r.OtherParentTree[0].Key != 2 || r.OtherParentTree[0].S != "Link" { + t.Errorf("otherParentTree = %+v, want key 2 -> Link", r.OtherParentTree) + } + if len(r.Problems) != 0 { + t.Errorf("problems = %+v, want none", r.Problems) + } +} + +func TestDumpStructureJSON_BrokenFileShowsFindings(t *testing.T) { + r, _ := dumpStructureJSON(t, "structure-broken.pdf") + + var cycle bool + eachKid(r.Elements, func(k *structKid) { + if k.Kind == "error" && strings.Contains(k.Errmsg, "already listed") { + cycle = true + } + }) + if !cycle { + t.Error("no \"already listed\" error kid for the /K cycle") + } + if p2 := page(t, r, 2); p2.ParentTreeErr == "" { + t.Error("page 2 has no parentTreeError for /StructParents 5") + } + var inconsistent bool + for _, a := range page(t, r, 1).Alignment { + if !a.Consistent { + inconsistent = true + } + } + if !inconsistent { + t.Error("page 1 has no inconsistent alignment row") + } + if len(r.Problems) == 0 { + t.Error("problems is empty, want the /Limits finding") + } +} + +func TestDumpStructureJSON_UntaggedFile(t *testing.T) { + r, raw := dumpStructureJSON(t, "untagged.pdf") + if r.HasStructTree { + t.Error("hasStructTree = true for a file with no /StructTreeRoot") + } + var top map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &top); err != nil { + t.Fatal(err) + } + for _, k := range []string{"elements", "roleMap", "otherParentTree", "problems", "pages"} { + if !strings.HasPrefix(string(top[k]), "[") { + t.Errorf("%q = %s, want an array", k, top[k]) + } + } + if len(r.Pages) == 0 { + t.Error("no pages listed") + } +} + +func TestDumpStructureJSON_PrettyIsIndented(t *testing.T) { + stdout, stderr, code := runCLI(t, "dump", "structure", "--json", "--pretty", fixture(t, "structure.pdf")) + if code != 0 { + t.Fatalf("exit = %d, want 0\nstderr: %s", code, stderr) + } + if !strings.Contains(stdout, "\n \"") { + t.Errorf("--pretty output is not indented:\n%s", stdout) + } + var r structureReport + if err := json.Unmarshal([]byte(stdout), &r); err != nil { + t.Errorf("--pretty output is not JSON: %v", err) + } +} + +func TestDumpStructurePlain_CleanFile(t *testing.T) { + out := dumpStructurePlain(t, "structure.pdf") + for _, prefix := range []string{"Structure tree:", "Document", "Page 1"} { + if !hasLinePrefix(out, prefix) { + t.Errorf("no line starting %q in plain output:\n%s", prefix, out) + } + } + for i, b := range []byte(out) { + if b > 0x7e || (b < 0x20 && b != '\n' && b != '\t') { + t.Errorf("plain output byte %d is 0x%02x, want printable ASCII", i, b) + break + } + } +} + +func TestDumpStructurePlain_BrokenFileMarksInconsistentRows(t *testing.T) { + out := dumpStructurePlain(t, "structure-broken.pdf") + var starred bool + for line := range strings.SplitSeq(out, "\n") { + if strings.HasSuffix(strings.TrimRight(line, " "), " *") { + starred = true + } + } + if !starred { + t.Errorf("no alignment row marked with *:\n%s", out) + } + if !hasLinePrefix(out, "* = ") { + t.Errorf("no \"* = ...\" legend line:\n%s", out) + } +} + +func TestDumpStructurePlain_UntaggedFile(t *testing.T) { + out := dumpStructurePlain(t, "untagged.pdf") + if !strings.Contains(out, "No structure tree (/StructTreeRoot absent)") { + t.Errorf("plain output does not say the structure tree is absent:\n%s", out) + } +} + +func TestDumpStructure_HelpListsCommand(t *testing.T) { + _, stderr, _ := runCLI(t, "--help") + if !hasLinePrefix(stderr, "dump structure [--json] [--pretty] ") { + t.Errorf("--help has no `dump structure [--json] [--pretty] ` line:\n%s", stderr) + } + if !strings.Contains(stderr, "pdfdebug dump structure file.pdf") { + t.Errorf("--help Examples has no `pdfdebug dump structure file.pdf` line:\n%s", stderr) + } +} + +func TestDumpStructure_UsageErrors(t *testing.T) { + path := fixture(t, "untagged.pdf") + cases := map[string][]string{ + "no file": {"dump", "structure"}, + "two files": {"dump", "structure", path, path}, + "unknown flag": {"dump", "structure", "--page", "1", path}, + } + for name, args := range cases { + t.Run(name, func(t *testing.T) { + stdout, stderr, code := runCLI(t, args...) + if code != 1 { + t.Errorf("exit %d, want 1", code) + } + if !strings.Contains(stderr, "Usage: pdfdebug dump structure") { + t.Errorf("stderr must carry the dump structure usage line, got: %s", stderr) + } + if stdout != "" { + t.Errorf("a usage error writes nothing to stdout, got: %s", stdout) + } + }) + } +} + +func TestDumpStructure_MissingFileExits2WithJSONError(t *testing.T) { + missing := t.TempDir() + "/does-not-exist.pdf" + stdout, stderr, code := runCLI(t, "dump", "structure", missing) + if code != 2 { + t.Errorf("exit %d, want 2", code) + } + if stdout != "" { + t.Errorf("stdout = %q, want nothing", stdout) + } + var obj map[string]string + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &obj); err != nil || obj["error"] == "" { + t.Errorf("stderr = %q, want a JSON {\"error\": ...} object", stderr) + } + _, objErr, objCode := runCLI(t, "dump", "objects", missing) + if objCode != code || strings.TrimSpace(objErr) != strings.TrimSpace(stderr) { + t.Errorf("dump structure on a missing file: exit %d stderr %q; dump objects: exit %d stderr %q", code, stderr, objCode, objErr) + } +} diff --git a/tests/dump-structure/go.mod b/tests/dump-structure/go.mod new file mode 100644 index 0000000..b8d1151 --- /dev/null +++ b/tests/dump-structure/go.mod @@ -0,0 +1,3 @@ +module dump-structure-test + +go 1.25 diff --git a/tests/dump-structure/helpers_test.go b/tests/dump-structure/helpers_test.go new file mode 100644 index 0000000..2e80d13 --- /dev/null +++ b/tests/dump-structure/helpers_test.go @@ -0,0 +1,238 @@ +// Acceptance harness for `pdfdebug dump structure`. +// +// Black-box: the CLI is built once and run as a subprocess over committed +// fixtures in testdata/ (structure.pdf, structure-broken.pdf, untagged.pdf). +// Covers output shape, exit codes and usage errors only; walk edge cases, +// caps and the marked-content scan live in the pdfcore unit tests. +// +// Run: cd tests/dump-structure && go test -count=1 ./... +package dump_structure_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" +) + +// projectRoot walks up from the test directory to the main module's go.mod. +func projectRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get working directory: %v", err) + } + for { + if content, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil { + if strings.Contains(string(content), "module unidoc-pdf-debugger") { + return dir + } + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("could not find project root (no go.mod with module unidoc-pdf-debugger found)") + } + dir = parent + } +} + +// fixture returns the absolute path of a file in testdata/, failing when it +// is missing. +func fixture(t *testing.T, name string) string { + t.Helper() + p := filepath.Join(projectRoot(t), "testdata", name) + if _, err := os.Stat(p); err != nil { + t.Fatalf("fixture %s missing: %v", name, err) + } + return p +} + +var ( + cliBuildOnce sync.Once + cliBinPath string + cliBuildErr string +) + +// buildCLI compiles the CLI once per test package and returns the binary path. +func buildCLI(t *testing.T) string { + t.Helper() + cliBuildOnce.Do(func() { + binName := "pdfdebug" + if runtime.GOOS == "windows" { + binName += ".exe" + } + tmpDir, err := os.MkdirTemp("", "pdfdebug-dump-structure-") + if err != nil { + cliBuildErr = "failed to create temp dir: " + err.Error() + return + } + binPath := filepath.Join(tmpDir, binName) + cmd := exec.Command("go", "build", "-o", binPath, "./cmd/cli/") + cmd.Dir = projectRoot(t) + if out, err := cmd.CombinedOutput(); err != nil { + cliBuildErr = "failed to build CLI binary: " + err.Error() + "\n" + string(out) + return + } + cliBinPath = binPath + }) + if cliBuildErr != "" { + t.Fatalf("%s", cliBuildErr) + } + return cliBinPath +} + +// runCLI runs the CLI with args and returns stdout, stderr and the exit code. +func runCLI(t *testing.T, args ...string) (stdout, stderr string, exitCode int) { + t.Helper() + cmd := exec.Command(buildCLI(t), args...) + var outBuf, errBuf strings.Builder + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + if err := cmd.Run(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("failed to run CLI: %v", err) + } + exitCode = exitErr.ExitCode() + } + return outBuf.String(), errBuf.String(), exitCode +} + +// structElem is the subset of an element object this suite reads. +type structElem struct { + Ref string `json:"ref"` + S string `json:"s"` + Page int `json:"page"` + Kids []*structKid `json:"kids"` + Errmsg string `json:"error"` +} + +// structKid is one /K entry. +type structKid struct { + Kind string `json:"kind"` + Element *structElem `json:"element"` + MCID *int `json:"mcid"` + Page int `json:"page"` + ObjType string `json:"objType"` + Errmsg string `json:"error"` +} + +// alignment is one per-MCID row of a page. +type alignment struct { + MCID int `json:"mcid"` + ContentCount int `json:"contentCount"` + ParentTreeElement string `json:"parentTreeElement"` + TreeElements []string `json:"treeElements"` + Consistent bool `json:"consistent"` +} + +// structPage is one page row. +type structPage struct { + PageNum int `json:"pageNum"` + StructParents *int `json:"structParents"` + ParentTreeErr string `json:"parentTreeError"` + Alignment []alignment `json:"alignment"` +} + +// parentTreeEntry is one ParentTree key no page names. +type parentTreeEntry struct { + Key int `json:"key"` + ElementRef string `json:"elementRef"` + S string `json:"s"` +} + +// problem is one top-level finding. +type problem struct { + Where string `json:"where"` + Message string `json:"message"` +} + +// structureReport is the subset of `dump structure --json` this suite reads. +type structureReport struct { + HasStructTree bool `json:"hasStructTree"` + HasParentTree bool `json:"hasParentTree"` + Elements []*structKid `json:"elements"` + Pages []*structPage `json:"pages"` + OtherParentTree []parentTreeEntry `json:"otherParentTree"` + Problems []problem `json:"problems"` +} + +// reportKeys is every top-level key of the --json object. +var reportKeys = []string{ + "hasStructTree", "rootRef", "hasParentTree", "parentTreeRef", "marked", + "docLang", "parentTreeNextKey", "roleMap", "elements", "pages", + "otherParentTree", "problems", "counts", +} + +// dumpStructureJSON runs `dump structure --json` on a fixture, requires exit +// 0 and a JSON object on stdout, and returns it with the raw output. +func dumpStructureJSON(t *testing.T, name string) (structureReport, string) { + t.Helper() + stdout, stderr, code := runCLI(t, "dump", "structure", "--json", fixture(t, name)) + if code != 0 { + t.Fatalf("dump structure --json %s exit = %d, want 0\nstderr: %s", name, code, stderr) + } + var r structureReport + if err := json.Unmarshal([]byte(stdout), &r); err != nil { + t.Fatalf("--json output is not a structure report object: %v\n%s", err, stdout) + } + return r, stdout +} + +// dumpStructurePlain runs `dump structure` in plain text and requires exit 0. +func dumpStructurePlain(t *testing.T, name string) string { + t.Helper() + stdout, stderr, code := runCLI(t, "dump", "structure", fixture(t, name)) + if code != 0 { + t.Fatalf("dump structure %s exit = %d, want 0\nstderr: %s", name, code, stderr) + } + return stdout +} + +// eachKid calls fn on every kid of kids and of their elements, depth first. +func eachKid(kids []*structKid, fn func(k *structKid)) { + for _, k := range kids { + fn(k) + if k.Element != nil { + eachKid(k.Element.Kids, fn) + } + } +} + +// kidS returns the /S names of the element kids in kids, in order. +func kidS(kids []*structKid) []string { + var out []string + for _, k := range kids { + if k.Element != nil { + out = append(out, k.Element.S) + } + } + return out +} + +// page returns page n of the report, failing when it is missing. +func page(t *testing.T, r structureReport, n int) *structPage { + t.Helper() + for _, p := range r.Pages { + if p.PageNum == n { + return p + } + } + t.Fatalf("page %d missing from the report", n) + return nil +} + +// hasLinePrefix reports whether any line of out, with leading spaces removed, +// starts with prefix. +func hasLinePrefix(out, prefix string) bool { + for line := range strings.SplitSeq(out, "\n") { + if strings.HasPrefix(strings.TrimLeft(line, " "), prefix) { + return true + } + } + return false +} From a9bf45fac4501acb3f34dca18956643ca4da57a4 Mon Sep 17 00:00:00 2001 From: Ade Anom A Date: Thu, 22 Oct 2026 15:33:04 +0700 Subject: [PATCH 2/3] Fix marked-content scan and structure dump review findings --- cmd/cli/cmd_structure.go | 7 +- cmd/cli/cmd_structure_plain_test.go | 27 +++- docs/cli/structure.md | 14 +- internal/pdfcore/markedcontent.go | 151 ++++++++++++++---- internal/pdfcore/markedcontent_test.go | 32 +++- internal/pdfcore/markedcontent_tokens_test.go | 53 +++++- internal/pdfcore/model.go | 6 +- internal/pdfcore/numbertree.go | 13 +- internal/pdfcore/numbertree_shapes_test.go | 5 +- internal/pdfcore/numbertree_test.go | 33 ++-- internal/pdfcore/stream.go | 24 ++- internal/pdfcore/structure.go | 127 ++++++++++----- internal/pdfcore/structure_shapes_test.go | 85 ++++++++-- 13 files changed, 437 insertions(+), 140 deletions(-) diff --git a/cmd/cli/cmd_structure.go b/cmd/cli/cmd_structure.go index 4a6d163..58a9eeb 100644 --- a/cmd/cli/cmd_structure.go +++ b/cmd/cli/cmd_structure.go @@ -252,11 +252,14 @@ func writeStructPages(b *strings.Builder, pages []*pdfcore.StructPage) { fmt.Fprintf(b, " ParentTree slot %d: error: %s\n", s.MCID, asciiSafe(s.Err)) } } - for _, s := range p.Streams { + for i, s := range p.Streams { + // Depth carries across a page's streams, so only the last stream's + // DepthAtEnd shows a sequence left open. + unclosed := i == len(p.Streams)-1 && s.DepthAtEnd != 0 switch { case s.Err != "": b.WriteString(" stream " + s.Ref + ": error: " + asciiSafe(s.Err) + "\n") - case s.MinDepth < 0 || s.DepthAtEnd != 0: + case s.MinDepth < 0 || unclosed: fmt.Fprintf(b, " stream %s: BDC/BMC %d, EMC %d, depth at end %d, min depth %d\n", s.Ref, s.Opens, s.Closes, s.DepthAtEnd, s.MinDepth) } } diff --git a/cmd/cli/cmd_structure_plain_test.go b/cmd/cli/cmd_structure_plain_test.go index c1c19cd..501ead2 100644 --- a/cmd/cli/cmd_structure_plain_test.go +++ b/cmd/cli/cmd_structure_plain_test.go @@ -148,8 +148,8 @@ func TestStructurePlain_PageErrorLines(t *testing.T) { {PageNum: 3, Streams: []pdfcore.MarkedStream{ {Ref: "20 0 R", Err: "failed to decode stream: bad"}, {Ref: "21 0 R", Opens: 1, Closes: 2, DepthAtEnd: 0, MinDepth: -1}, - {Ref: "22 0 R", Opens: 1, Closes: 0, DepthAtEnd: 1, MinDepth: 0}, {Ref: "23 0 R", Opens: 1, Closes: 1}, + {Ref: "22 0 R", Opens: 1, Closes: 0, DepthAtEnd: 1, MinDepth: 0}, }, ContentMCIDs: []pdfcore.ContentMCID{ {MCID: -1, Tag: "Span", Stream: 1, Line: 2, Col: 3, Err: "property list /MC9 not found"}, {MCID: -1, Stream: 7, Line: 1, Col: 1, Err: "BDC has 0 of its 2 operands"}, @@ -180,6 +180,31 @@ func TestStructurePlain_PageErrorLines(t *testing.T) { } } +func TestStructurePlain_SequenceClosedInALaterStreamIsNotReportedOpen(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{ + {PageNum: 1, Streams: []pdfcore.MarkedStream{ + {Ref: "30 0 R", Opens: 1, DepthAtEnd: 1}, + {Ref: "31 0 R", Closes: 1, MinDepth: 0}, + }}, + {PageNum: 2, Streams: []pdfcore.MarkedStream{ + {Ref: "40 0 R", Opens: 2, Closes: 1, DepthAtEnd: 1}, + {Ref: "41 0 R", Opens: 1, DepthAtEnd: 2, MinDepth: 1}, + }}, + }, + } + out := structurePlain(r) + for _, absent := range []string{"stream 30 0 R", "stream 31 0 R", "stream 40 0 R"} { + if strings.Contains(out, absent) { + t.Errorf("output has %q for a sequence that a later stream closes or that is still counted on the last stream:\n%s", absent, out) + } + } + if want := " stream 41 0 R: BDC/BMC 1, EMC 0, depth at end 2, min depth 1\n"; !strings.Contains(out, want) { + t.Errorf("output missing %q for the page's last stream:\n%s", want, out) + } +} + func TestStructurePlain_LegendOnlyUnderFirstStarredTable(t *testing.T) { inconsistent := []pdfcore.MCIDAlignment{{MCID: 0, ContentCount: 2, TreeElements: []string{}}} r := &pdfcore.StructureReport{ diff --git a/docs/cli/structure.md b/docs/cli/structure.md index ad9a14d..e715268 100644 --- a/docs/cli/structure.md +++ b/docs/cli/structure.md @@ -82,9 +82,14 @@ tree deeper than 256 levels says where it stops. Each page with a `/StructParents` entry, any marked content, an MCID the tree attributes to it, or an error gets a block. The first line has the `/StructParents` key, the number of content streams and the BDC/BMC and EMC -counts. A stream that fails to decode, ends with sequences still open, or -closes one it never opened gets its own line, and so does each BDC and each -ParentTree slot that could not be read. The table has one row per MCID: +counts. A page's content streams are read as one concatenated stream, so a +sequence opened in one stream may close in a later one and a BDC's operands may +start in one stream and end in the next. A stream that fails to decode or +closes a sequence it never opened gets its own line, and so does the page's +last stream when sequences are still open at its end. Each BDC and each +ParentTree slot that could not be read gets a line too. A BDC's tag and +property list are its last two operands; anything before them is ignored. The +table has one row per MCID: - `CONTENT` - how many BDCs in the page's content carry that MCID. - `PARENTTREE` - the element in that slot of the page's ParentTree array. @@ -94,7 +99,8 @@ A row is marked `*` unless the content marks the MCID once, the ParentTree slot names an element, and that element is the only one in `TREE`. The pages that get no block are collapsed into one line. ParentTree keys no page names (an annotation's `/StructParent`, or an orphan) are listed under -"Other ParentTree keys", and malformed ParentTree nodes under "Problems". +"Other ParentTree keys". "Problems" lists malformed ParentTree nodes and pages +that share a `/StructParents` key with an earlier page. MCIDs inside Form XObjects are not aligned: an MCR with `/Stm` is listed in the tree but left out of the page table, and `Do` is not followed. diff --git a/internal/pdfcore/markedcontent.go b/internal/pdfcore/markedcontent.go index a344c73..fe7653f 100644 --- a/internal/pdfcore/markedcontent.go +++ b/internal/pdfcore/markedcontent.go @@ -14,10 +14,14 @@ type mcOpen struct { hasMCID bool } -// mcState is the stack of open marked-content sequences, carried from one -// content stream of a page to the next. +// mcState is the scan state carried from one content stream of a page to the +// next, so the page's streams scan as one concatenated stream: the stack of +// open marked-content sequences and the operands not yet consumed by an +// operator, with the index of the stream each operand came from. type mcState struct { - stack []mcOpen + stack []mcOpen + operands []Token + opStreams []int } // propertyLookup resolves a named property list (BDC with a name operand) in @@ -28,7 +32,8 @@ type propertyLookup func(name string) (mcid pdfcpu_types.Object, err error) // scanMarkedContent counts BMC/BDC/EMC in one stream's tokens into ms and // returns a row for every BDC whose property list carries an /MCID. st is -// updated in place so nesting carries into the next stream. +// updated in place so nesting and pending operands carry into the next +// stream. // // The tokenizer types delimiters and the keywords true, false and null as // operators, so operands are collected by value: <<, >>, [, ], true, false and @@ -37,18 +42,17 @@ type propertyLookup func(name string) (mcid pdfcpu_types.Object, err error) func scanMarkedContent(tokens []Token, streamIdx int, props propertyLookup, st *mcState, ms *MarkedStream) []ContentMCID { var rows []ContentMCID ms.MinDepth = len(st.stack) - var operands []Token for _, tok := range tokens { if tok.Type == "comment" { continue } if tok.Type != "operator" { - operands = append(operands, tok) + st.pushOperand(tok, streamIdx) continue } switch tok.Value { case "<<", ">>", "[", "]", "true", "false", "null": - operands = append(operands, tok) + st.pushOperand(tok, streamIdx) continue } switch tok.Value { @@ -57,11 +61,10 @@ func scanMarkedContent(tokens []Token, streamIdx int, props propertyLookup, st * st.stack = append(st.stack, mcOpen{}) case "BDC": ms.Opens++ - row, ok := bdcRow(operands, tok, props) + row, ok := bdcRow(st.operands, st.opStreams, tok, streamIdx, props) parent := innermostMCID(st.stack) open := mcOpen{} if ok { - row.Stream = streamIdx row.ParentMCID = parent if row.Err == "" { open = mcOpen{mcid: row.MCID, hasMCID: true} @@ -81,12 +84,24 @@ func scanMarkedContent(tokens []Token, streamIdx int, props propertyLookup, st * ms.MinDepth = min(ms.MinDepth, len(st.stack)) } } - operands = operands[:0] + st.clearOperands() } ms.DepthAtEnd = len(st.stack) return rows } +// pushOperand records tok, read from stream streamIdx, as a pending operand. +func (st *mcState) pushOperand(tok Token, streamIdx int) { + st.operands = append(st.operands, tok) + st.opStreams = append(st.opStreams, streamIdx) +} + +// clearOperands drops the pending operands. +func (st *mcState) clearOperands() { + st.operands = st.operands[:0] + st.opStreams = st.opStreams[:0] +} + // innermostMCID returns the MCID of the innermost open sequence that has // one, or nil. func innermostMCID(stack []mcOpen) *int { @@ -99,32 +114,52 @@ func innermostMCID(stack []mcOpen) *int { return nil } -// bdcRow reads a BDC's operands (tag, then an inline dictionary or a property -// list name). ok is false when the property list has no /MCID, so the BDC -// gets no row. Operands that are not a tag name followed by a dictionary or a -// name, a property list that cannot be read, or an /MCID that is not a -// non-negative integer give a row with MCID -1 and Err set. -func bdcRow(operands []Token, op Token, props propertyLookup) (row ContentMCID, ok bool) { - pos := op - if len(operands) > 0 { - pos = operands[0] +// bdcRow reads a BDC's operands. The property list is the last operand +// object (an inline dictionary or a property list name), or, when that is not +// a closed dictionary, a dictionary still open at the operator; the tag is the object before it, and anything earlier +// is ignored. opStreams holds the stream index of each operand token; the +// row's Stream, Line and Col are the tag's, else the first operand's, else the +// operator's (in stream opStream). ok is false when the property list has no +// /MCID, so the BDC gets no row. Fewer than two operand objects, a tag that is +// not a name, a property list that is neither a dictionary nor a name or +// cannot be read, or an /MCID that is not a non-negative integer give a row +// with MCID -1 and Err set. +func bdcRow(operands []Token, opStreams []int, op Token, opStream int, props propertyLookup) (row ContentMCID, ok bool) { + row = ContentMCID{MCID: -1, Stream: opStream, Line: op.Line, Col: op.Col} + at := func(i int) { + row.Stream, row.Line, row.Col = opStreams[i], operands[i].Line, operands[i].Col } - row = ContentMCID{MCID: -1, Line: pos.Line, Col: pos.Col} - if len(operands) > 0 && operands[0].Type == "name" { - row.Tag = strings.TrimPrefix(operands[0].Value, "/") + n := len(operands) + if n == 0 { + row.Err = "BDC has 0 of its 2 operands (a tag and a property list)" + return row, true + } + propStart := operandStart(operands, n-1) + // With no closed dictionary last, a "<<" still open is the property list. + if operands[propStart].Value != "<<" { + if u := unclosedDictStart(operands); u > 0 { + propStart = u + } } - if len(operands) < 2 { - row.Err = fmt.Sprintf("BDC has %d of its 2 operands (a tag and a property list)", len(operands)) + if propStart == 0 { + at(0) + if n == 1 && operands[0].Type == "name" { + row.Tag = strings.TrimPrefix(operands[0].Value, "/") + } + row.Err = "BDC has 1 of its 2 operands (a tag and a property list)" return row, true } - if operands[0].Type != "name" { - row.Err = fmt.Sprintf("BDC operands start with %q, not a tag name; property list not read", operands[0].Value) + tagStart := operandStart(operands, propStart-1) + at(tagStart) + if tagStart != propStart-1 || operands[tagStart].Type != "name" { + row.Err = fmt.Sprintf("BDC operand %q before the property list is not a tag name; property list not read", operands[tagStart].Value) return row, true } - prop := operands[1] + row.Tag = strings.TrimPrefix(operands[tagStart].Value, "/") + prop := operands[propStart:] switch { - case prop.Type == "operator" && prop.Value == "<<": - val, found := inlineMCID(operands[1:]) + case prop[0].Type == "operator" && prop[0].Value == "<<": + val, found := inlineMCID(prop) if !found { return row, false } @@ -136,8 +171,13 @@ func bdcRow(operands []Token, op Token, props propertyLookup) (row ContentMCID, } row.Err = fmt.Sprintf("/MCID %s is not an integer", val.Value) return row, true - case prop.Type == "name": - name := strings.TrimPrefix(prop.Value, "/") + case len(prop) == 1 && prop[0].Type == "name": + // /Properties keys are #xx-decoded by the parser, so the operand is too. + name, err := pdfcpu_types.DecodeName(strings.TrimPrefix(prop[0].Value, "/")) + if err != nil { + row.Err = fmt.Sprintf("property list name %s does not decode: %v", prop[0].Value, err) + return row, true + } mcid, err := props(name) if err != nil { row.Err = err.Error() @@ -153,10 +193,57 @@ func bdcRow(operands []Token, op Token, props propertyLookup) (row ContentMCID, } return row, true } - row.Err = fmt.Sprintf("BDC property list operand %q is not a dictionary or a name", prop.Value) + row.Err = fmt.Sprintf("BDC property list operand %q is not a dictionary or a name", prop[0].Value) return row, true } +// unclosedDictStart returns the index of the outermost "<<" still open at the +// end of toks, or -1 when none is. +func unclosedDictStart(toks []Token) int { + var open []int + for i, t := range toks { + if t.Type != "operator" { + continue + } + switch t.Value { + case "<<", "[": + open = append(open, i) + case ">>", "]": + if len(open) > 0 { + open = open[:len(open)-1] + } + } + } + for _, i := range open { + if toks[i].Value == "<<" { + return i + } + } + return -1 +} + +// operandStart returns the index of the first token of the operand object +// that ends at toks[end]: a dictionary or array is matched back to its +// opening delimiter. An unmatched closing delimiter extends the object to +// index 0. +func operandStart(toks []Token, end int) int { + depth := 0 + for i := end; i >= 0; i-- { + if t := toks[i]; t.Type == "operator" { + switch t.Value { + case ">>", "]": + depth++ + case "<<", "[": + depth-- + } + } + if depth <= 0 { + return i + } + } + return 0 +} + // setMCID stores a non-negative n as the row's MCID, or an error for a // negative one. func setMCID(row *ContentMCID, n int) { diff --git a/internal/pdfcore/markedcontent_test.go b/internal/pdfcore/markedcontent_test.go index 059b167..a141836 100644 --- a/internal/pdfcore/markedcontent_test.go +++ b/internal/pdfcore/markedcontent_test.go @@ -207,12 +207,39 @@ func TestMarkedContentScan_StrayOpenSpanEndsAtNextOperator(t *testing.T) { checkStream(t, p.Streams[0], 1, 1, 0, 0) } -func TestMarkedContentScan_BDCWithoutTagIsAnErrorRow(t *testing.T) { - p := scanPage(t, "<< /Properties << /MCID << /MCID 5 >> >> >>", []string{rawStream("<> BDC EMC")}) +func TestMarkedContentScan_BDCWhoseTagIsNotANameIsAnErrorRow(t *testing.T) { + p := scanPage(t, "<< /Properties << /MCID << /MCID 5 >> >> >>", []string{rawStream("5 <> BDC EMC")}) checkContentMCIDs(t, p, []mcidRow{{-1, "", 0, 1, -1, "not a tag name"}}) checkStream(t, p.Streams[0], 1, 1, 0, 0) } +func TestMarkedContentScan_EscapedPropertyListNameIsDecodedBeforeLookup(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 4 >> >> >>", []string{rawStream("/Span /MC#30 BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{4, "Span", 0, 1, -1, ""}}) +} + +func TestMarkedContentScan_BadEscapeInPropertyListNameIsAnErrorRow(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 4 >> >> >>", []string{rawStream("/Span /MC#3 BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{-1, "Span", 0, 1, -1, "does not decode"}}) +} + +func TestMarkedContentScan_BDCOperandsSplitAcrossStreams(t *testing.T) { + p := scanPage(t, "<< /Properties << /MC0 << /MCID 1 >> >> >>", []string{ + rawStream("/P <>"), + rawStream("BDC EMC /Span"), + rawStream("/MC0 BDC EMC"), + }) + checkContentMCIDs(t, p, []mcidRow{{0, "P", 0, 1, -1, ""}, {1, "Span", 1, 1, -1, ""}}) + if a := alignmentMCIDs(p); len(a) != 2 { + t.Errorf("alignment MCIDs = %v, want 0 and 1", a) + } +} + +func TestMarkedContentScan_UndecodableStreamDropsPendingOperands(t *testing.T) { + p := scanPage(t, "", []string{rawStream("/P <>"), undecodableStream, rawStream("BDC EMC")}) + checkContentMCIDs(t, p, []mcidRow{{-1, "", 2, 1, -1, "0 of its 2 operands"}}) +} + func TestMarkedContentScan_DanglingMCIDInNamedPropertyListIsAnErrorRow(t *testing.T) { p := scanPage(t, "<< /Properties << /MC0 << /MCID 99 0 R >> >> >>", []string{rawStream("/Span /MC0 BDC EMC")}) checkContentMCIDs(t, p, []mcidRow{{-1, "Span", 0, 1, -1, "does not resolve"}}) @@ -226,6 +253,7 @@ func TestMarkedContentScan_BDCWithUnreadableOperandsIsAnErrorRow(t *testing.T) { }{ {"tag only", "/P BDC EMC", "1 of its 2 operands"}, {"no operands", "BDC EMC", "0 of its 2 operands"}, + {"property list only", "<> BDC EMC", "1 of its 2 operands"}, {"number as property list", "/P 5 BDC EMC", "not a dictionary or a name"}, {"array as property list", "/P [1] BDC EMC", "not a dictionary or a name"}, } diff --git a/internal/pdfcore/markedcontent_tokens_test.go b/internal/pdfcore/markedcontent_tokens_test.go index 1638b39..9e139ea 100644 --- a/internal/pdfcore/markedcontent_tokens_test.go +++ b/internal/pdfcore/markedcontent_tokens_test.go @@ -230,17 +230,68 @@ func TestScanMarkedContentTokens_InlineImagePayloadIsNotScanned(t *testing.T) { checkStream(t, streams[0], 1, 1, 0, 0) } -func TestScanMarkedContentTokens_RowPositionIsTheFirstOperand(t *testing.T) { +func TestScanMarkedContentTokens_RowPositionIsTheTag(t *testing.T) { rows, _ := scanTokens(noProps(t), "q\n /P <> BDC EMC Q") if len(rows) != 1 || rows[0].Line != 2 || rows[0].Col != 3 { t.Errorf("rows = %+v, want line 2 col 3 (the tag)", rows) } + rows, _ = scanTokens(noProps(t), "1 2\n /P <> BDC EMC") + if len(rows) != 1 || rows[0].Line != 2 || rows[0].Col != 2 { + t.Errorf("rows = %+v, want line 2 col 2 (the tag, not the stray operands)", rows) + } rows, _ = scanTokens(noProps(t), " BDC") if len(rows) != 1 || rows[0].Line != 1 || rows[0].Col != 3 || !strings.Contains(rows[0].Err, "0 of its 2 operands") { t.Errorf("rows = %+v, want the BDC position when it has no operands", rows) } } +func TestScanMarkedContentTokens_StrayLeadingOperandsKeepTheMCID(t *testing.T) { + cases := []struct { + name string + content string + mcid int + tag string + }{ + {"names before the tag", "/X /P <> BDC EMC", 0, "P"}, + {"numbers before the tag", "1 2 3 /Span <> BDC EMC", 4, "Span"}, + {"unclosed array before the tag", "[ /P <> BDC EMC", 2, "P"}, + {"unclosed dict before the tag", "<< /P <> BDC EMC", 5, "P"}, + {"name and unclosed dict before the tag", "/X << /P <> BDC EMC", 8, "P"}, + {"stray closer before the tag", "] /P <> BDC EMC", 6, "P"}, + {"property list dict never closed", "/P << /MCID 3 BDC EMC", 3, "P"}, + {"unclosed array then unclosed dict", "[ /P << /MCID 7 BDC EMC", 7, "P"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, streams := scanTokens(noProps(t), tc.content) + if len(rows) != 1 || rows[0].MCID != tc.mcid || rows[0].Tag != tc.tag || rows[0].Err != "" { + t.Errorf("rows = %+v, want MCID %d tag %s with no error", rows, tc.mcid, tc.tag) + } + checkStream(t, streams[0], 1, 1, 0, 0) + }) + } +} + +func TestScanMarkedContentTokens_TagThatIsAContainerIsAnErrorRow(t *testing.T) { + rows, _ := scanTokens(noProps(t), "/X [/P] <> BDC EMC") + if len(rows) != 1 || rows[0].MCID != -1 || !strings.Contains(rows[0].Err, "not a tag name") { + t.Errorf("rows = %+v, want one not-a-tag-name error row", rows) + } +} + +func TestScanMarkedContentTokens_OperandsCarryAcrossStreams(t *testing.T) { + rows, streams := scanTokens(noProps(t), "q /P", "<> BDC EMC Q") + if len(rows) != 1 || rows[0].MCID != 0 || rows[0].Tag != "P" || rows[0].Err != "" { + t.Fatalf("rows = %+v, want MCID 0 tag P from operands split over three streams", rows) + } + if r := rows[0]; r.Stream != 0 || r.Line != 1 || r.Col != 3 || r.Depth != 1 { + t.Errorf("row = %+v, want stream 0 line 1 col 3 (the tag) at depth 1", r) + } + checkStream(t, streams[0], 0, 0, 0, 0) + checkStream(t, streams[1], 0, 0, 0, 0) + checkStream(t, streams[2], 1, 1, 0, 0) +} + func TestScanMarkedContentTokens_OtherOperatorsResetOperands(t *testing.T) { rows, streams := scanTokens(noProps(t), "/P <> Tj BDC EMC") if len(rows) != 1 || !strings.Contains(rows[0].Err, "0 of its 2 operands") { diff --git a/internal/pdfcore/model.go b/internal/pdfcore/model.go index 56b4355..1c56174 100644 --- a/internal/pdfcore/model.go +++ b/internal/pdfcore/model.go @@ -850,7 +850,8 @@ type ParentTreeSlot struct { // MarkedStream is the marked-content balance of one content stream. Depth is // the number of open marked-content sequences, carried from one stream of a -// page to the next. +// page to the next, so only the page's last stream's DepthAtEnd shows a +// sequence left open. type MarkedStream struct { Ref string `json:"ref"` Opens int `json:"opens"` // BMC + BDC @@ -862,7 +863,8 @@ type MarkedStream struct { // ContentMCID is one BDC in a page's content whose property list carries an // /MCID, or whose operands or property list could not be read. MCID is -1 -// when Err is set. +// when Err is set. Stream, Line and Col locate the BDC's tag operand, which +// may sit in an earlier stream than the operator. type ContentMCID struct { MCID int `json:"mcid"` Tag string `json:"tag"` // the BDC tag name, e.g. "P" diff --git a/internal/pdfcore/numbertree.go b/internal/pdfcore/numbertree.go index 1b84368..9232e8c 100644 --- a/internal/pdfcore/numbertree.go +++ b/internal/pdfcore/numbertree.go @@ -2,7 +2,6 @@ package pdfcore import ( "fmt" - "sort" "strings" pdfcpu_model "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" @@ -25,15 +24,14 @@ type numberTreeWalker struct { // walkNumberTree reads the number tree (ISO 32000-1 7.9.7) rooted at root and // returns every key with its value as stored (possibly an indirect reference; -// callers dereference), the keys in ascending order, and every malformed node -// it met. A malformed node is reported and the walk goes on: a node that is +// callers dereference) and every malformed node it met. A malformed node is reported and the walk goes on: a node that is // not a dictionary, has neither /Kids nor /Nums, an odd-length /Nums, a // non-integer key, a bad /Limits or a key outside it, keys out of order // within one leaf, a // duplicate key (the first value is kept), a node reached twice, and a node // deeper than maxNumberTreeDepth. where prefixes every problem location. // Callers hold pdfMu. -func walkNumberTree(ctx *pdfcpu_model.Context, root pdfcpu_types.Object, where string) (entries map[int]pdfcpu_types.Object, order []int, problems []StructureProblem) { +func walkNumberTree(ctx *pdfcpu_model.Context, root pdfcpu_types.Object, where string) (entries map[int]pdfcpu_types.Object, problems []StructureProblem) { w := &numberTreeWalker{ ctx: ctx, where: where, @@ -42,12 +40,7 @@ func walkNumberTree(ctx *pdfcpu_model.Context, root pdfcpu_types.Object, where s visited: map[int]bool{}, } w.visit(root, true, 1) - order = make([]int, 0, len(w.entries)) - for k := range w.entries { - order = append(order, k) - } - sort.Ints(order) - return w.entries, order, w.problems + return w.entries, w.problems } // ntNodeLabel names a node for problem locations: its reference, or "(direct)". diff --git a/internal/pdfcore/numbertree_shapes_test.go b/internal/pdfcore/numbertree_shapes_test.go index 28c604c..f8e6f25 100644 --- a/internal/pdfcore/numbertree_shapes_test.go +++ b/internal/pdfcore/numbertree_shapes_test.go @@ -64,7 +64,7 @@ func TestWalkNumberTree_NodeShapes(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - entries, order, problems := walkRawNumberTree(t, tc.objs...) + entries, problems := walkRawNumberTree(t, tc.objs...) if len(problems) != len(tc.problems) { t.Errorf("problems:\n%s\nwant %d: %v", problemsText(problems), len(tc.problems), tc.problems) } @@ -80,9 +80,6 @@ func TestWalkNumberTree_NodeShapes(t *testing.T) { } } checkIntEntries(t, entries, tc.want) - if len(order) != len(tc.want) { - t.Errorf("order = %v, want %d keys", order, len(tc.want)) - } }) } } diff --git a/internal/pdfcore/numbertree_test.go b/internal/pdfcore/numbertree_test.go index 26037a6..f41dd83 100644 --- a/internal/pdfcore/numbertree_test.go +++ b/internal/pdfcore/numbertree_test.go @@ -2,7 +2,6 @@ package pdfcore import ( "fmt" - "slices" "strings" "testing" @@ -17,16 +16,16 @@ const numberTreeWhere = "NT" // walkRawNumberTree reads objs as a document and walks the number tree rooted // at object 10. -func walkRawNumberTree(t *testing.T, objs ...rawObj) (map[int]pdfcpu_types.Object, []int, []StructureProblem) { +func walkRawNumberTree(t *testing.T, objs ...rawObj) (map[int]pdfcpu_types.Object, []StructureProblem) { t.Helper() _, doc := openUnvalidated(t, numberTreePDF(objs...)) - entries, order, problems := walkNumberTree(doc.PDFContext, *pdfcpu_types.NewIndirectRef(10, 0), numberTreeWhere) + entries, problems := walkNumberTree(doc.PDFContext, *pdfcpu_types.NewIndirectRef(10, 0), numberTreeWhere) for _, p := range problems { if !strings.Contains(p.Where, numberTreeWhere) { t.Errorf("problem location %q does not carry the caller's prefix %q", p.Where, numberTreeWhere) } } - return entries, order, problems + return entries, problems } // problemText is a problem's location and message on one line. @@ -75,19 +74,11 @@ func TestWalkNumberTree_WellFormed(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - entries, order, problems := walkRawNumberTree(t, tc.objs...) + entries, problems := walkRawNumberTree(t, tc.objs...) if len(problems) != 0 { t.Errorf("unexpected problems:\n%s", problemsText(problems)) } checkIntEntries(t, entries, tc.want) - want := make([]int, 0, len(tc.want)) - for k := range tc.want { - want = append(want, k) - } - slices.Sort(want) - if !slices.Equal(order, want) { - t.Errorf("order = %v, want %v", order, want) - } }) } } @@ -136,7 +127,7 @@ func TestWalkNumberTree_ReportsMalformedNodes(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - entries, _, problems := walkRawNumberTree(t, tc.objs...) + entries, problems := walkRawNumberTree(t, tc.objs...) if len(problems) == 0 { t.Fatalf("no problem reported") } @@ -158,15 +149,12 @@ func TestWalkNumberTree_ReportsMalformedNodes(t *testing.T) { } func TestWalkNumberTree_DuplicateKeyKeepsFirstValueAndNamesBothNodes(t *testing.T) { - entries, order, problems := walkRawNumberTree(t, + entries, problems := walkRawNumberTree(t, rawObj{10, "<< /Kids [11 0 R 12 0 R] >>"}, rawObj{11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, rawObj{12, "<< /Limits [1 2] /Nums [1 201 2 202] >>"}, ) checkIntEntries(t, entries, map[int]int{0: 100, 1: 101, 2: 202}) - if !slices.Equal(order, []int{0, 1, 2}) { - t.Errorf("order = %v, want every key once: [0 1 2]", order) - } found := false for _, p := range problems { text := problemText(p) @@ -180,14 +168,11 @@ func TestWalkNumberTree_DuplicateKeyKeepsFirstValueAndNamesBothNodes(t *testing. } func TestWalkNumberTree_KidPointingBackAtRootIsReportedOnce(t *testing.T) { - entries, order, problems := walkRawNumberTree(t, + entries, problems := walkRawNumberTree(t, rawObj{10, "<< /Kids [11 0 R 10 0 R] >>"}, rawObj{11, "<< /Limits [0 1] /Nums [0 100 1 101] >>"}, ) checkIntEntries(t, entries, map[int]int{0: 100, 1: 101}) - if !slices.Equal(order, []int{0, 1}) { - t.Errorf("order = %v, want every key once: [0 1]", order) - } n := 0 for _, p := range problems { if strings.Contains(p.Message, "reached twice") { @@ -216,7 +201,7 @@ func TestWalkNumberTree_DepthPastCapIsReported(t *testing.T) { objs = append(objs, rawObj{num, "<< /Limits [0 0] /Nums [0 100] >>"}) } } - _, _, problems := walkRawNumberTree(t, objs...) + _, problems := walkRawNumberTree(t, objs...) found := false for _, p := range problems { if strings.Contains(p.Message, "64") && strings.Contains(p.Message, "deep") { @@ -229,7 +214,7 @@ func TestWalkNumberTree_DepthPastCapIsReported(t *testing.T) { } func TestWalkNumberTree_ReturnsValuesAsStored(t *testing.T) { - entries, _, problems := walkRawNumberTree(t, + entries, problems := walkRawNumberTree(t, rawObj{10, "<< /Nums [0 11 0 R] >>"}, rawObj{11, "[12 0 R null]"}, rawObj{12, "<< /S /P >>"}, diff --git a/internal/pdfcore/stream.go b/internal/pdfcore/stream.go index c97402a..c147403 100644 --- a/internal/pdfcore/stream.go +++ b/internal/pdfcore/stream.go @@ -40,6 +40,22 @@ func (ins *Inspector) pageContentStreamNodeIDs(tabID string, pageNum int) ([]str // concatenation order, with the contract pageContentStreamNodeIDs documents. // Callers that dereference through pdfcpu hold pdfMu. func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { + refs, err := contentStreamRefs(pageDict) + if err != nil || refs == nil { + return nil, err + } + ids := make([]string, len(refs)) + for i, ref := range refs { + ids[i] = nodeIDForRef(ref) + } + return ids, nil +} + +// contentStreamRefs returns the indirect references of pageDict's /Contents +// stream(s) in concatenation order: nil when there is none, one for a single +// reference, and every reference element of an array with null elements +// skipped. Any other array element or /Contents value is an error. +func contentStreamRefs(pageDict pdfcpu_types.Dict) ([]pdfcpu_types.IndirectRef, error) { contents, found := pageDict.Find("Contents") if !found || contents == nil { return nil, nil @@ -47,12 +63,12 @@ func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { switch v := contents.(type) { case pdfcpu_types.IndirectRef: - return []string{fmt.Sprintf("obj:%d:%d", v.GenerationNumber.Value(), v.ObjectNumber.Value())}, nil + return []pdfcpu_types.IndirectRef{v}, nil case pdfcpu_types.Array: if len(v) == 0 { return nil, nil } - ids := make([]string, 0, len(v)) + refs := make([]pdfcpu_types.IndirectRef, 0, len(v)) for i, e := range v { // A null element carries no content, so skipping it drops nothing // (pdfcpu decodes PDF null to a Go nil element) - at ANY index, @@ -68,9 +84,9 @@ func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { if !ok { return nil, fmt.Errorf("contents array element %d is not an indirect reference: %T", i, e) } - ids = append(ids, fmt.Sprintf("obj:%d:%d", ref.GenerationNumber.Value(), ref.ObjectNumber.Value())) + refs = append(refs, ref) } - return ids, nil + return refs, nil default: return nil, fmt.Errorf("unexpected Contents type: %T", contents) } diff --git a/internal/pdfcore/structure.go b/internal/pdfcore/structure.go index 44ea686..cacad88 100644 --- a/internal/pdfcore/structure.go +++ b/internal/pdfcore/structure.go @@ -67,6 +67,22 @@ type structureWalker struct { claims map[int]map[int][]string // parentTree holds the ParentTree entries by key. parentTree map[int]pdfcpu_types.Object + // streamUses counts, per content stream node ID, the page scans still to + // come that read it. + streamUses map[string]int + // streamMemo holds the decoded tokens of a content stream that a later + // page scan still reads, so a shared stream is decoded once per walk. + // An entry is dropped after its last use. + streamMemo map[string]decodedStream + // decodes counts content stream decodes. + decodes int +} + +// decodedStream is one content stream's tokens, or the error that stopped +// its decode. +type decodedStream struct { + tokens []Token + err string } func newStructureWalker(ctx *pdfcpu_model.Context) *structureWalker { @@ -84,6 +100,8 @@ func newStructureWalker(ctx *pdfcpu_model.Context) *structureWalker { visited: map[int]bool{}, claims: map[int]map[int][]string{}, parentTree: map[int]pdfcpu_types.Object{}, + streamUses: map[string]int{}, + streamMemo: map[string]decodedStream{}, } } @@ -127,6 +145,7 @@ func (d *DocumentState) structurePhase1(w *structureWalker) ([]*pageLeaf, error) pt, ptErr := d.pageTree() leaves = pt.leaves w.readTree(leaves) + w.countStreamUses(leaves) if ptErr != nil { w.problem("page tree", fmt.Sprintf("page-tree walk stopped after %d pages: %v; later pages not listed", len(leaves), ptErr)) } @@ -148,6 +167,19 @@ func (d *DocumentState) scanPageLocked(w *structureWalker, leaf *pageLeaf, sp *S }) } +// countStreamUses counts how many page scans read each content stream. +func (w *structureWalker) countStreamUses(leaves []*pageLeaf) { + for _, leaf := range leaves { + if leaf.err != nil { + continue + } + refs, _ := contentStreamRefs(leaf.dict) + for _, ref := range refs { + w.streamUses[nodeIDForRef(ref)]++ + } + } +} + // problem records a finding with no row of its own. func (w *structureWalker) problem(where, msg string) { w.report.Problems = append(w.report.Problems, StructureProblem{Where: where, Message: msg}) @@ -177,7 +209,8 @@ func (w *structureWalker) readTree(leaves []*pageLeaf) { w.readCatalog(catalog) } - named := map[int]bool{} + // named maps each /StructParents key to the first page that has it. + named := map[int]int{} for i, leaf := range leaves { sp := &StructPage{ PageNum: i + 1, @@ -198,7 +231,12 @@ func (w *structureWalker) readTree(leaves []*pageLeaf) { sp.ParentTreeErr = "the ParentTree entry could not be read: " + msg }) if sp.StructParents != nil { - named[*sp.StructParents] = true + key := *sp.StructParents + if first, dup := named[key]; dup { + w.problem(fmt.Sprintf("page %d", sp.PageNum), fmt.Sprintf("/StructParents %d is also the key of page %d; both pages bind to the same ParentTree entry", key, first)) + } else { + named[key] = sp.PageNum + } } } w.readOtherParentTree(named) @@ -290,7 +328,7 @@ func (w *structureWalker) readRoot(root pdfcpu_types.Dict, where string) { r.ParentTreeRef = refString(ref) ptWhere += " " + r.ParentTreeRef } - entries, _, problems := walkNumberTree(w.ctx, o, ptWhere) + entries, problems := walkNumberTree(w.ctx, o, ptWhere) w.parentTree = entries r.Problems = append(r.Problems, problems...) } @@ -528,7 +566,7 @@ func (w *structureWalker) readElement(d pdfcpu_types.Dict, e *StructElem, id str default: e.S = s.Value() } - if typ := nameOf(w.deref(d["Type"])); typ != "" && typ != "StructElem" && e.S == "" { + if typ := nameOf(w.deref(d["Type"])); typ != "" && typ != "StructElem" { errs = append(errs, fmt.Sprintf("/Type /%s is not StructElem", typ)) } if e.S != "" { @@ -683,10 +721,10 @@ func (w *structureWalker) elementRefOf(o pdfcpu_types.Object) (ref, s, msg strin } // readOtherParentTree lists the ParentTree keys no page names. -func (w *structureWalker) readOtherParentTree(named map[int]bool) { +func (w *structureWalker) readOtherParentTree(named map[int]int) { keys := make([]int, 0, len(w.parentTree)) for k := range w.parentTree { - if !named[k] { + if _, isNamed := named[k]; !isNamed { keys = append(keys, k) } } @@ -704,31 +742,30 @@ func (w *structureWalker) readOtherParentTree(named map[int]bool) { } // scanPage decodes the page's content streams in /Contents order and scans -// their marked content. Callers hold pdfMu. +// their marked content. A stream that a later page also reads is decoded once +// and its tokens kept until that page's scan. Callers hold pdfMu. func (w *structureWalker) scanPage(d *DocumentState, leaf *pageLeaf, sp *StructPage) { if leaf.err != nil { return } - ids, err := contentStreamIDs(leaf.dict) + refs, err := contentStreamRefs(leaf.dict) if err != nil { sp.Err = joinErr(sp.Err, err.Error()) return } props := w.propertyLookup(leaf) st := &mcState{} - for i, id := range ids { - ms := MarkedStream{Ref: nodeIDToRef(id)} - raw, decodeErr, err := d.decodeContentStreamLocked(id) - switch { - case err != nil: - ms.Err = err.Error() - case decodeErr != "": - ms.Err = decodeErr - } + for i, ref := range refs { + ms := MarkedStream{Ref: refString(ref)} + ds := w.decodeStream(d, nodeIDForRef(ref)) + ms.Err = ds.err if ms.Err != "" { ms.DepthAtEnd, ms.MinDepth = len(st.stack), len(st.stack) + // The stream's content is unknown, so operands read before it + // do not carry past it. + st.clearOperands() } else { - rows := scanMarkedContent(tokenizeContentStream(raw), i, props, st, &ms) + rows := scanMarkedContent(ds.tokens, i, props, st, &ms) sp.ContentMCIDs = append(sp.ContentMCIDs, rows...) } sp.Streams = append(sp.Streams, ms) @@ -740,22 +777,45 @@ func (w *structureWalker) scanPage(d *DocumentState, leaf *pageLeaf, sp *StructP } } +// decodeStream returns the tokens of the content stream id for one use by a +// page scan, from streamMemo when an earlier page decoded it. Callers hold +// pdfMu. +func (w *structureWalker) decodeStream(d *DocumentState, id string) decodedStream { + ds, memo := w.streamMemo[id] + if !memo { + w.decodes++ + raw, decodeErr, err := d.decodeContentStreamLocked(id) + switch { + case err != nil: + ds.err = err.Error() + case decodeErr != "": + ds.err = decodeErr + default: + ds.tokens = tokenizeContentStream(raw) + } + } + w.streamUses[id]-- + if w.streamUses[id] > 0 { + w.streamMemo[id] = ds + } else { + delete(w.streamMemo, id) + delete(w.streamUses, id) + } + return ds +} + // propertyLookup returns the named-property-list lookup for a page: its -// effective /Resources /Properties, each level dereferenced. +// effective /Resources /Properties, each level dereferenced once here. +// Callers hold pdfMu, and so does every call of the returned lookup. func (w *structureWalker) propertyLookup(leaf *pageLeaf) propertyLookup { + var props pdfcpu_types.Dict + if res, ok := w.deref(leaf.attrs.resources).(pdfcpu_types.Dict); ok { + props, _ = w.deref(res["Properties"]).(pdfcpu_types.Dict) + } return func(name string) (pdfcpu_types.Object, error) { - notFound := fmt.Errorf("property list /%s not found in /Resources /Properties", name) - res, ok := w.deref(leaf.attrs.resources).(pdfcpu_types.Dict) - if !ok { - return nil, notFound - } - props, ok := w.deref(res["Properties"]).(pdfcpu_types.Dict) - if !ok { - return nil, notFound - } entry, found := props.Find(name) if !found { - return nil, notFound + return nil, fmt.Errorf("property list /%s not found in /Resources /Properties", name) } pl, ok := w.deref(entry).(pdfcpu_types.Dict) if !ok { @@ -811,15 +871,6 @@ func (w *structureWalker) align(sp *StructPage) []MCIDAlignment { return out } -// nodeIDToRef renders an "obj::" node ID as "N G R". -func nodeIDToRef(id string) string { - var gen, num int - if _, err := fmt.Sscanf(id, "obj:%d:%d", &gen, &num); err != nil { - return id - } - return fmt.Sprintf("%d %d R", num, gen) -} - // appendErr appends msg to errs when it is not empty. func appendErr(errs []string, msg string) []string { if msg == "" { diff --git a/internal/pdfcore/structure_shapes_test.go b/internal/pdfcore/structure_shapes_test.go index c8ae734..830d0df 100644 --- a/internal/pdfcore/structure_shapes_test.go +++ b/internal/pdfcore/structure_shapes_test.go @@ -206,11 +206,14 @@ func TestGetStructure_ElementEntryShapes(t *testing.T) { t.Errorf("Alt %v ActualText %v, want nil when absent", e.Alt, e.ActualText) } }, ""}, - {"/Type other than StructElem with an /S", "<< /Type /Foo /S /P >>", func(t *testing.T, e *StructElem) { + {"/Type other than StructElem with an /S", "<< /Type /Foo /S /P /Pg 3 0 R /K 0 >>", func(t *testing.T, e *StructElem) { if e.S != "P" { t.Errorf("S = %q, want P", e.S) } - }, ""}, + if len(e.Kids) != 1 || e.Kids[0].Kind != "mcid" { + t.Errorf("kids = %v, want the element walked with its one MCID kid", kindsOf(e.Kids)) + } + }, "/Type /Foo is not StructElem"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -401,6 +404,70 @@ func TestGetStructure_NamedPropertyListLookup(t *testing.T) { } } +func TestGetStructure_PagesSharingAStructParentsKeyAreAProblem(t *testing.T) { + all := []rawObj{ + {1, "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>"}, + {5, "<< /Type /StructTreeRoot /K [10 0 R] /ParentTree 6 0 R >>"}, + {6, "<< /Nums [0 [10 0 R]] >>"}, + {10, "<< /Type /StructElem /S /P /Pg 3 0 R /K 0 >>"}, + } + all = append(all, structPagesObjs(3, nil, []string{"/StructParents 0", "/StructParents 1", "/StructParents 0"})...) + r := rawStructure(t, rawPDF(all...)) + var found []StructureProblem + for _, p := range r.Problems { + if strings.Contains(p.Message, "/StructParents 0") { + found = append(found, p) + } + } + if len(found) != 1 || found[0].Where != "page 3" || !strings.Contains(found[0].Message, "page 1") { + t.Errorf("problems = %+v, want one at page 3 naming key 0 and page 1", r.Problems) + } + for _, p := range r.Problems { + if strings.Contains(p.Message, "/StructParents 1") { + t.Errorf("unshared key reported: %+v", p) + } + } + if len(r.OtherParentTree) != 0 { + t.Errorf("OtherParentTree = %+v, want none: key 0 is named", r.OtherParentTree) + } +} + +func TestGetStructure_SharedContentStreamIsDecodedOnce(t *testing.T) { + _, doc := openUnvalidated(t, rawPDF( + rawObj{1, rawCatalog}, + rawObj{2, "<< /Type /Pages /Kids [3 0 R 4 0 R 7 0 R] /Count 3 >>"}, + rawObj{3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 20 0 R >>"}, + rawObj{4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [20 0 R 21 0 R] >>"}, + rawObj{7, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [21 0 R 20 0 R] >>"}, + rawObj{20, rawStream("/P <> BDC EMC")}, + rawObj{21, rawStream("/P <> BDC EMC")}, + )) + w := newStructureWalker(doc.PDFContext) + r, err := doc.buildStructure(w) + if err != nil { + t.Fatalf("buildStructure: %v", err) + } + if w.decodes != 2 { + t.Errorf("decodes = %d, want 2 (one per distinct stream)", w.decodes) + } + if len(w.streamMemo) != 0 || len(w.streamUses) != 0 { + t.Errorf("memo %d entries, uses %d entries after the walk, want both empty", len(w.streamMemo), len(w.streamUses)) + } + want := [][]string{{"20 0 R"}, {"20 0 R", "21 0 R"}, {"21 0 R", "20 0 R"}} + for i, p := range r.Pages { + var refs []string + for _, s := range p.Streams { + if s.Err != "" { + t.Errorf("page %d stream %s: %s", p.PageNum, s.Ref, s.Err) + } + refs = append(refs, s.Ref) + } + if !slices.Equal(refs, want[i]) || len(p.ContentMCIDs) != len(want[i]) { + t.Errorf("page %d streams %v with %d MCID rows, want %v with one row each", p.PageNum, refs, len(p.ContentMCIDs), want[i]) + } + } +} + func TestGetStructure_CloseBeforeTreeWalkCancels(t *testing.T) { ins, doc := openUnvalidated(t, threePagePDF()) w := newStructureWalker(doc.PDFContext) @@ -422,20 +489,6 @@ func TestGetStructure_CloseBeforeTreeWalkCancels(t *testing.T) { } } -func TestNodeIDToRef(t *testing.T) { - cases := map[string]string{ - "obj:0:12": "12 0 R", - "obj:3:7": "7 3 R", - "root": "root", - "": "", - } - for in, want := range cases { - if got := nodeIDToRef(in); got != want { - t.Errorf("nodeIDToRef(%q) = %q, want %q", in, got, want) - } - } -} - func TestJoinErr(t *testing.T) { cases := []struct{ a, b, want string }{ {"", "b", "b"}, From 81af806954796226598dce267914f1e110ae7b80 Mon Sep 17 00:00:00 2001 From: Ade Anom A Date: Fri, 23 Oct 2026 13:44:35 +0700 Subject: [PATCH 3/3] Address review comments on the structure dump --- cmd/cli/cmd_structure.go | 47 +++++---- cmd/cli/cmd_structure_plain_test.go | 47 ++++++++- docs/cli/structure.md | 15 ++- internal/pdfcore/markedcontent.go | 13 ++- internal/pdfcore/markedcontent_test.go | 1 + internal/pdfcore/markedcontent_tokens_test.go | 32 ++++++- internal/pdfcore/model.go | 11 ++- internal/pdfcore/numbertree.go | 3 +- internal/pdfcore/numbertree_test.go | 8 ++ internal/pdfcore/stream.go | 23 +++-- internal/pdfcore/structure.go | 33 +++++-- internal/pdfcore/structure_shapes_test.go | 95 ++++++++++++++++++- 12 files changed, 275 insertions(+), 53 deletions(-) diff --git a/cmd/cli/cmd_structure.go b/cmd/cli/cmd_structure.go index 58a9eeb..090f5ed 100644 --- a/cmd/cli/cmd_structure.go +++ b/cmd/cli/cmd_structure.go @@ -68,7 +68,7 @@ func structurePlain(r *pdfcore.StructureReport) string { } else { writeStructureHeader(&b, r) } - if r.HasStructTree || len(r.Elements) > 0 { + if r.HasStructTree { c := r.Counts fmt.Fprintf(&b, "\nElements (%d; kids: %d MCID, %d MCR, %d OBJR, %d error):\n", c.Elements, c.MCIDKids, c.MCRKids, c.OBJRKids, c.ErrorKids) @@ -79,17 +79,20 @@ func structurePlain(r *pdfcore.StructureReport) string { if len(r.OtherParentTree) > 0 { b.WriteString("\nOther ParentTree keys:\n") for _, e := range r.OtherParentTree { - line := fmt.Sprintf(" %d -> %s", e.Key, joinNonEmpty(e.ElementRef, e.S)) + line := fmt.Sprintf(" %d -> %s", e.Key, joinNonEmpty(asciiSafe(e.ElementRef), asciiSafe(e.S))) + if e.Note != "" { + line += " note: " + asciiSafe(e.Note) + } if e.Err != "" { - line += " error: " + e.Err + line += " error: " + asciiSafe(e.Err) } - b.WriteString(asciiSafe(line) + "\n") + b.WriteString(line + "\n") } } if len(r.Problems) > 0 { fmt.Fprintf(&b, "\nProblems (%d):\n", len(r.Problems)) for _, p := range r.Problems { - b.WriteString(" " + asciiSafe(p.Where+": "+p.Message) + "\n") + b.WriteString(" " + asciiSafe(p.Where) + ": " + asciiSafe(p.Message) + "\n") } } return b.String() @@ -116,9 +119,9 @@ func writeStructureHeader(b *strings.Builder, r *pdfcore.StructureReport) { if len(r.RoleMap) > 0 { pairs := make([]string, len(r.RoleMap)) for i, e := range r.RoleMap { - pairs[i] = e.From + " -> " + orText(e.To, "?") + pairs[i] = asciiSafe(e.From) + " -> " + asciiSafe(orText(e.To, "?")) } - b.WriteString("RoleMap: " + asciiSafe(strings.Join(pairs, ", ")) + "\n") + b.WriteString("RoleMap: " + strings.Join(pairs, ", ") + "\n") } } @@ -141,17 +144,17 @@ func writeStructKids(b *strings.Builder, kids []*pdfcore.StructKid, level int) { line += fmt.Sprintf(" page %d", k.Page) } if k.StmRef != "" { - line += " stream " + k.StmRef + line += " stream " + asciiSafe(k.StmRef) } case "objr": - line = "OBJR " + joinNonEmpty(orText(k.ObjRef, "-"), k.ObjType) + line = "OBJR " + joinNonEmpty(asciiSafe(orText(k.ObjRef, "-")), asciiSafe(k.ObjType)) default: - line = "error: " + k.Err + line = "error: " + asciiSafe(k.Err) } if k.Kind != "error" && k.Kind != "element" && k.Err != "" { - line += " error: " + k.Err + line += " error: " + asciiSafe(k.Err) } - b.WriteString(indent + asciiSafe(line) + "\n") + b.WriteString(indent + line + "\n") if k.Element != nil { writeStructKids(b, k.Element.Kids, level+1) } @@ -159,12 +162,14 @@ func writeStructKids(b *strings.Builder, kids []*pdfcore.StructKid, level int) { } // elementLine renders one element: S, mapped S, ref, page and text fields. +// Each value is made ASCII-safe on its own, so the line is never escaped +// again as a whole. func elementLine(e *pdfcore.StructElem) string { - parts := []string{orText(e.S, "(no /S)")} + parts := []string{asciiSafe(orText(e.S, "(no /S)"))} if e.MappedS != "" { - parts[0] += " (" + e.MappedS + ")" + parts[0] += " (" + asciiSafe(e.MappedS) + ")" } - parts[0] += " " + orText(e.Ref, "(direct)") + parts[0] += " " + asciiSafe(orText(e.Ref, "(direct)")) if e.Page > 0 { p := fmt.Sprintf("page %d", e.Page) if e.PageInherited { @@ -172,7 +177,7 @@ func elementLine(e *pdfcore.StructElem) string { } parts = append(parts, p) } else if e.PgRef != "" { - parts = append(parts, "/Pg "+e.PgRef) + parts = append(parts, "/Pg "+asciiSafe(e.PgRef)) } text := func(label, v string) { c := pdfcore.ClampDisplayValue(v, pdfcore.TreeValueCap) @@ -197,7 +202,7 @@ func elementLine(e *pdfcore.StructElem) string { } } if e.Err != "" { - parts = append(parts, "error: "+e.Err) + parts = append(parts, "error: "+asciiSafe(e.Err)) } return strings.Join(parts, " ") } @@ -282,14 +287,16 @@ func writeStructPages(b *strings.Builder, pages []*pdfcore.StructPage) { if !a.Consistent { mark, starred = "*", true } + // Cells are escaped before rendering so column widths count the + // escaped text. t.AddRow(strconv.Itoa(a.MCID), strconv.Itoa(a.ContentCount), - orText(joinNonEmpty(a.ParentTreeElement, a.ParentTreeS), "-"), - orText(strings.Join(a.TreeElements, ", "), "-"), mark) + orText(joinNonEmpty(asciiSafe(a.ParentTreeElement), asciiSafe(a.ParentTreeS)), "-"), + asciiSafe(orText(strings.Join(a.TreeElements, ", "), "-")), mark) } var tb strings.Builder _ = t.Render(&tb) for line := range strings.SplitSeq(strings.TrimSuffix(tb.String(), "\n"), "\n") { - b.WriteString(" " + asciiSafe(strings.TrimRight(line, " ")) + "\n") + b.WriteString(" " + strings.TrimRight(line, " ") + "\n") } if starred && !legend { b.WriteString(" * = content count, ParentTree slot and tree claimants do not line up on one element\n") diff --git a/cmd/cli/cmd_structure_plain_test.go b/cmd/cli/cmd_structure_plain_test.go index 501ead2..4097f53 100644 --- a/cmd/cli/cmd_structure_plain_test.go +++ b/cmd/cli/cmd_structure_plain_test.go @@ -238,13 +238,14 @@ func TestStructurePlain_OtherParentTreeAndProblems(t *testing.T) { HasStructTree: true, OtherParentTree: []pdfcore.ParentTreeEntry{ {Key: 2, ElementRef: "17 0 R", S: "Link"}, - {Key: 7, Err: "value is an array, but no page's /StructParents names this key"}, + {Key: 7, Note: "array value no page's /StructParents names; may belong to a Form XObject's /StructParents"}, + {Key: 9, Err: "value is an integer, not a reference to an element"}, }, Problems: []pdfcore.StructureProblem{{Where: "ParentTree 31 0 R, node 33 0 R", Message: "key 1 outside /Limits [0 0]"}}, } out := structurePlain(r) for _, want := range []string{ - "\nOther ParentTree keys:\n 2 -> 17 0 R Link\n 7 -> error: value is an array, but no page's /StructParents names this key\n", + "\nOther ParentTree keys:\n 2 -> 17 0 R Link\n 7 -> note: array value no page's /StructParents names; may belong to a Form XObject's /StructParents\n 9 -> error: value is an integer, not a reference to an element\n", "\nProblems (1):\n ParentTree 31 0 R, node 33 0 R: key 1 outside /Limits [0 0]\n", } { if !strings.Contains(out, want) { @@ -383,3 +384,45 @@ func TestStructureDumpOnFixtures(t *testing.T) { }) } } + +func TestStructurePlain_NonASCIIElementFieldsEscapedOnce(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + Elements: []*pdfcore.StructKid{{Kind: "element", Element: &pdfcore.StructElem{ + S: "\u00dcberschrift", MappedS: "H1", Ref: "10 0 R", Alt: strPtr("Logo"), + }}}, + } + out := structurePlain(r) + want := `"\u00dcberschrift" (H1) 10 0 R Alt "Logo"` + if !strings.Contains(out, "\n"+want+"\n") { + t.Errorf("output missing line %q:\n%s", want, out) + } +} + +func TestStructurePlain_NonASCIIAlignmentCellKeepsColumnsAligned(t *testing.T) { + r := &pdfcore.StructureReport{ + HasStructTree: true, + Pages: []*pdfcore.StructPage{{PageNum: 1, Alignment: []pdfcore.MCIDAlignment{ + {MCID: 0, ContentCount: 1, ParentTreeElement: "10 0 R", ParentTreeS: "\u00dcberschrift", TreeElements: []string{"10 0 R"}, Consistent: true}, + }}}, + } + out := structurePlain(r) + header := lineOf(t, out, "PARENTTREE") + row := lineOf(t, out, "10 0 R") + if !strings.Contains(row, `10 0 R "\u00dcberschrift"`) { + t.Errorf("row %q does not carry the escaped /S cell", row) + } + if strings.HasPrefix(strings.TrimSpace(row), `"`) { + t.Errorf("row %q is escaped as a whole", row) + } + if h, c := strings.LastIndex(header, "TREE"), strings.LastIndex(row, "10 0 R"); h != c { + t.Errorf("TREE column at %d in header %q but at %d in row %q", h, header, c, row) + } +} + +func TestStructurePlain_ElementSectionOnlyWithStructTree(t *testing.T) { + r := &pdfcore.StructureReport{Elements: []*pdfcore.StructKid{{Kind: "error", Err: "x"}}} + if out := structurePlain(r); strings.Contains(out, "Elements (") { + t.Errorf("element section printed with no structure tree:\n%s", out) + } +} diff --git a/docs/cli/structure.md b/docs/cli/structure.md index e715268..454ea19 100644 --- a/docs/cli/structure.md +++ b/docs/cli/structure.md @@ -82,14 +82,16 @@ tree deeper than 256 levels says where it stops. Each page with a `/StructParents` entry, any marked content, an MCID the tree attributes to it, or an error gets a block. The first line has the `/StructParents` key, the number of content streams and the BDC/BMC and EMC -counts. A page's content streams are read as one concatenated stream, so a +counts. A page's content streams (a `/Contents` array, direct or behind a reference) +are read as one concatenated stream, so a sequence opened in one stream may close in a later one and a BDC's operands may start in one stream and end in the next. A stream that fails to decode or closes a sequence it never opened gets its own line, and so does the page's last stream when sequences are still open at its end. Each BDC and each ParentTree slot that could not be read gets a line too. A BDC's tag and -property list are its last two operands; anything before them is ignored. The -table has one row per MCID: +property list are its last two operands; anything before them is ignored. An +inline property list with no closing `>>` gets a line as well, but its MCID is +still read and counted in the table. The table has one row per MCID: - `CONTENT` - how many BDCs in the page's content carry that MCID. - `PARENTTREE` - the element in that slot of the page's ParentTree array. @@ -99,8 +101,11 @@ A row is marked `*` unless the content marks the MCID once, the ParentTree slot names an element, and that element is the only one in `TREE`. The pages that get no block are collapsed into one line. ParentTree keys no page names (an annotation's `/StructParent`, or an orphan) are listed under -"Other ParentTree keys". "Problems" lists malformed ParentTree nodes and pages -that share a `/StructParents` key with an earlier page. +"Other ParentTree keys". A key whose value is an array is listed with a +`note:` rather than an error, since a Form XObject with marked content has its +own `/StructParents` and an array value there. "Problems" lists malformed +ParentTree nodes and pages that share a `/StructParents` key with an earlier +page. MCIDs inside Form XObjects are not aligned: an MCR with `/Stm` is listed in the tree but left out of the page table, and `Do` is not followed. diff --git a/internal/pdfcore/markedcontent.go b/internal/pdfcore/markedcontent.go index fe7653f..1e0d07f 100644 --- a/internal/pdfcore/markedcontent.go +++ b/internal/pdfcore/markedcontent.go @@ -66,7 +66,7 @@ func scanMarkedContent(tokens []Token, streamIdx int, props propertyLookup, st * open := mcOpen{} if ok { row.ParentMCID = parent - if row.Err == "" { + if row.MCID >= 0 { open = mcOpen{mcid: row.MCID, hasMCID: true} } } @@ -123,7 +123,8 @@ func innermostMCID(stack []mcOpen) *int { // /MCID, so the BDC gets no row. Fewer than two operand objects, a tag that is // not a name, a property list that is neither a dictionary nor a name or // cannot be read, or an /MCID that is not a non-negative integer give a row -// with MCID -1 and Err set. +// with MCID -1 and Err set. An inline dictionary with no closing ">>" keeps +// its MCID and sets Err. func bdcRow(operands []Token, opStreams []int, op Token, opStream int, props propertyLookup) (row ContentMCID, ok bool) { row = ContentMCID{MCID: -1, Stream: opStream, Line: op.Line, Col: op.Col} at := func(i int) { @@ -135,9 +136,10 @@ func bdcRow(operands []Token, opStreams []int, op Token, opStream int, props pro return row, true } propStart := operandStart(operands, n-1) - // With no closed dictionary last, a "<<" still open is the property list. + // With no closed dictionary last, a "<<" still open is the property list, + // even as the first operand. if operands[propStart].Value != "<<" { - if u := unclosedDictStart(operands); u > 0 { + if u := unclosedDictStart(operands); u >= 0 { propStart = u } } @@ -166,6 +168,9 @@ func bdcRow(operands []Token, opStreams []int, op Token, opStream int, props pro if val.Type == "number" { if n, err := strconv.Atoi(val.Value); err == nil { setMCID(&row, n) + if row.Err == "" && unclosedDictStart(prop) == 0 { + row.Err = "property-list dictionary has no closing >>" + } return row, true } } diff --git a/internal/pdfcore/markedcontent_test.go b/internal/pdfcore/markedcontent_test.go index a141836..31370e7 100644 --- a/internal/pdfcore/markedcontent_test.go +++ b/internal/pdfcore/markedcontent_test.go @@ -254,6 +254,7 @@ func TestMarkedContentScan_BDCWithUnreadableOperandsIsAnErrorRow(t *testing.T) { {"tag only", "/P BDC EMC", "1 of its 2 operands"}, {"no operands", "BDC EMC", "0 of its 2 operands"}, {"property list only", "<> BDC EMC", "1 of its 2 operands"}, + {"unclosed property list only", "<< /MCID 0 BDC EMC", "1 of its 2 operands"}, {"number as property list", "/P 5 BDC EMC", "not a dictionary or a name"}, {"array as property list", "/P [1] BDC EMC", "not a dictionary or a name"}, } diff --git a/internal/pdfcore/markedcontent_tokens_test.go b/internal/pdfcore/markedcontent_tokens_test.go index 9e139ea..f323f88 100644 --- a/internal/pdfcore/markedcontent_tokens_test.go +++ b/internal/pdfcore/markedcontent_tokens_test.go @@ -258,8 +258,6 @@ func TestScanMarkedContentTokens_StrayLeadingOperandsKeepTheMCID(t *testing.T) { {"unclosed dict before the tag", "<< /P <> BDC EMC", 5, "P"}, {"name and unclosed dict before the tag", "/X << /P <> BDC EMC", 8, "P"}, {"stray closer before the tag", "] /P <> BDC EMC", 6, "P"}, - {"property list dict never closed", "/P << /MCID 3 BDC EMC", 3, "P"}, - {"unclosed array then unclosed dict", "[ /P << /MCID 7 BDC EMC", 7, "P"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -272,6 +270,36 @@ func TestScanMarkedContentTokens_StrayLeadingOperandsKeepTheMCID(t *testing.T) { } } +func TestScanMarkedContentTokens_UnclosedPropertyListKeepsMCIDWithError(t *testing.T) { + cases := []struct { + name string + content string + mcid int + }{ + {"property list dict never closed", "/P << /MCID 3 BDC EMC", 3}, + {"unclosed array then unclosed dict", "[ /P << /MCID 7 BDC EMC", 7}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rows, streams := scanTokens(noProps(t), tc.content) + if len(rows) != 1 || rows[0].MCID != tc.mcid || rows[0].Tag != "P" || !strings.Contains(rows[0].Err, "no closing >>") { + t.Errorf("rows = %+v, want MCID %d tag P with a no-closing->> error", rows, tc.mcid) + } + checkStream(t, streams[0], 1, 1, 0, 0) + }) + } +} + +func TestScanMarkedContentTokens_UnclosedPropertyListMCIDIsParentOfNestedBDC(t *testing.T) { + rows, _ := scanTokens(noProps(t), "/P << /MCID 3 BDC /Span <> BDC EMC EMC") + if len(rows) != 2 || rows[0].Err == "" || rows[1].Err != "" { + t.Fatalf("rows = %+v, want an error row for MCID 3 and a clean row for MCID 4", rows) + } + if rowParent(rows[1]) != 3 { + t.Errorf("MCID 4 parent = %d, want 3", rowParent(rows[1])) + } +} + func TestScanMarkedContentTokens_TagThatIsAContainerIsAnErrorRow(t *testing.T) { rows, _ := scanTokens(noProps(t), "/X [/P] <> BDC EMC") if len(rows) != 1 || rows[0].MCID != -1 || !strings.Contains(rows[0].Err, "not a tag name") { diff --git a/internal/pdfcore/model.go b/internal/pdfcore/model.go index 1c56174..3dcd409 100644 --- a/internal/pdfcore/model.go +++ b/internal/pdfcore/model.go @@ -863,8 +863,10 @@ type MarkedStream struct { // ContentMCID is one BDC in a page's content whose property list carries an // /MCID, or whose operands or property list could not be read. MCID is -1 -// when Err is set. Stream, Line and Col locate the BDC's tag operand, which -// may sit in an earlier stream than the operator. +// when it could not be read; a row with MCID 0 or more counts in the +// alignment even when Err is set (an inline property list with no closing +// ">>"). Stream, Line and Col locate the BDC's tag operand, which may sit in +// an earlier stream than the operator. type ContentMCID struct { MCID int `json:"mcid"` Tag string `json:"tag"` // the BDC tag name, e.g. "P" @@ -890,11 +892,14 @@ type MCIDAlignment struct { } // ParentTreeEntry is a ParentTree key that no page's /StructParents names, -// typically an annotation's or XObject's /StructParent. +// typically an annotation's or XObject's /StructParent. An array value is +// not read and carries Note instead of ElementRef: it may be a Form +// XObject's /StructParents entry. type ParentTreeEntry struct { Key int `json:"key"` ElementRef string `json:"elementRef"` S string `json:"s"` + Note string `json:"note,omitempty"` Err string `json:"error,omitempty"` } diff --git a/internal/pdfcore/numbertree.go b/internal/pdfcore/numbertree.go index 9232e8c..8ecc02f 100644 --- a/internal/pdfcore/numbertree.go +++ b/internal/pdfcore/numbertree.go @@ -168,7 +168,8 @@ func (w *numberTreeWalker) readNums(nums pdfcpu_types.Array, label string, lo, h if hasLimits && (key < lo || key > hi) { w.problem(label, fmt.Sprintf("key %d outside /Limits [%d %d]", key, lo, hi)) } - if havePrev && key <= prev { + // An equal key is reported once, as a duplicate below. + if havePrev && key < prev { w.problem(label, fmt.Sprintf("key %d after key %d: keys not in ascending order", key, prev)) } prev, havePrev = key, true diff --git a/internal/pdfcore/numbertree_test.go b/internal/pdfcore/numbertree_test.go index f41dd83..b922117 100644 --- a/internal/pdfcore/numbertree_test.go +++ b/internal/pdfcore/numbertree_test.go @@ -227,3 +227,11 @@ func TestWalkNumberTree_ReturnsValuesAsStored(t *testing.T) { t.Errorf("key 0 = %#v, want the reference 11 0 R as stored", entries[0]) } } + +func TestWalkNumberTree_RepeatedKeyInOneLeafIsReportedOnceAsDuplicate(t *testing.T) { + entries, problems := walkRawNumberTree(t, rawObj{10, "<< /Nums [1 101 1 102 3 103] >>"}) + if len(problems) != 1 || !strings.Contains(problems[0].Message, "key 1 already in node") { + t.Errorf("problems = %q, want one duplicate-key problem", problemsText(problems)) + } + checkIntEntries(t, entries, map[int]int{1: 101, 3: 103}) +} diff --git a/internal/pdfcore/stream.go b/internal/pdfcore/stream.go index c147403..c6d1cdf 100644 --- a/internal/pdfcore/stream.go +++ b/internal/pdfcore/stream.go @@ -4,13 +4,15 @@ import ( "fmt" "strings" + pdfcpu_model "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" pdfcpu_types "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types" ) // pageContentStreamNodeIDs resolves a 1-based page number to the ordered node // IDs of its content stream(s) in a SINGLE page-dict resolution: empty slice // (no error) when the page has no /Contents, one ID for a single indirect ref, -// and, for an array, every indirect-ref element in order. A null element is +// and, for an array (direct, or behind an indirect ref), every indirect-ref +// element in order. A null element is // skipped - it is not a stream and carries no content (ISO 32000-1 7.8.2) - but // any other non-ref element is an ERROR naming its index and type, never a // silent skip, since dropping it could omit part of the page. An array whose @@ -33,14 +35,14 @@ func (ins *Inspector) pageContentStreamNodeIDs(tabID string, pageNum int) ([]str if leaf == nil { return nil, fmt.Errorf("page %d not found", pageNum) } - return contentStreamIDs(leaf.dict) + return contentStreamIDs(doc.PDFContext, leaf.dict) } // contentStreamIDs returns the node IDs of pageDict's /Contents stream(s) in // concatenation order, with the contract pageContentStreamNodeIDs documents. -// Callers that dereference through pdfcpu hold pdfMu. -func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { - refs, err := contentStreamRefs(pageDict) +// Callers hold pdfMu. +func contentStreamIDs(ctx *pdfcpu_model.Context, pageDict pdfcpu_types.Dict) ([]string, error) { + refs, err := contentStreamRefs(ctx, pageDict) if err != nil || refs == nil { return nil, err } @@ -54,12 +56,19 @@ func contentStreamIDs(pageDict pdfcpu_types.Dict) ([]string, error) { // contentStreamRefs returns the indirect references of pageDict's /Contents // stream(s) in concatenation order: nil when there is none, one for a single // reference, and every reference element of an array with null elements -// skipped. Any other array element or /Contents value is an error. -func contentStreamRefs(pageDict pdfcpu_types.Dict) ([]pdfcpu_types.IndirectRef, error) { +// skipped. A reference to an array is followed and read as that array (ISO +// 32000-1 7.8.2); a reference to anything else is returned as is. Any other +// array element or /Contents value is an error. Callers hold pdfMu. +func contentStreamRefs(ctx *pdfcpu_model.Context, pageDict pdfcpu_types.Dict) ([]pdfcpu_types.IndirectRef, error) { contents, found := pageDict.Find("Contents") if !found || contents == nil { return nil, nil } + if ref, ok := contents.(pdfcpu_types.IndirectRef); ok { + if arr, isArr := derefIn(ctx, ref).(pdfcpu_types.Array); isArr { + contents = arr + } + } switch v := contents.(type) { case pdfcpu_types.IndirectRef: diff --git a/internal/pdfcore/structure.go b/internal/pdfcore/structure.go index cacad88..633d351 100644 --- a/internal/pdfcore/structure.go +++ b/internal/pdfcore/structure.go @@ -23,6 +23,11 @@ const noPageMsg = "no page: the nearest /Pg (on the kid, its element or an ances // root's /K, where no element owns it. const rootMCIDMsg = "MCID directly under the structure tree root, with no element to own it; not aligned" +// formXObjectNote is the note on a ParentTree array value no page's +// /StructParents names: a Form XObject with marked content has its own +// /StructParents and an array value (ISO 32000-1 14.7.4.4). +const formXObjectNote = "array value no page's /StructParents names; may belong to a Form XObject's /StructParents" + // directElement stands for a direct-dictionary element where a ref is shown. const directElement = "(direct)" @@ -37,7 +42,8 @@ var errStructureCanceled = fmt.Errorf("structure walk canceled: %w", context.Can // malformed in the document is a finding in the report; the call fails only // for an unknown tab, a close during the walk (errors.Is context.Canceled) or // a pdfcpu failure outside every per-node guard. Not cached. Every content -// stream of every page is decoded, without writing the stream cache. +// stream of every page is read: from the stream cache when GetContentStream +// already decoded it, else decoded here without writing the cache. func (ins *Inspector) GetStructure(tabID string) (*StructureReport, error) { doc, err := ins.GetDocument(tabID) if err != nil { @@ -173,7 +179,7 @@ func (w *structureWalker) countStreamUses(leaves []*pageLeaf) { if leaf.err != nil { continue } - refs, _ := contentStreamRefs(leaf.dict) + refs, _ := contentStreamRefs(w.ctx, leaf.dict) for _, ref := range refs { w.streamUses[nodeIDForRef(ref)]++ } @@ -733,7 +739,7 @@ func (w *structureWalker) readOtherParentTree(named map[int]int) { e := ParentTreeEntry{Key: k} v := w.parentTree[k] if _, isArr := w.deref(v).(pdfcpu_types.Array); isArr { - e.Err = "value is an array, but no page's /StructParents names this key" + e.Note = formXObjectNote } else { e.ElementRef, e.S, e.Err = w.elementRefOf(v) } @@ -748,7 +754,7 @@ func (w *structureWalker) scanPage(d *DocumentState, leaf *pageLeaf, sp *StructP if leaf.err != nil { return } - refs, err := contentStreamRefs(leaf.dict) + refs, err := contentStreamRefs(w.ctx, leaf.dict) if err != nil { sp.Err = joinErr(sp.Err, err.Error()) return @@ -771,17 +777,28 @@ func (w *structureWalker) scanPage(d *DocumentState, leaf *pageLeaf, sp *StructP sp.Streams = append(sp.Streams, ms) } for _, c := range sp.ContentMCIDs { - if c.Err == "" { + if c.MCID >= 0 { w.report.Counts.ContentMCIDs++ } } } // decodeStream returns the tokens of the content stream id for one use by a -// page scan, from streamMemo when an earlier page decoded it. Callers hold -// pdfMu. +// page scan: from streamMemo when an earlier page decoded it, else from an +// entry GetContentStream already put in d's stream cache, else by decoding +// it. The stream cache is read, never written. Callers hold pdfMu, so +// streamMu is taken inside it, the order GetContentStream uses. func (w *structureWalker) decodeStream(d *DocumentState, id string) decodedStream { ds, memo := w.streamMemo[id] + if !memo { + d.streamMu.Lock() + cached, hit := d.streamCache[id] + d.streamMu.Unlock() + memo = hit + if hit { + ds = decodedStream{tokens: cached.Tokenized, err: cached.Error} + } + } if !memo { w.decodes++ raw, decodeErr, err := d.decodeContentStreamLocked(id) @@ -845,7 +862,7 @@ func (w *structureWalker) align(sp *StructPage) []MCIDAlignment { return a } for _, c := range sp.ContentMCIDs { - if c.Err == "" { + if c.MCID >= 0 { row(c.MCID).ContentCount++ } } diff --git a/internal/pdfcore/structure_shapes_test.go b/internal/pdfcore/structure_shapes_test.go index 830d0df..5ca7a05 100644 --- a/internal/pdfcore/structure_shapes_test.go +++ b/internal/pdfcore/structure_shapes_test.go @@ -303,7 +303,7 @@ func TestGetStructure_OtherParentTreeValueShapes(t *testing.T) { rawObj{6, "<< /Nums [0 [] 7 [10 0 R] 8 11 0 R 9 5] >>"}, )) want := []ParentTreeEntry{ - {Key: 7, Err: "value is an array, but no page's /StructParents names this key"}, + {Key: 7, Note: formXObjectNote}, {Key: 8, ElementRef: "11 0 R", S: "Span"}, {Key: 9, Err: "value is an integer, not a reference to an element"}, } @@ -501,3 +501,96 @@ func TestJoinErr(t *testing.T) { } } } + +func TestGetStructure_ContentsReferenceToArray(t *testing.T) { + ins, _ := openUnvalidated(t, rawPDF( + rawObj{1, rawCatalog}, + rawObj{2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"}, + rawObj{3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 8 0 R >>"}, + rawObj{8, "[20 0 R null 21 0 R]"}, + rawObj{20, rawStream("/P <> BDC EMC")}, + rawObj{21, rawStream("/P <> BDC EMC")}, + )) + r, err := ins.GetStructure("raw") + if err != nil { + t.Fatalf("GetStructure: %v", err) + } + p := mustPage(t, r, 1) + var refs []string + for _, s := range p.Streams { + if s.Err != "" { + t.Errorf("stream %s: %s", s.Ref, s.Err) + } + refs = append(refs, s.Ref) + } + if p.Err != "" || !slices.Equal(refs, []string{"20 0 R", "21 0 R"}) { + t.Errorf("page Err %q, streams %v; want no error and streams 20 0 R, 21 0 R", p.Err, refs) + } + if len(p.ContentMCIDs) != 2 || p.ContentMCIDs[0].MCID != 0 || p.ContentMCIDs[1].MCID != 1 { + t.Errorf("ContentMCIDs = %+v, want MCIDs 0 and 1", p.ContentMCIDs) + } + for _, a := range p.Alignment { + if a.ContentCount != 1 { + t.Errorf("alignment row %+v, want content count 1", a) + } + } + + cs, err := ins.GetPageContentStream("raw", 1) + if err != nil || cs == nil || cs.Error != "" { + t.Fatalf("GetPageContentStream = %+v, %v; want both streams", cs, err) + } + if cs.NodeID != "obj:0:20" || !strings.Contains(cs.Raw, "MCID 0") || !strings.Contains(cs.Raw, "MCID 1") { + t.Errorf("GetPageContentStream node %s raw %q, want node obj:0:20 with both streams", cs.NodeID, cs.Raw) + } +} + +func TestGetStructure_UnclosedPropertyListMCIDCountsInAlignment(t *testing.T) { + all := []rawObj{ + {1, "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>"}, + {5, "<< /Type /StructTreeRoot /K [10 0 R] /ParentTree 6 0 R >>"}, + {6, "<< /Nums [0 [10 0 R]] >>"}, + {10, "<< /Type /StructElem /S /P /Pg 3 0 R /K 0 >>"}, + } + all = append(all, structPagesObjs(1, []string{"/P << /MCID 0 BDC (x) Tj EMC"}, []string{"/StructParents 0"})...) + r := rawStructure(t, rawPDF(all...)) + p := mustPage(t, r, 1) + if len(p.ContentMCIDs) != 1 || p.ContentMCIDs[0].MCID != 0 || !strings.Contains(p.ContentMCIDs[0].Err, "no closing >>") { + t.Fatalf("ContentMCIDs = %+v, want MCID 0 with a no-closing->> error", p.ContentMCIDs) + } + if a := alignmentRow(t, p, 0); a.ContentCount != 1 || !a.Consistent { + t.Errorf("MCID 0 row = %+v, want content count 1 and consistent", a) + } + if r.Counts.ContentMCIDs != 1 { + t.Errorf("Counts.ContentMCIDs = %d, want 1", r.Counts.ContentMCIDs) + } +} + +func TestGetStructure_ReadsStreamCacheEntryWithoutDecoding(t *testing.T) { + ins, doc := openUnvalidated(t, rawPDF( + rawObj{1, rawCatalog}, + rawObj{2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"}, + rawObj{3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [20 0 R 21 0 R] >>"}, + rawObj{20, rawStream("/P <> BDC EMC")}, + rawObj{21, rawStream("/P <> BDC EMC")}, + )) + if _, err := ins.GetContentStream("raw", "obj:0:20"); err != nil { + t.Fatalf("GetContentStream: %v", err) + } + w := newStructureWalker(doc.PDFContext) + r, err := doc.buildStructure(w) + if err != nil { + t.Fatalf("buildStructure: %v", err) + } + if w.decodes != 1 { + t.Errorf("decodes = %d, want 1 (stream 20 0 R read from the cache)", w.decodes) + } + if p := mustPage(t, r, 1); len(p.ContentMCIDs) != 2 || p.ContentMCIDs[0].MCID != 0 || p.ContentMCIDs[1].MCID != 1 { + t.Errorf("ContentMCIDs = %+v, want MCIDs 0 and 1", p.ContentMCIDs) + } + doc.streamMu.Lock() + n := len(doc.streamCache) + doc.streamMu.Unlock() + if n != 1 { + t.Errorf("streamCache holds %d entries, want 1 (GetStructure does not write it)", n) + } +}