From 8c4b41bd007929e35a0ab1c3dd57ce9b938363ac Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 26 Aug 2026 10:10:15 -0500 Subject: [PATCH 01/44] PT-4201: Standard-view copy/paste USFM fidelity (squashed for rebase) Co-Authored-By: Claude Fable 5 --- .../specs/2026-08-06-clipboard-semantics.md | 288 +++++++ .../src/nodes/features/UnknownNode.test.ts | 37 + libs/shared/src/nodes/features/UnknownNode.ts | 69 +- .../editor/CommandMenuPlugin.gate.test.tsx | 205 ++++- packages/platform/src/editor/Editor.tsx | 1 + .../editor/markerEdit/MarkerEditPlugin.tsx | 73 +- .../attributeContextPasteFidelity.test.tsx | 788 ++++++++++++++++++ .../markerEdit/clipboardCopyFidelity.test.tsx | 420 ++++++++++ .../clipboardCorpusRoundTrip.test.tsx | 277 ++++++ .../markerEdit/markerEdit.test-helpers.tsx | 69 ++ .../markerEditDeletion.utils.test.tsx | 145 +++- .../markerEdit/markerEditTier1.utils.test.tsx | 10 + .../markerEdit/markerEditTier1.utils.ts | 28 +- .../markerEditTier2Trigger.utils.test.tsx | 6 + .../markerEdit/markerPasteFidelity.test.tsx | 526 ++++++++++++ .../milestoneAttributeSettle.test.tsx | 2 + .../editor/markerEdit/noteEnterFp.test.tsx | 97 ++- .../optbreakClipboardFidelity.test.tsx | 483 +++++++++++ .../markerEdit/tier2Rebuild.utils.test.tsx | 96 +++ .../editor/markerEdit/tier2Rebuild.utils.ts | 91 +- .../whitespaceDisplay.plugin.utils.test.tsx | 647 ++++++++++++-- .../whitespaceDisplay.plugin.utils.ts | 419 +++++++++- 22 files changed, 4632 insertions(+), 145 deletions(-) create mode 100644 docs/superpowers/specs/2026-08-06-clipboard-semantics.md create mode 100644 packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx create mode 100644 packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx create mode 100644 packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx create mode 100644 packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx create mode 100644 packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md new file mode 100644 index 000000000..d8fa8ac4a --- /dev/null +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -0,0 +1,288 @@ +# Standard-View Clipboard USFM Semantics (PT-4201) + +## Overview + +This document specifies the copy/paste behavior for USFM content in Platform.Bible's Standard-view scripture editor. These semantics ensure that clipboard operations preserve USFM fidelity, allowing users to copy from the Standard view, paste into external editors, and vice versa while maintaining marker integrity and positional accuracy. + +--- + +## Semantics Decisions (S1–S7) + +### S1. Copy (Standard view) — `text/plain` is valid USFM of the selection + +Already ~true because markers are real text nodes; this plan closes the exceptions: + +- **Note callers:** `\f` + caller + content + `\f*` with the note's USJ `caller` value (`+`, `-`, or literal) — currently the caller is silently dropped (and rewrites to `+` on re-paste). This deliberately *exceeds* P9, where plain-text copy carries only the caller glyph and full notes survive only in CF_HTML. Rationale: in P10 there is no CF_HTML fidelity carrier for external apps, and the ticket requires "copy from Standard → paste into a plain-text editor: markers present and correctly placed". + +- **Collapsed notes** contribute their full (visually hidden) `\f …\f*` bytes. "What you copy" > "what you see", for fidelity. This is documented, not changed. + +- **Display-NBSP handling:** Display-NBSP → space; data-NBSP → `~`; `text/html` and `application/x-lexical-editor` stay unnormalized (existing, pinned). + +- **Byte fidelity around note-internal markers:** Copied text must not contain spaces the source USFM lacks. Live repro (2026-08-07): copying `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*Den` produced `… \xo* \xt 2Cor 4:6\xt* \x*Den` — phantom spaces after each closing marker (display-separator NBSPs leaking through the blanket NBSP→space mapping). The copy walker must be source-faithful, mirroring the serialization inverse, not a blanket `replaceAll`. + +- **Multi-block selections:** One `\n` between blocks; a selection starting mid-paragraph omits that paragraph's own `\p ` glyph (matches P9's rendered-text copy). Pinned, not changed. + +- **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). + +### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) + +**Fixed gap (2026-08-13), lexical-JSON flavor only:** the "exact node tree" promise did not hold for a selection containing an `UnknownNode` (figure, table, sidebar, periph, ref, optbreak) — fixed here for optbreak; the identical mechanism affects the other five kinds too, left as a measured, un-fixed residual (see "Deferred / Out of Scope" below). `@lexical/clipboard`'s JSON generator (`$appendNodesToJSON`, the function behind the `application/x-lexical-editor` flavor) computes node exclusion from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, hardcoded for EVERY copy-out destination `$appendNodesToJSON` itself handles, not only actual HTML generation (`'clone'` is never passed by any Lexical-shipped code path in the installed version). This change touches ONLY that flavor: `$appendNodesToHTML` (`@lexical/html`, the function behind `text/html`) computes the same `excludeFromCopy('html')` value, but `UnknownNode.exportDOM()` always returns `{element: null}` for every kind, and `$appendNodesToHTML` returns early on a null element BEFORE ever consulting the exclusion value it just computed — so `text/html` output is byte-identical before and after this fix, for every `UnknownNode` kind. + +`UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. + +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked a CHILD-BEARING optbreak "selected" under Lexical's default `isSelected`, producing a CHILDLESS `{type:"optbreak"}` placeholder in the copy even though the live node has a child — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. Every OTHER `UnknownNode` kind (figure, table, sidebar, periph, ref) keeps the OLD, still-excluding behavior — not because their content differs meaningfully from optbreak's (it doesn't: every kind's marker/attribute bytes are the same content-free `ImmutableTypedTextNode` display decorators, built by the identical `unknownDisplayParts` machinery), but pending per-kind paste verification this task did not do. Pinned end-to-end (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, plus a direct unit pin on the narrowed predicate in `UnknownNode.test.ts`. + +Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` +list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when +the clipboard carries the same-namespace flavor — attribute-context paste always claims plain-text +insertion instead (see S3's "Paste inside an attribute display run" subsection below). This closes a +gap the fact recorded in S3 ("the private Lexical flavor is dead on Ctrl+V") does not cover: that +fact is about the RECONSTRUCTED `DataTransfer` a programmatic/reconstructed paste dispatch uses +(`clipboard.utils.ts`'s `pasteSelection`), not about a genuine LIVE NATIVE `paste` event, whose +`clipboardData` is the browser's own and can still carry the flavor for a same-page/session copy — +this is the same "live native paste event that still has it" case S2's own node-tree fast path is +written to keep for ordinary content. + +### S3. Paste (Standard view, external) — plain text IS the fidelity carrier + +Any paste without a same-namespace Lexical payload is treated as a USFM text fragment: take `text/plain` (fall back to text derived from `text/html` with block boundaries → `\n`), normalize NBSPs **positionally**, insert as text, let Tier 2 re-tokenize markers. + +- **One extraction layer for every paste claim:** `getPastePayload` (`whitespaceDisplay.plugin.utils.ts`) is the single place a `PASTE_COMMAND` payload is read. It performs the jsdom-safe clipboard duck-check, decodes `text/html` through `htmlPasteText`, normalizes `\r\n` and bare `\r` to `\n` BEFORE any caller tests for a line break, and reports whether the clipboard carries this editor's own `application/x-lexical-editor` flavor. Four handlers race on `PASTE_COMMAND` — the in-note `\fp` claim at CRITICAL, this Standard-view claim and the character-stack line replay at HIGH, the paragraph-split arm at LOW — and they must agree byte-for-byte on what was pasted, so none of them reads the clipboard itself. What each claim DOES with `isInternal` still differs by design (the in-note claim covers internal pastes; the Standard-view claim declines them unless the selection touches an attribute run). + +- **A multi-line payload is replayed line by line through `INSERT_PARAGRAPH_COMMAND`**, not through a bare `selection.insertParagraph()` per newline. Going through the command is what makes a paste landing inside a character-style stack close and reopen that stack at every line break, exactly as Enter does — a raw split tears the span, dropping its closing marker and leaving every line after the first outside the reopened style. The selection is removed up front rather than relying on the first `insertText` to replace it, so a payload whose first line is empty (a leading newline) cannot split a still-selected range. `MarkerEditContext.splitExpected` is armed before the first line goes in — the command's own handler arms it for each dispatch, but the first line is inserted before any dispatch runs — and `Tier2Context.pasteRebuildArmed` is armed unconditionally, single-line pastes included, since a single line can trigger the own-marker-prefix dedup just as easily. Foreign HTML formatting is dropped by design — Standard view is markers-as-text, exactly like P9's Standard view where the reformat pipeline re-tokenizes everything. This generalizes the existing NBSP-gated handler and kills both live-observed corruptions plus the latent doubled-glyph path (re-imported Standard-view HTML producing a `CharNode` via `data-marker` importDOM *and* literal `\nd` text, since MarkerNode has no importDOM). + +- **Positional NBSP normalization (replaces the blanket NBSP→`~`), three-part rule:** (1) a leading NBSP — string-start or right after a newline — is a structural separator with nothing in front of it to match against and becomes a space; (2) an NBSP immediately FOLLOWING a marker token (`\marker`, nested `\+marker`, either's closer, or a milestone's anonymous `\*` self-closer) is the required opener/closer separator and becomes a space; (3) an NBSP immediately PRECEDING a marker token is a structural spacer with no source counterpart and is DROPPED entirely (neither spaced nor kept as data) — `createNote` (`usj-editor.adaptor.ts`) appends a spacer after EVERY note child, not just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller; a browser-hop/html-derived copy of a collapsed note therefore carries structural NBSPs on BOTH sides of its interior markers, not only after the opener. Every remaining NBSP is user data → `~`. This is P9's `PostprocessUsfm` model. Live repro that mandates it (2026-08-07): pasting P10's own copied footnote back produced `\f~ \fr~1:1 ~ \ft~Caller test.~ \f*` (every display-NBSP became `~`, breaking marker recognition); a browser-hop paste of `\nd …\nd*` produced `\nd~light … \nd*` with an unmatched pair. **Rule precedence when both position rules match the same NBSP:** a closed-children note shape (`\xo*` immediately followed by NBSP immediately followed by `\xt`) sits after a closing marker AND before the next opening marker at once; the after-marker rule wins and the NBSP becomes a space rather than being dropped, because `$normalizePastedNbsp`'s `.replace` chain applies `AFTER_MARKER_NBSP` before `BEFORE_MARKER_NBSP`, consuming the NBSP into a space before the before-marker pass ever sees it. The tie-break is correct, not incidental: an after-closer NBSP in ordinary body text (`\nd*and`) is a real content space that dropping would silently lose, and the case is reachable only via a foreign NBSP-preserving carrier — P10's own `text/plain` copy (S1) never emits an NBSP in this position to begin with. + +- **Optbreak (`//`) is a non-backslash marker token the positional NBSP rule does not recognize (characterized, not a live gap):** `AFTER_MARKER_NBSP`/`BEFORE_MARKER_NBSP` (`whitespaceDisplay.plugin.utils.ts`) only match `\`-shaped tokens, so an NBSP adjacent to `//` is not treated as display whitespace and falls to the blanket `~` rule like any other interior NBSP. This is UNREACHABLE via P10's own copy, and DOUBLY so: (1) S1 above establishes `text/plain` never carries an NBSP next to `//` to begin with, and the plain payload is always preferred when present; (2) even if `text/plain` were somehow absent, P10's own `text/html` never carries `//` at ALL for an optbreak (see the "Known Accepted Asymmetries" entry below), so there is no `//` for a foreign-sourced NBSP to even be adjacent to in P10's own html either. Reachable only via a genuinely foreign clipboard source supplying its OWN `text/html` (never P10's) with no `text/plain` at all: a synthetic `

before // after

`-shaped payload settles to `before~// after` — the optbreak token itself still recognizes correctly (the tokenizer's `//` split is a plain string `.split("//")`, unaffected by an adjacent `~`), and the NBSP itself is genuine data FROM that foreign source (not a byte P10 invented) — the `~` it becomes is the correct display form for real data-NBSP under the "every remaining NBSP is user data → `~`" rule above, just landing one position earlier than a marker-adjacent rule would recognize. Characterized (not extended/fixed — no live repro reaches it) in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). + +- **The private Lexical flavor is dead on Ctrl+V — mechanically verified, not inferred:** `pasteSelection` (`clipboard.utils.ts`) rebuilds its `DataTransfer` from `navigator.clipboard.read()`, Chromium's async Clipboard-API read. That API, with no `unsanitized`/custom-format opt-in (none is used here), exposes only a fixed, sanctioned MIME allow-list (`text/plain`, `text/html`, and a short list of others) — `application/x-lexical-editor` is not one of them, so the rebuilt `DataTransfer` a real Ctrl+V dispatches can **never** contain it, by construction of the read API itself. This is a static fact about the code path (confirmed by reading `clipboard.utils.ts`), not an inference from the live tilde-corruption symptom — that symptom is merely consistent with it. Consequence: the USFM text carrier IS the internal path too — acceptable once S1's copy is byte-faithful — and the sync `ClipboardEvent` path keeps the node-tree fast path when the flavor is present (S2). + +- **Recorded limitation — structure-protected editors get no NBSP normalization:** `$handlePasteForStandardView` declines outright when the document is structure-protected, so `StructureKeyboardPlugin`'s HTML sanitizer governs the paste instead (in its `"protected"` mode) (both register `PASTE_COMMAND` at the same priority; the marker-edit engine mounts first, so an explicit decline is required or it would starve the sanitizer). A structure-protected editor's external plain-text pastes therefore receive NO positional NBSP normalization at all — protection governs, and this trade-off is accepted rather than teaching the sanitizer path a second NBSP policy. + +- **Marker-bearing lines own their markers:** A pasted line starting with a paragraph-marker literal does NOT also get the host paragraph's cloned prefix (no doubled/empty paragraphs). Marker-free lines inherit the host marker. **Live-verified correct on c7e666fa (2026-08-07)** — `\p one\n\p two` and `tail\n\q1 line` both produced exactly the target structure. + +- **Pasted `\c` / `\id`:** Strip during paste normalization (approved). Live-verified harm (2026-08-07): pasting `\c 2` mid-chapter put a chapter node in the editor and poisoned the save loop — PDP rejects every save with "Multiple chapter markers present", the error surfaces only in the renderer log, disk and other editors silently stop updating. (P9 destroys pasted `\c` at save with a user-facing error.) + +- **Paste inside an attribute display run — paste ≡ typing:** a selection that TOUCHES attribute-display text at either end — a char span's bare `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value — is inserted exactly as the SAME characters typed at the SAME caret (or over the same selection) would be: each `\n` becomes a single space, per-newline, not run-collapsed (attribute values are single-line; there is no multi-line attribute byte shape to collapse INTO — `"a\n\nb"` pastes as `"a b"`, two spaces), and NO chapter/book-id strip, NO positional NBSP mapping, and NO marker tokenization run against the pasted bytes — a pasted `\c 5` or `\p` here is literal value text, not a structural marker, and a pasted NBSP passes through unchanged (matching `$displayWhitespaceTransform`'s own skip of attribute-tagged text). A selection that only PARTLY sits inside attribute-display text (one end in, one end out) is included too — see below. The existing attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, `$resolvePendingMarkers`) then re-tokenizes the displayed bytes back into node state identically whether they arrived by typing or paste. + + TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the saved file diverged from the editor. `sid="things"` carries no NBSP; the pre-branch build's paste handler still had its OLD NBSP-gated form (see the "positional NBSP normalization" bullet above — that gate was generalized to claim every external paste, NBSP or not, on 2026-08-07), so the most plausible mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion — NOT a same-namespace `application/x-lexical-editor` flavor on the clipboard (this document's own "private Lexical flavor is dead on Ctrl+V" fact, above, still holds for the reconstructed-`DataTransfer` paste path this repro most likely used). + + What matters for THIS branch's code is the SHAPE the corruption takes once ANY handler declines an attribute-context paste to Lexical's default rich-paste node insertion — reproduced directly here (`attributeContextPasteFidelity.test.tsx`'s "root cause" describe): it has no notion that an attribute run's text must stay inside its one tagged TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph's sibling text into one plain node, destroying the attribute display and the closing marker and leaving the pasted bytes loose in body content. On this branch's CURRENT code (a single-line, NBSP-free, flavor-free external paste is already safe since 2026-08-07's generalization), the confirmed regression classes this paste-≡-typing rule closes are: (1) a live native paste event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for when that reaches a real handler); (2) a multi-line plain-text payload, which the ordinary pipeline would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND`; (3) a marker-bearing payload (`\c 5`), which the ordinary pipeline's `\c`/`\id` strip would eat bytes out of an attribute VALUE that were never a chapter token; (4) a MIXED selection (one end in the run, one end out) combined with either (1) or (2) — the selection touches attribute context, so it must not reach either risky branch merely because its other end sits outside. + + Pre-existing precedence, UNCHANGED by this rule: the CRITICAL-priority in-note multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs before the Standard-view external-paste handler and still wins for a multi-line payload whose selection touches EXPANDED note content — an attribute run that happens to sit inside an expanded note's content is reached by this rule only when that in-note claim itself declines. See `$handlePasteForStandardView`'s doc comment (`whitespaceDisplay.plugin.utils.ts`) for the full mechanism. + +### S4. Paste-as-plain-text (Ctrl+Shift+V / context menu) — narrows the payload to `text/plain` + +Under S3 this is semantically identical to a normal external paste in Standard view. There is NO "paste literally, don't tokenize" mode; P9 has none either (no Paste Special exists in P9). Live-confirmed identical 2026-08-07. Documented + pinned as equivalence, no new UI. TJ note: the equivalence is Standard-view-scoped — other views may legitimately differentiate the two commands later; out of scope here. + +### S5. Hidden-marker views (`formatted`, `paragraph-structure`) + +- **Copy-out is prose** (no marker text) — existing, gets a gate test protecting Standard-view handlers from leaking there. + +- **The paste gate** (swallow anything containing `\` **or `/`**) is over-broad — it eats URLs, dates, "and/or" (live-confirmed 2026-08-07). **Pre-existing upstream behavior** (`CommandMenuPlugin` on `origin/main`, predates the standard-view branches) — per TJ, NOT fixed here; recorded in the semantics doc's deferred list. Structural apply-markers-on-paste in formatted views likewise out of scope. + +### S6. Cut = copy + `removeText()` (existing) + +The WI-2 filed quirk — a selection-delete ending exactly at a just-settled char-span boundary absorbing one adjacent character — gets a targeted regression pin. **Outcome (2026-08-07):** does NOT reproduce at the Lexical selection level. Pinned as regression armor for both `CUT_COMMAND` and a plain `removeText()`, each given a selection whose focus is an element point mirroring DOM `range.setEndAfter(spanElement)` — the exact boundary shape the live repro's programmatic DOM selection used. Both variants produced byte-exact, correctly-scoped results, narrowing the live repro's root cause to the DOM Range → Lexical selection-resolution layer (`applyDOMRange`/`$internalResolveSelectionPoints`), which is not exercised by a Lexical-level `RangeSelection` and so remains covered only by the E2E selection steps (Task 8). + +### S7. Undo — every paste, including rebuilds it triggers, is one undo step + +This holds for BOTH paste shapes, and the earlier qualification recorded here no longer applies. + +- A paste whose rebuild happens SYNCHRONOUSLY, within the paste's own commit (a terminated marker, or a marker-bearing line's Tier 2 rebuild via `armPasteRebuildDedup`) is one commit and one undo step, as it always was. +- A paste whose literal text PENDS and only re-tokenizes on a LATER caret departure (e.g. a bare `//` with no terminator) is still TWO commits — the insertion and the departure-triggered settle — but only ONE undo step, because a settle is never its own history entry: it merges into the entry holding the edit it completes. A single press therefore lands on the true pre-paste state rather than stranding the user on a half-settled literal they never typed. Which carrier the clipboard happened to provide (plain payload vs. the same-namespace Lexical flavor) is invisible to undo. + +The multi-line `INSERT_PARAGRAPH_COMMAND` replay does not change this either: every dispatch and every `insertText` runs inside the ONE `editor.update()` the `PASTE_COMMAND` dispatch is already in, so a three-line paste is still a single history entry. + +Measured, not assumed: pinned for optbreak in `optbreakClipboardFidelity.test.tsx`'s "undo after a PLAIN-payload paste is ONE step" and "undo after the lexical-flavor paste restores the pre-paste (empty host) USJ in one step" pins, and for the synchronous shape in `markerPasteFidelity.test.tsx`'s `undoAndSettle` pins. + +--- + +## Paratext 9 Reference Behavior + +Understanding P9's clipboard model provides context for P10's design decisions: + +- **Data formats:** P9 writes three clipboard formats: (1) `text/plain` containing the rendered selection text; in Standard view, markers are rendered as literal text, so plain-text copy includes markers; (2) `CF_HTML` (or `text/html`), which is the fidelity carrier — contains a DOM fragment where notes travel only as HTML comments with USFM syntax (``), while plain text receives only the caller glyph; (3) a private `ParatextLanguageId` format for app-internal use. + +- **Paste workflow:** P9's paste is HTML-first and unsanitized. All normalization happens in a subsequent reformat pipeline: `CleanHtml` → Unicode NFC/NFD per project settings → reverse XSLT → `PostprocessUsfm` (removing FEFF marks; applying NBSP policy — token-leading NBSP → space, interior kept per `AllowInvisibleChars` setting) → `UsfmToken.NormalizeUsfm` (full re-tokenization: whitespace collapse, newlines inserted before paragraph/verse markers, RTL mark handling). + +- **Paste Special:** P9 has no Paste Special command. Both normal paste and plain-text paste ride the same normalization path. + +- **Structural errors:** P9 destroys pasted `\c` markers at save time with a user-facing error message: "You cannot put a \c marker in the middle of a chapter". + +- **Unknown markers:** Pasted unknown USFM markers are kept and flagged red in the UI, never stripped. + +--- + +## Known Accepted Asymmetries vs Paratext 9 + +These differences are documented and intentional; they are not implemented as workarounds: + +1. **P9→P10 paste from P9's Formatted view:** P9's Formatted-view copy carries markers only in `CF_HTML` using P9-specific CSS classes, which P10 does not parse. When P10 pastes this content, it receives it as prose (markers are lost). P9's Standard-view copy, by contrast, pastes into P10 perfectly via plain text. + +2. **P10 formatted-view copy loses note callers in plain text:** P9 Standard-view copy preserved the caller glyph in plain text (though full notes survived only in `CF_HTML`). P10's formatted-view copy produces prose-only output, so note callers do not appear in plain text. This is acceptable — prose copy stays clean and users can switch to Standard view for marker-aware copy. + +3. **Standard-view external paste from an html-only clipboard payload (no `text/plain`) loses the note caller:** P9's fidelity carrier is `CF_HTML` (footnotes travel as HTML-comment USFM); P10's is `text/plain` (S3). `ImmutableNoteCallerNode`'s own DOM export carries a collapsed note's caller only as a `data-caller` attribute, never as visible text, so `htmlPasteText`'s plain `body.textContent` read (used only when `text/plain` is absent) can't recover it — a genuinely html-only round trip loses the caller entirely. The gap is narrow: a real Ctrl+V of P10's own copy always carries `text/plain` alongside `text/html` (S1/S2), so it is reachable only via a foreign clipboard source that supplies `text/html` without `text/plain`. Currently documented only as a test comment (`whitespaceDisplay.plugin.utils.test.tsx`, the "browser-hop/html-derived collapsed-footnote shape" test). + +4. **An optbreak's `//` is entirely absent from `text/html`:** `UnknownNode.exportDOM()` unconditionally returns `{element: null}` for every kind (not just optbreak) — `@lexical/html`'s node walk (`$appendNodesToHTML`) treats a null-element export output as "stop here, do not visit children" and returns immediately, so the optbreak's `//` display child is never even reached, let alone rendered. Unlike the note-caller gap above (a browser-hop/foreign paste can still recover most of the content, just not the caller), an html-only optbreak copy loses the `//` completely with no trace — the flanking text closes up around the gap. The gap is equally narrow: a real Ctrl+V of P10's own copy always carries `text/plain` alongside `text/html` (S1), so this is reachable only via a foreign clipboard source supplying `text/html` without `text/plain`. Characterized (not fixed — `exportDOM`'s null-element short-circuit is a separate mechanism from the `excludeFromCopy` gap S2 fixes, and giving `UnknownNode` an actual DOM export is a bigger design question this task did not take on) in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). + +--- + +## Known Lossy Constructs — Copy→Paste Round Trip + +Found by a corpus-style sweep (`clipboardCorpusRoundTrip.test.tsx`): for every fixture in the +shared USJ round-trip corpus (`corpus-data.ts`), select the chapter's content, copy, paste into a +fresh editor holding the same chapter header, and compare the resulting USJ to the source. Two of +the six failures below are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes +capable of representing the construct, so no paste-side fix is possible without a different +carrier. Two are real, out-of-scope structural gaps in how a paste's rebuild is scoped (the same +gap, reached by two different constructs). One is an out-of-scope interaction between a deferred +settle and the paste-scoped prefix dedup. One is ACCEPTED normalization that matches Paratext 9's +own behavior, not a bug at all. All six fixtures stay in the sweep as `it.skip`, not deleted, so a +future engine change un-skips them automatically instead of the gap going unnoticed. + +1. **Cross-reference `` target wrapper (inherent):** USJ's `ref` element is a wrapper USFM + itself never carried (`unknownUsfm.utils.ts`'s own doc comment: "USJ invented this container, + USFM never carried it... only its child text renders"). Source content + `["See ", {type:"ref", loc:"GEN 1:1", content:["Genesis 1:1"]}, " for details."]` copies as + plain `"See Genesis 1:1 for details."` with no marker bytes anywhere marking the wrapper's + extent; paste re-tokenizes it as ordinary prose (`["See Genesis 1:1 for details."]`, the `ref` + wrapper gone). A raw USFM export of this same fixture has the identical gap — not specific to + clipboard mechanics. + +2. **Sidebar `\esb`/`\esbe` (structural — a paste-rebuild SCOPING gap, not a missing tokenizer + rule; out of scope here):** the tokenizer itself is not the problem — `usfmFragmentToUsj.ts` + already implements the `\esb`/`\esbe` pairing (its `SIDEBAR_MARKER`/`SIDEBAR_END_MARKER` + assembly case tracks an open sidebar across tokens and closes it on `\esbe`). The real mechanism + is upstream and structural: a sidebar's nested `\p` child is a real `ParaNode`, and the copy + walker (`$selectionToUsfmText`) inserts a `\n` before any non-inline `ElementNode` boundary it + crosses — a `ParaNode` is non-inline, so a `\n` lands between `\esb \cat History\cat*` and the + nested paragraph's own content, even though both came from ONE sidebar. On paste, + the paste replays every `\n` as an `INSERT_PARAGRAPH_COMMAND` dispatch, so that single `\n` + turns into TWO sibling `ParaNode`s where the source had one sidebar wrapping one paragraph. Tier + 2 then re-tokenizes strictly per paragraph — `$requestTier2ForNode` (`tier2Rebuild.utils.ts`), + the only production call site, always invokes `$rebuildParas([current], context)` with a + single-element array — so the tokenizer's "current open sidebar" state can never span the two + separate `$rebuildParas` calls the two now-sibling paragraphs each trigger; the pairing that DOES + exist in `usfmFragmentToUsj.ts` never gets the chance to run across both lines at once. Result: + an UNCLOSED sidebar (`closed:"false"`, no content), the inner paragraph hoisted out to become a + top-level sibling, and a stray EMPTY paragraph with marker `"esbe"`. Tables hit the identical mode + — their rows and cells are real block-level `ElementNode`s + (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the copy walker's + non-inline-boundary `\n`-insertion rule fires for every one of them; see item 4 below. A real + fix means grouping a paste's newly-inserted SIBLING + paragraphs that originated from one selection back into a single rebuild fragment before Tier 2 + tokenizes — a Tier-2 rebuild-granularity change, genuinely out of this work item's scope. The + byte-level corruption stays pinned by the skip rather than fixed. + +3. **`closed="false"` char span followed by more paragraph content (inherent):** a `closed="false"` + span has, by definition, no closing marker byte anywhere in its own USFM. When such a span is + not the last thing in its paragraph (`Tell the Lord plainly.`), the + copied text (`\nd Lord plainly.`) carries no byte marking where the span's content ends and the + trailing prose resumes, so paste has nothing to stop at "Lord" on — it swallows the rest of the + paragraph into the span (`{marker:"nd", content:["Lord plainly."]}`, the top-level `" plainly."` + string gone). The sibling `"unclosed note (closed=false)"` fixture, whose unclosed span IS the + last thing in its paragraph, has no such trailing content to lose and round-trips clean — + confirming the ambiguity is specifically about trailing content after an implicit close, not + `closed="false"` itself. + +4. **Table rows and cells split into sibling paragraphs (structural — the SAME paste-rebuild + SCOPING gap as item 2, out of scope here):** a table's rows and cells are now real block-level + `ElementNode`s (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), not the + inline `UnknownNode`s they were when this sweep was first written, whose unconditionally-`true` + `isInline()` kept a table byte-contiguous within one paste-insertion unit. The copy walker + inserts a `\n` at every non-inline element boundary it crosses, so one table copies out as eight + lines (`\tr `, `\th1 Day`, `\th2 Tribe`, `\tr `, `\tc1 First`, `\tc2 Judah`, plus the flanking + prose). Paste replays each line as its own paragraph and Tier 2 re-tokenizes strictly per + paragraph, so the `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never sees + the whole table in one pass: the result is empty `table:row` wrappers with the cells stranded as + top-level sibling paragraphs. Same fix as item 2 — group a paste's newly-split sibling paragraphs + back into one rebuild fragment — and equally a Tier-2 rebuild-granularity change out of scope + here. + +5. **A pasted milestone that ejects content leaves the host's redundant paragraph glyph behind + (structural — a paste-PROVENANCE lifetime gap, out of scope here):** the copied text is + byte-correct (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host + paragraph with no marker prefix of its own round-trips clean — so neither the copy walker nor the + positional NBSP rule is involved. Two deliberate behaviors compound instead. A milestone whose + rebuild would EJECT content does NOT settle inside the commit that produced it + (`markerEditTier2Trigger.utils.ts`: ejection moves bytes out of the milestone and past a closer, + so applying it the instant the `\*` lands would rearrange the line under a caret the user is + still on); it pends until caret departure, and the departure settle then performs the identical + rebuild correctly. But `Tier2Context.pasteRebuildArmed` — the flag that lets + `$withoutRedundantOwnPrefix` drop the host's now-redundant `\p ` glyph when the pasted line + carries its own — is armed only for the PASTE's own update. By the time the deferred settle runs, + the rebuild no longer knows the fragment came from a paste, so it takes the TYPED path + (split-with-empty, the P9-parity behavior PT-4201 deliberately kept for typed input) and the + document ends with an empty `\p` paragraph ahead of the correct one. Closing it means giving the + dedup's arm a lifetime that spans the deferred settle — a paste-provenance change, not a + tokenizer one — which also has to answer what happens if the user TYPES between the paste and the + departure. Recorded rather than guessed at. + +6. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a + bug):** isolated with a minimal non-corpus repro: pasting the literal text `"\p X"` (marker, its + own required separator, and a SECOND, real content-leading space) into a fresh empty `"\p"` host + produces `"\p X"` — one space, not two. The mechanism is `consumeSeparator()` + (`usfmFragmentToUsj.ts`), whose own comment states exactly this: "Consume the separator + whitespace after an opening marker (PT9 skips it) — all leading whitespace, not just a single + space." That mirrors Paratext 9's own `NormalizeUsfm` re-tokenization pass (see "Paratext 9 + Reference Behavior" above: "whitespace collapse, newlines inserted before paragraph/verse + markers"), which likewise collapses a whitespace run after a marker during its own paste reformat + pipeline — P10 doing the same is parity with P9, not a divergence from it. Corpus symptom: + copying ` Leading space precedes this text.` (source content + `" Leading space precedes this text."`) round-trips through paste to + `"Leading space precedes this text."` — the leading space is gone, the same as it would be in + P9. Kept in the sweep's skip list (the byte-level comparison genuinely differs from the source) + but is NOT a fix candidate. + +--- + +## Deferred / Out of Scope + +The following items are recorded as deferred and not addressed in this plan: + +1. **Pre-existing upstream CommandMenuPlugin `/`-swallow:** The paste gate in hidden-marker views inspects only `text/plain` (`event.clipboardData?.getData("text/plain")`) and blocks the whole paste — no partial insertion — whenever that string contains `\` or `/`, so it also eats URLs, dates, and common phrases like "and/or" outright; the only feedback is a `logger?.info(...)` call, which never reaches the user (no toast, no visible indication the paste was dropped). The check is `text/plain`-only, so a clipboard payload carrying **just** `text/html` (no `text/plain` key at all) skips the gate entirely — confirmed by exercising the paste path directly (2026-08-07): an html-only, backslash-bearing payload reaches Lexical's own HTML-import fallback instead of being blocked, unlike the byte-identical `text/plain` payload. This behavior originates in the `CommandMenuPlugin` on `origin/main` and predates the standard-view branches. As of 2026-08-07, this is not fixed here per TJ's guidance; it belongs to a separate upstream cleanup task. + +2. **Typing `\c` mid-chapter poisons the save loop:** If a user manually types `\c 2` in the middle of a chapter, it corrupts the editor state. The PDP error "Multiple chapter markers present" surfaces only in the renderer log, and disk/other editors silently stop updating. This is a data-fidelity issue tracked separately (WI-10's data-fidelity audit) and is not addressed by paste normalization. + +3. **Structural marker application on paste in hidden-marker views:** When a user pastes USFM markers into a hidden-marker (formatted/paragraph-structure) view, there is no logic to apply those markers as structural elements. This is by design — hidden-marker views are not marker-aware and should reject or convert marker input. Implementation is out of scope. + +4. **P9-style CF_HTML class parsing:** When P9's Formatted-view copy (which uses CSS classes to encode markers in `CF_HTML`) is pasted into P10, the classes are ignored because P10 does not parse P9's class-based marker encoding. This is an asymmetry vs P9; users should switch to Standard view for marker-aware copy from P9 into P10. + +5. **Custom `text/usfm` MIME type / "Copy as USFM" command:** A dedicated `text/usfm` MIME type or separate "Copy as USFM" command was considered and rejected. Once `text/plain` IS USFM (S1), a separate command adds no value; the existing single-copy-format approach is sufficient. + +6. **Popover expanded-note `isStandardView` gating issue:** An issue with note-expansion behavior in popovers has an open tracking document at `docs/superpowers/specs/2026-07-07-standard-view-followups.md`. This is not addressed in the current plan. + +7. **Paste-rebuild scoping loses a sidebar's `\esb`/`\esbe` pairing across a copy-introduced paragraph split (corpus-found 2026-08-07):** a sidebar's nested paragraph is a real `ParaNode`, so copying a sidebar inserts a `\n` before it and pasting that `\n` splits one sidebar into two sibling paragraphs; Tier 2 rebuilds strictly per paragraph (`$requestTier2ForNode` always calls `$rebuildParas` with a single-element array), so the `\esb`/`\esbe` pairing `usfmFragmentToUsj.ts` already implements never sees both halves in the same tokenize pass. See "Known Lossy Constructs — Copy→Paste Round Trip" above (item 2) for the full mechanism. A real fix means grouping a paste's newly-split sibling paragraphs back into one rebuild fragment — a Tier-2 rebuild-granularity change, out of scope for this test-only task. + +8. **2SA corpus (`libs/test-data/src/data/2sa.usj.ts`) not included in the copy→paste corpus sweep:** this fixture IS single-chapter (one top-level chapter object), so the earlier "multi-chapter/book-level" exclusion reason recorded in an earlier draft of this work was wrong. It was tried against the sweep's harness directly and does not round-trip clean: it hits the sidebar gap above three times, plus several additional, unrelated fidelity gaps (an empty `\b` blank-line paragraph, a verse's derived `sid` attribute, a `\ref` target) that this sweep's one-construct-per-fixture design has no clean way to itemize as a single byte diff. `tier2Rebuild.corpus.test.tsx` already exercises this fixture, but for a narrower, different property (an unedited `$rebuildParas` call refusing as a fixed point) that does not cover the paste path this sweep does — so the exclusion is a real coverage gap, not a redundant re-test. Recorded here rather than force-fit into the sweep. + +9. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. + +--- + +10. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). + +11. **Lexical-flavor copy/paste of figure/table/sidebar/periph/ref UnknownNode kinds (measured, 2026-08-13):** S2's `excludeFromCopy` fix above is scoped to "optbreak" only; every other `UnknownNode` kind keeps the OLD, still-excluding behavior, PENDING per-kind paste verification this task did not do. The residual shape was directly measured (not just inferred from the mechanism), and it is WORSE, in user terms, than the optbreak symptom this task fixed: a same-namespace lexical-flavor copy/paste of `
caption
` renders the figure's FULL literal USFM bytes on screen after paste — `\fig caption|src="cn01617.jpg" size="span" ref="1.18"\fig*` — because the figure's hoisted marker/attribute decorators still `decorate()` their own literal text as loose, unwrapped siblings, giving a completely convincing on-screen appearance that the figure came through intact. The pasted document's USJ, measured against that same on-screen result, silently drops the figure node AND every one of its attributes entirely — only its real caption TextNode child survives, merged into the surrounding paragraph's prose (`content: ["Before caption after."]`, no `figure` entry, no `file`/`size`/`ref` anywhere). A save at this point would silently persist the loss with no error, no warning, and a screen that still shows the figure's bytes — genuine silent data loss with a convincing display. `table`, `sidebar`, `periph`, and `ref` share the identical `excludeFromCopy`/`exportDOM(null)` mechanism (the same content-free `ImmutableTypedTextNode` display-decorator shape `unknownDisplayParts` builds for every kind) and are expected, not merely suspected, to exhibit the same class of residual, though only the figure case above was directly measured. Fixing this for all five kinds the way S2 now does for optbreak is a plausible, low-risk follow-up — narrowing `excludeFromCopy` further, one verified kind at a time — but is out of scope here. + +12. **The same paste-rebuild scoping gap now reaches TABLES:** item 7's sidebar mechanism is no longer sidebar-specific. A table's rows and cells are real block-level `ElementNode`s (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the copy walker's non-inline-boundary `\n` rule fires for every one of them and a single table pastes back as eight sibling paragraphs Tier 2 rebuilds one at a time. The `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never sees the whole table in one pass. Same fix, same scope call as item 7 — grouping a paste's newly-split sibling paragraphs back into one rebuild fragment. See "Known Lossy Constructs" item 4 above for the byte-level shape. + +13. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 5 above. + +## Test Mapping (S1–S7) + +| Semantic | Pinning Test | +|----------|--------------| +| **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"` describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | +| **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"` and `"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"` (external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction); `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | +| **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | +| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (confirmed regression class on this branch, not TJ's literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"milestone attribute run paste"`, and `"verse \\va run paste"` describes | +| **S4** Paste-as-plain-text (Ctrl+Shift+V) — equivalence to external paste | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"paste-as-plain-text equivalence (S4): no literal mode, plain always wins"` describe | +| **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | +| **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | +| **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | +| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; six known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"` fixture (tight spacing, no flanking spaces) is swept clean, not skipped | +| **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | +| **E2E** Real-browser clipboard round trip (Standard view, written, not yet run in this task) | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | + +--- diff --git a/libs/shared/src/nodes/features/UnknownNode.test.ts b/libs/shared/src/nodes/features/UnknownNode.test.ts index 3656b114a..c732c5caf 100644 --- a/libs/shared/src/nodes/features/UnknownNode.test.ts +++ b/libs/shared/src/nodes/features/UnknownNode.test.ts @@ -1,3 +1,4 @@ +import { $createImmutableTypedTextNode, ImmutableTypedTextNode } from "./ImmutableTypedTextNode.js"; import { $createParaNode, ParaNode } from "../usj/ParaNode.js"; import { createBasicTestEnvironment } from "../usj/test.utils.js"; import { $createUnknownNode, UnknownNode } from "./UnknownNode.js"; @@ -341,4 +342,40 @@ describe("UnknownNode", () => { ); }); }); + + // The narrowed predicate behind the same-namespace `application/x-lexical-editor` optbreak copy + // fix: @lexical/clipboard's `$appendNodesToJSON` computes node exclusion from + // `excludeFromCopy('html')` for EVERY copy-out format it handles, not only + // actual `text/html` generation (see `excludeFromCopy`'s own doc comment). A direct unit pin on + // the predicate here — independent of the integration-level clipboard pins in + // `optbreakClipboardFidelity.test.tsx` — catches a regression to this method locally, without + // needing the full editor/selection/clipboard-event machinery those pins exercise. + describe("excludeFromCopy()", () => { + it("does NOT exclude a CHILD-BEARING optbreak — the fix that lets a copied optbreak survive the lexical-flavor paste path", () => { + const { editor } = createBasicTestEnvironment([UnknownNode, ImmutableTypedTextNode]); + editor.update(() => { + const optbreak = $createUnknownNode("optbreak"); + optbreak.append($createImmutableTypedTextNode("marker", "//")); + expect(optbreak.excludeFromCopy("html")).toBe(false); + }); + }); + + it('excludes a CHILDLESS optbreak — a genuinely empty live husk falls back to the old, excluding behavior, matching text/plain\'s own "nothing here" for that shape', () => { + const { editor } = createBasicTestEnvironment([UnknownNode]); + editor.update(() => { + const optbreak = $createUnknownNode("optbreak"); + expect(optbreak.getChildrenSize()).toBe(0); + expect(optbreak.excludeFromCopy("html")).toBe(true); + }); + }); + + it('still excludes a figure, child-bearing or not — the fix is scoped to "optbreak" only; every other UnknownNode kind is an un-fixed, documented residual (clipboard semantics doc, "Deferred / Out of Scope")', () => { + const { editor } = createBasicTestEnvironment([UnknownNode, ImmutableTypedTextNode]); + editor.update(() => { + const figure = $createUnknownNode("figure", "fig"); + figure.append($createImmutableTypedTextNode("marker", "\\fig ")); + expect(figure.excludeFromCopy("html")).toBe(true); + }); + }); + }); }); diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index 2ccbd6f5e..8c7117c9c 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -2,6 +2,8 @@ import { UnknownAttributes } from "../usj/node-constants.js"; import { MarkerObject } from "@eten-tech-foundation/scripture-utilities"; import { $applyNodeReplacement, + $getSelection, + BaseSelection, DOMConversionMap, DOMConversionOutput, DOMExportOutput, @@ -205,8 +207,73 @@ export class UnknownNode extends ElementNode { return false; } + // Narrowed from an unconditional `return destination !== "clone"` to leave a CHILD-BEARING + // "optbreak" out of the exclusion. This changes ONLY the `application/x-lexical-editor` (lexical- + // JSON) flavor — `text/html` is byte-identical before and after, for every `UnknownNode` kind, not + // only optbreak: `$appendNodesToHTML` (`@lexical/html`) computes this same `excludeFromCopy('html')` + // value too, but `UnknownNode.exportDOM()` always returns `{element: null}`, and + // `$appendNodesToHTML` returns early on a null `element` BEFORE ever consulting the exclusion + // value it just computed — so this predicate's return value has no observable effect on `text/html` + // output at all, confirmed by reading `LexicalHtml.dev.js`. `'clone'` is never passed by any + // Lexical-shipped code path in the installed version (confirmed by reading + // `LexicalClipboard.dev.js`), so the old, unconditional form excluded every `UnknownNode` from the + // lexical-JSON flavor unconditionally. Excluding a node does not drop it silently: + // `$appendNodesToJSON` hoists the excluded node's own children into its parent's list in its place. + // For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a + // content-free DecoratorNode with no meaning once separated from its owning `UnknownNode`), that + // stranded a loose decorator on the same-namespace `application/x-lexical-editor` paste fast + // path — `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, so the + // paste silently lost the discretionary line break (live report: copying an optbreak worked, + // pasting the same clipboard back did not restore it). + // + // Every OTHER `UnknownNode` kind (figure, table, sidebar, periph, ref) keeps the OLD, still- + // excluding behavior — NOT because their content differs meaningfully from optbreak's (it doesn't: + // every kind's marker/attribute bytes are the SAME content-free `ImmutableTypedTextNode` display + // decorators optbreak's `//` is, built by the identical `unknownDisplayParts` machinery), but + // PENDING per-kind paste verification this task did not do. The measured shape of the un-fixed + // residual for those kinds (a lexical-flavor copy of `\fig …\fig*` renders its full literal USFM + // bytes on screen after paste — the hoisted decorators still `decorate()` their own text as loose + // siblings — while the pasted document's USJ silently drops the figure node AND its attributes + // entirely, keeping only its real caption text merged into surrounding prose) is recorded in the + // clipboard semantics doc's "Deferred / Out of Scope" list rather than fixed here. + // + // The `getChildrenSize() > 0` guard covers a genuinely childless live optbreak (an emptied husk + // mid-settle — `markerEditTier1.utils.ts`'s own husk-removal check uses the identical shape, + // "optbreak" plus zero children): such a node falls back to excluded, matching `text/plain`'s + // "nothing here" for the same case. It does NOT, on its own, cover a DIFFERENT carrier-agreement + // gap: a selection whose ending boundary resolves to an ELEMENT-type point ON this node at + // offset 0 (touching the wrapper without covering its own `//` child) still marks a CHILD-BEARING + // optbreak "selected" under Lexical's default `isSelected` (key-membership in + // `selection.getNodes()`, unaffected by this node's own child count), while the child itself is + // not — `$appendNodesToJSON` then serializes a CHILDLESS `{type:"optbreak"}` placeholder even + // though the live node has a child, disagreeing with `text/plain` (`$selectionToUsfmText`, which + // walks that same `getNodes()` list and correctly emits nothing for this boundary). The + // `isSelected` override below closes that second gap directly, at its actual source, since + // `excludeFromCopy` has no visibility into which of a node's children will end up selected. override excludeFromCopy(destination: "clone" | "html"): boolean { - return destination !== "clone"; + return this.getTag() === "optbreak" && this.getChildrenSize() > 0 + ? false + : destination !== "clone"; + } + + // An optbreak is only meaningfully "selected" (and so only copy-included, per the + // `excludeFromCopy` guard above) when at least one of its own display children is — mirrors + // `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at the same selection + // boundary, the same way Lexical's own base `isSelected` (`LexicalNode.prototype.isSelected`) + // already special-cases an inline DECORATOR node sitting as a parent's last child at an + // exactly-there boundary point, for the identical reason. Without this, a selection ending + // exactly at this node's own start (an ElementNode touch-boundary that covers none of its + // content) still counts the WRAPPER as selected via the default `ElementNode.isSelected` + // (key-membership in `selection.getNodes()`), while the child is not — producing a childless + // `{type:"optbreak"}` entry in the `application/x-lexical-editor` copy with no corresponding + // `//` in `text/plain`. Scoped to "optbreak" only, matching `excludeFromCopy`'s own scoping; every + // other kind keeps the inherited default. + override isSelected(selection?: BaseSelection | null): boolean { + if (this.getTag() !== "optbreak") return super.isSelected(selection); + const targetSelection = selection ?? $getSelection(); + if (!targetSelection) return false; + const selectedNodes = targetSelection.getNodes(); + return this.getChildren().some((child) => selectedNodes.some((node) => node.is(child))); } } diff --git a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx index 8a4374ec7..f80f0a939 100644 --- a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx +++ b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx @@ -3,10 +3,19 @@ import { EditorRef } from "./editor.model"; import { act, render } from "@testing-library/react"; import { Usj } from "@eten-tech-foundation/scripture-utilities"; import { createRef } from "react"; -import { KEY_DOWN_COMMAND, LexicalEditor } from "lexical"; +import { + $createPoint, + $createRangeSelection, + $getRoot, + $setSelection, + COPY_COMMAND, + KEY_DOWN_COMMAND, + LexicalEditor, + PASTE_COMMAND, +} from "lexical"; import { EditorRefPlugin } from "@lexical/react/LexicalEditorRefPlugin"; import { FORMATTED_VIEW_MODE, getViewOptions, STANDARD_VIEW_MODE } from "shared-react"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; // The Editor-level gate controls shared-react's `CommandMenuPlugin`, which `preventDefault`s a // typed or pasted `\` and `/` so those characters never land in the document. Editable marker @@ -76,3 +85,195 @@ describe("CommandMenuPlugin editable-mode gate", () => { expect(await keyWasBlocked(editor, "/")).toBe(true); }); }); + +// The Standard-view-only clipboard engine ($handleCopyForStandardView, shared by the COPY_COMMAND +// and CUT_COMMAND registrations, plus $handlePasteForStandardView — all in MarkerEditPlugin.tsx) +// is registered only inside a block gated on `hasStandardViewWhitespace(viewOptions)`, itself +// nested inside the whole plugin's `markerMode === "editable"` gate. Formatted view's `markerMode` +// is "hidden", so neither gate opens and none of those handlers register at all. These tests prove +// that absence behaviorally, the same way the suite above proves CommandMenuPlugin's gate: render +// the real Editor (no mock), dispatch the real command, and check what actually happened to the +// clipboard/document — not which plugin mounted. + +// A note whose caller is a printable, uncommon-elsewhere glyph ("+") so its presence in a copied +// payload is unambiguous evidence of the Standard-view USFM walker having run. +// `closed: "false"` marks the `\ft` char as never having its own closer, matching real +// ParatextData USJ for footnote content (see markerEdit.test-helpers.tsx's `noteUsx` doc comment) +// — not part of the `Usj` package type's `MarkerObject`, hence the cast below. +const sampleUsjWithNote = { + type: "USJ", + version: "3.1", + content: [ + { type: "book", marker: "id", code: "GEN", content: ["Test Book"] }, + { type: "chapter", marker: "c", number: "1" }, + { + type: "para", + marker: "p", + content: [ + { type: "verse", marker: "v", number: "1" }, + "first verse text", + { + type: "note", + marker: "f", + caller: "+", + content: [{ type: "char", marker: "ft", content: ["A note"], closed: "false" }], + }, + " after", + ], + }, + ], +} as unknown as Usj; + +/** + * jsdom does not implement the DOM `ClipboardEvent` class (`typeof ClipboardEvent` is + * `undefined` under this project's jsdom test environment). Formatted view registers no + * Standard-view handler ahead of `@lexical/rich-text`'s own COPY_COMMAND fallback, so a dispatch + * here falls all the way through to that fallback — which reads the bare, unimported, ambient + * global `ClipboardEvent` identifier before it ever touches the event. With the global entirely + * undefined that reference throws a `ReferenceError`, before any clipboard data is written. + * `@lexical/utils`'s `objectKlassEquals` (which the fallback uses to decide whether to honor the + * event) only compares constructor NAMES (`Object.getPrototypeOf(x).constructor.name`), not real + * inheritance, so a same-named stand-in registered on `globalThis` satisfies both the reference + * and the name check without jsdom needing to implement the real class. `StubClipboardEvent` is + * declared and renamed here, rather than a class literally named `ClipboardEvent`, so this file's + * own `ClipboardEvent` TYPE annotations keep referring to the real DOM type. + */ +class StubClipboardEvent { + clipboardData: { + getData: (type: string) => string; + setData: (type: string, data: string) => void; + }; + constructor(clipboardData: StubClipboardEvent["clipboardData"]) { + this.clipboardData = clipboardData; + } + preventDefault(): void { + /* not asserted on by these tests */ + } +} +Object.defineProperty(StubClipboardEvent, "name", { value: "ClipboardEvent" }); + +/** A jsdom-safe `ClipboardEvent`-shaped stub, with a real read/write `clipboardData` store, that + * satisfies `@lexical/rich-text`'s own `objectKlassEquals(event, ClipboardEvent)` check (see + * `StubClipboardEvent`'s doc comment) so RichText's default COPY_COMMAND handler actually runs + * instead of throwing. Registers `StubClipboardEvent` as the global `ClipboardEvent` for the + * caller to `vi.unstubAllGlobals()` afterward. */ +function formattedViewCopyEvent(): { event: ClipboardEvent; getData: (type: string) => string } { + const store = new Map(); + const clipboardData = { + getData: (type: string) => store.get(type) ?? "", + setData: (type: string, data: string) => store.set(type, data), + }; + vi.stubGlobal("ClipboardEvent", StubClipboardEvent); + return { + event: new StubClipboardEvent(clipboardData) as unknown as ClipboardEvent, + getData: (type: string) => clipboardData.getData(type), + }; +} + +/** Selects the whole document (first descendant through last descendant). */ +function selectWholeDocument(editor: LexicalEditor): void { + editor.update(() => { + const root = $getRoot(); + const first = root.getFirstDescendant(); + const last = root.getLastDescendant(); + if (!first || !last) throw new Error("expected the loaded document to have content"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(first.getKey(), 0, "text"); + const lastSize = "getTextContentSize" in last ? last.getTextContentSize() : 0; + selection.focus = $createPoint(last.getKey(), lastSize, "text"); + $setSelection(selection); + }); +} + +describe("MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("COPY_COMMAND yields Lexical's default payload: prose only, no marker glyphs, no note-caller character", async () => { + const ref = createRef(); + const lexicalRef = createRef(); + await act(async () => { + render( + + + , + ); + }); + const editor = lexicalRef.current; + if (!editor) throw new Error("lexical editor was not captured"); + + await act(async () => selectWholeDocument(editor)); + const { event, getData } = formattedViewCopyEvent(); + await act(async () => { + editor.dispatchCommand(COPY_COMMAND, event); + }); + + const text = getData("text/plain"); + // Real content was copied (proves the command actually ran, not that everything declined). + expect(text).toContain("first verse text"); + // No marker glyphs: `$selectionToUsfmText` (the Standard-view walker, gated out of this view) + // is the only thing in this codebase that renders literal backslash marker syntax like `\p`, + // `\v`, or `\f`; Formatted view's own node types (ImmutableTypedTextNode et al.) carry no + // backslash text at all, so any backslash here is direct evidence of the walker having run. + expect(text).not.toContain("\\"); + expect(text).not.toContain("\\f"); + // No note-caller character: the collapsed note's caller renders as a DecoratorNode + // (`ImmutableNoteCallerNode`) with no literal text of its own, so Lexical's default text + // extraction never emits it — only the Standard-view walker renders the caller as visible "+" + // text. Confirmed by direct comparison: calling `$getStandardViewClipboardData` against this + // exact selection produces "...first verse text + A note after" (caller present); Lexical's + // real default (asserted here) has no caller at all. + expect(text).not.toContain("+"); + }); + + it("an external \\-bearing paste is swallowed by CommandMenuPlugin's gate, not claimed by the Standard-view paste handler — nothing is inserted", async () => { + const editor = await renderEditor(FORMATTED_VIEW_MODE); + let before = ""; + editor.getEditorState().read(() => { + before = $getRoot().getTextContent(); + }); + + await act(async () => + editor.update(() => { + const last = $getRoot().getLastDescendant(); + if (!last) throw new Error("expected the loaded document to have content"); + const lastSize = "getTextContentSize" in last ? last.getTextContentSize() : 0; + const point = $createPoint(last.getKey(), lastSize, "text"); + const selection = $createRangeSelection(); + selection.anchor = point; + selection.focus = point; + $setSelection(selection); + }), + ); + + const payload = { "text/plain": "\\nd inserted\\nd*" }; + let prevented = false; + const event = { + clipboardData: { getData: (type: string) => payload[type as keyof typeof payload] ?? "" }, + preventDefault: () => { + prevented = true; + }, + } as unknown as ClipboardEvent; + + let handled: boolean | undefined; + await act(async () => { + handled = editor.dispatchCommand(PASTE_COMMAND, event); + }); + + // CommandMenuPlugin's PASTE_COMMAND handler (COMMAND_PRIORITY_NORMAL, shared-react) is the + // one that claims this: it swallows any paste containing `\` or `/`. The consolidated + // Standard-view handler ($handlePasteForStandardView, COMMAND_PRIORITY_HIGH inside + // MarkerEditPlugin) is not registered in Formatted view at all, so it never enters the race — + // were it wrongly registered here, its HIGHER priority would let it win instead, tokenize the + // marker text, and insert real structure, which the unchanged-document assertion below would + // catch. + expect(handled).toBe(true); + expect(prevented).toBe(true); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).toBe(before); + }); + }); +}); diff --git a/packages/platform/src/editor/Editor.tsx b/packages/platform/src/editor/Editor.tsx index 6ca91dff3..88cf0cdec 100644 --- a/packages/platform/src/editor/Editor.tsx +++ b/packages/platform/src/editor/Editor.tsx @@ -1128,6 +1128,7 @@ const Editor = forwardRef(function Editor( getMarker={markerLookup} logger={stableLogger} markerSettleDelayMs={markerSettleDelayMs} + structureProtectionMode={structureProtectionMode} /> { if (!isEnabled || !viewOptions) return; @@ -527,8 +553,10 @@ export function MarkerEditPlugin({ splitExpected: { current: false }, wholeParaDeleteExpected: new Set(), collapsedDeleteCaretParas: new Set(), + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), logger, + structureProtectionMode, }; contextRef.current = context; // Publishes the live pending set to the self-healing display syncs through a side channel. @@ -865,6 +893,20 @@ export function MarkerEditPlugin({ event && typeof event === "object" && "clipboardData" in event ? (event as ClipboardEvent) : null, + context.structureProtectionMode === "protected", + // Consumed by $paraMarkerDeletionTransform below, same as the + // INSERT_PARAGRAPH_COMMAND and LOW-priority PASTE_COMMAND handlers arm it for + // the paste paths that reach them — this HIGH-priority claim reaches the + // former only from its second line on, and the latter never. + () => { + context.splitExpected.current = true; + }, + // Consumed by $rebuildParas (tier2Rebuild.utils.ts) to scope the own-marker- + // prefix dedup to THIS paste's own update — see Tier2Context.pasteRebuildArmed's + // doc comment for why it must not also fire for typed input. + () => { + context.pasteRebuildArmed.current = true; + }, ), COMMAND_PRIORITY_HIGH, ), @@ -1073,6 +1115,7 @@ export function MarkerEditPlugin({ context.splitExpected.current = false; context.wholeParaDeleteExpected?.clear(); context.collapsedDeleteCaretParas?.clear(); + context.pasteRebuildArmed.current = false; context.rebuildAttempted.clear(); // Typing path: ScriptureReferencePlugin's async scrRef echo re-enters // `$moveCursorToVerseStart` and yanks the caret to the para/verse start via diff --git a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx new file mode 100644 index 000000000..9d4c91f50 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx @@ -0,0 +1,788 @@ +/** + * Paste (and cut) fidelity for a selection that TOUCHES an attribute display run — a char span's + * `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` run. + * + * TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span + * `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text + * `sid="things"`. Observed: the `who` attribute display AND the closing `\nd*` glyph disappeared + * from the editor, the pasted text rendered visually outside the span, and the saved file diverged + * from the editor. `sid="things"` carries no NBSP, and the pre-branch build's paste handler still + * had its OLD NBSP-gated form (generalized to claim every external paste, NBSP or not, on + * 2026-08-07 — see `$handlePasteForStandardView`'s own doc comment) — so the most plausible + * mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's + * default rich-paste node insertion, NOT a same-namespace `application/x-lexical-editor` payload on + * the clipboard. On THIS branch's current code, a single-line, NBSP-free, flavor-free external + * paste like TJ's literal repro was already safe (see the "typed characterization"/"paste ≡ typed" + * pins below, which pin that fact rather than a corruption). + * + * What this file's "root cause" describe block reproduces and fixes is the SHAPE that corruption + * takes whenever ANY handler declines an attribute-context paste to Lexical's default rich-paste + * node insertion: it has no notion that an attribute run's text must stay inside its ONE tagged + * TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph sibling's text + * into one plain node — destroying the attribute display, the closing marker, and the paragraph + * boundary in one move. Confirmed regression classes on this branch, each pinned below: a live + * native paste event that still carries a same-namespace `application/x-lexical-editor` flavor; a + * multi-line plain-text payload (the ordinary pipeline splits it via `insertParagraph()`); a + * marker-bearing payload (the ordinary pipeline's `\c`/`\id` strip eats bytes out of an attribute + * VALUE that were never a chapter token); and a selection that only PARTLY touches the attribute + * run combined with either of the first two. Fixed by always routing a paste whose selection + * TOUCHES attribute-display text through plain-text insertion, regardless of what other MIME + * flavors the clipboard also carries or how much of the selection sits outside the run — see + * `$handlePasteForStandardView`'s doc comment for the full design. + * + * Binding design principle: paste in attribute context ≡ typing the same characters at the same + * caret (or over the same selection). Every "paste ≡ typed" pin below proves paste and the + * character-by-character TYPED equivalent settle to the identical USJ, not merely to + * individually-plausible-looking results. + */ + +import { + $appendVerseAttributeRun, + requireDefined, + testEnvironment, + testEnvironmentWithCharSync, + testEnvironmentWithCharSyncAndHistory, + testEnvironmentWithSpacing, + viewOptions, + pasteEvent, + copyEvent, +} from "./markerEdit.test-helpers"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; +import { $createMarkerPrefix } from "./markerEditDeletion.utils"; +import { act } from "@testing-library/react"; +import { $getLexicalContent } from "@lexical/clipboard"; +import { + $createTextNode, + $getRoot, + $getSelection, + $getState, + $isRangeSelection, + $isTextNode, + $setState, + CUT_COMMAND, + LexicalEditor, + PASTE_COMMAND, + TextNode, + UNDO_COMMAND, +} from "lexical"; +import { + $createCharNode, + $createMarkerNode, + $createMilestoneNode, + $createParaNode, + $createVerseNode, + $isCharNode, + $isMarkerNode, + $isMilestoneNode, + $isParaNode, + $isVerseNode, + $milestoneAttributeRunPieces, + $verseAttributeRunPieces, + getVisibleOpenMarkerText, + MilestoneNode, + NBSP, + ParaNode, + textTypeState, + VerseNode, +} from "shared"; +import { Usj } from "@eten-tech-foundation/scripture-utilities"; + +// jsdom doesn't implement `getBoundingClientRect` on `Range`; moving the caret gives the editor +// root DOM focus, and Lexical's post-commit scroll-into-view reads a Range rect. Stub it (a zero +// rect nothing here asserts on), same as the sibling marker-edit tests. +if (typeof Range.prototype.getBoundingClientRect !== "function") { + Range.prototype.getBoundingClientRect = function (): DOMRect { + return { + x: 0, + y: 0, + width: 0, + height: 0, + top: 0, + right: 0, + bottom: 0, + left: 0, + toJSON() { + return this; + }, + }; + }; +} +// jsdom implements neither `ClipboardEvent` nor `DragEvent`; Lexical's default rich-paste path +// (reached in the "root cause" block below, where the fix must NOT let it run) checks +// `instanceof`/class-name against both. Same stub as clipboardCopyFidelity.test.tsx / +// noteEnterFp.test.tsx; only defined if not already present. +const globalStubs: { DragEvent?: unknown; ClipboardEvent?: unknown } = globalThis; +if (typeof globalStubs.DragEvent === "undefined") + globalStubs.DragEvent = class DragEvent extends Event {}; +if (typeof globalStubs.ClipboardEvent === "undefined") + globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; + +type EditorHandle = LexicalEditor; + +/** Types `text` one character at a time via `selection.insertText`, each in its own commit — the + * same update path a real keystroke takes (matches the idiom in markerEditTier2Trigger.utils.test.tsx). */ +async function typeCharByChar(editor: EditorHandle, text: string): Promise { + for (const character of text) { + await act(async () => + editor.update(() => { + const selection = $getSelection(); + if ($isRangeSelection(selection)) selection.insertText(character); + }), + ); + } +} + +/** Dispatches `PASTE_COMMAND` with `payload` as the clipboard's MIME map, in one commit — a single + * real paste, not a per-character sequence. Flushes Tier 2's post-paste double microtask. */ +async function pasteAndFlush( + editor: EditorHandle, + payload: { [key: string]: string }, +): Promise { + await act(async () => + editor.update(() => { + editor.dispatchCommand(PASTE_COMMAND, pasteEvent(payload).event); + }), + ); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** Moves the caret to `$select` and flushes the deferred departure-settle microtask. */ +async function departAndSettle(editor: EditorHandle, $select: () => void): Promise { + await act(async () => editor.update($select)); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** One `UNDO_COMMAND` dispatch, flushed the same way a paste is — matches + * `markerPasteFidelity.test.tsx`'s `undoAndSettle`. */ +async function undoAndSettle(editor: EditorHandle): Promise { + await act(async () => editor.dispatchCommand(UNDO_COMMAND, undefined)); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** The current document as USJ, via the same `toJSON` -> deserialize path every sibling suite + * reads settled state through. */ +function usjOf(editor: EditorHandle): Usj { + initializeDeserialize(undefined); + const usj = editor + .getEditorState() + .read(() => deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions)); + if (!usj) throw new Error("editor state did not serialize to USJ"); + return usj; +} + +// --------------------------------------------------------------------------------------------- +// Char span fixture: TJ's exact repro shape, `\nd asdf|who="hi"\nd*`, plus a second paragraph to +// depart to. Mounts BOTH CharNodePlugin (self-heal) and MarkerEditPlugin (pend/settle) — the real +// app's plugin stack — matching charAttributeDeletionSettle.test.tsx's idiom. +// --------------------------------------------------------------------------------------------- + +function $charFixture(): void { + const char = $createCharNode("nd"); + char.setUnknownAttributes({ who: "hi" }); + const run = $createTextNode('|who="hi"'); + $setState(run, textTypeState, "attribute"); + char.append( + $createMarkerNode("nd"), + $createTextNode(`${NBSP}asdf`), + run, + $createMarkerNode("nd", "closing"), + ); + $getRoot().append( + $createParaNode("p").append($createMarkerNode("p"), $createTextNode(NBSP), char), + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + $createTextNode("body"), + ), + ); +} + +const $firstChar = () => + requireDefined( + $getRoot().getChildren().filter($isParaNode)[0].getChildren().find($isCharNode), + "char missing", + ); +const $charAttributeRun = (char: ReturnType): TextNode => + requireDefined( + char + .getChildren() + .find( + (c): c is TextNode => + $isTextNode(c) && !$isMarkerNode(c) && $getState(c, textTypeState) === "attribute", + ), + "run missing", + ); +const $charCloser = (char: ReturnType) => + requireDefined( + char.getChildren().find((c) => $isMarkerNode(c) && c.getMarkerSyntax() === "closing"), + "closer missing", + ); +const $bodyTextNode = () => { + const body = $getRoot().getChildren().filter($isParaNode)[1].getLastChild(); + if (!$isTextNode(body)) throw new Error("body text node missing"); + return body; +}; + +describe("typed characterization (baseline): typing at the end of a char span's attribute run", () => { + it("does NOT reproduce the corruption class — the run, the closer, and the attributes all survive intact", async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + await typeCharByChar(editor, 'sid="things"'); + + // Pre-departure: no escaped text, closer intact, run still tagged "attribute". + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + expect($charAttributeRun(char).getTextContent()).toBe('|who="hi"sid="things"'); + expect(char.getUnknownAttributes()).toEqual({ who: "hi" }); // not yet re-tokenized + }); + + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + expect(char.getUnknownAttributes()).toEqual({ who: "hi", sid: "things" }); + }); + expect(usjOf(editor).content).toEqual([ + { + type: "para", + marker: "p", + content: [{ type: "char", marker: "nd", who: "hi", sid: "things", content: ["asdf"] }], + }, + { type: "para", marker: "p", content: [" body"] }, + ]); + }); +}); + +describe("paste ≡ typed (TJ's repro shape): plain-text-only paste at the end of the run", () => { + async function typedResult(): Promise { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + await typeCharByChar(editor, 'sid="things"'); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + return usjOf(editor); + } + + it("settles to the byte-for-byte SAME USJ as the character-by-character typed equivalent", async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": 'sid="things"' }); + + // Pre-settle: no escaped text, closer intact, attribute display present (the exact live-repro + // symptoms — hidden display, escaped text — must not appear even transiently). + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + expect($charAttributeRun(char).getTextContent()).toBe('|who="hi"sid="things"'); + }); + + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + expect(usjOf(editor)).toEqual(await typedResult()); + }); + + it("undo after the paste restores the exact pre-paste USJ in one step", async () => { + const { editor } = await testEnvironmentWithCharSyncAndHistory($charFixture); + const preUsj = usjOf(editor); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": 'sid="things"' }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); +}); + +describe("root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run", () => { + it("pasting plain text ALONGSIDE a same-namespace application/x-lexical-editor payload settles identically to a plain-only paste (regression for the live corruption)", async () => { + // Build the same-namespace rich payload the way a real same-editor Ctrl+C would: select some + // plain text elsewhere in the SAME editor and capture $getLexicalContent, exactly as + // copyToClipboard does. `$handlePasteForStandardView`'s same-namespace-flavor guard exists so + // a live native paste event that still carries the flavor (a genuine same-page copy, not the + // reconstructed-DataTransfer paste path S3's own doc comment shows can never carry it) keeps + // Lexical's exact-node-tree fast path for ORDINARY content — but that guard must never win over + // an attribute-context destination; this reproduces the corruption class the guard used to + // cause there (a confirmed regression class on this branch — not necessarily TJ's own + // pre-branch repro, whose most plausible mechanism was the OLD NBSP-gated handler; see this + // file's header comment). + const { editor } = await testEnvironmentWithCharSync(() => { + $charFixture(); + $getRoot().append( + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + $createTextNode('sid="things"'), + ), + ); + }); + let lexicalPayload = ""; + await act(async () => + editor.update(() => { + const source = $getRoot().getChildren().filter($isParaNode)[2].getLastChild(); + if (!$isTextNode(source)) throw new Error("copy source text missing"); + source.select(0, source.getTextContentSize()); + lexicalPayload = $getLexicalContent(editor) ?? ""; + }), + ); + expect(lexicalPayload).not.toBe(""); + + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + await pasteAndFlush(editor, { + "text/plain": 'sid="things"', + "application/x-lexical-editor": lexicalPayload, + }); + + // The live corruption, pinned as a MUST-NOT: the run and closer must still exist, and no + // sibling paragraph text may have been swallowed into the char span. + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + expect($charAttributeRun(char).getTextContent()).toBe('|who="hi"sid="things"'); + expect(char.getTextContent()).not.toContain("body"); + }); + + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + editor.getEditorState().read(() => { + const char = $firstChar(); + expect(char.getUnknownAttributes()).toEqual({ who: "hi", sid: "things" }); + }); + const usj = usjOf(editor); + const firstPara = usj.content[0]; + if (typeof firstPara === "string") + throw new Error("first paragraph corrupted into a bare string"); + expect(firstPara.content).toEqual([ + { type: "char", marker: "nd", who: "hi", sid: "things", content: ["asdf"] }, + ]); + }); +}); + +describe("leading-space payload: the well-formed case ends fully correct on disk", () => { + it('paste " sid=\\"things\\"" (leading space already separates the pair) settles to both attributes present', async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": ' sid="things"' }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + editor.getEditorState().read(() => { + expect($firstChar().getUnknownAttributes()).toEqual({ who: "hi", sid: "things" }); + }); + }); +}); + +describe("replace-selection paste inside the attribute value", () => { + it('selecting "hi" inside |who="hi" and pasting "bye" settles to who="bye"', async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + const text = run.getTextContent(); // '|who="hi"' + const valueStart = text.indexOf('"') + 1; + const valueEnd = text.lastIndexOf('"'); + run.select(valueStart, valueEnd); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "bye" }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + editor.getEditorState().read(() => { + expect($firstChar().getUnknownAttributes()).toEqual({ who: "bye" }); + }); + }); + + it("undo after the replace-selection paste restores the exact pre-paste USJ in one step", async () => { + const { editor } = await testEnvironmentWithCharSyncAndHistory($charFixture); + const preUsj = usjOf(editor); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + const text = run.getTextContent(); + const valueStart = text.indexOf('"') + 1; + const valueEnd = text.lastIndexOf('"'); + run.select(valueStart, valueEnd); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "bye" }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); +}); + +describe("multi-line payload collapses to a single space, per newline (attribute values are single-line)", () => { + it('paste "a\\nb" into the attribute run: the raw display text becomes "...a b...", not two paragraphs', async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "a\nb" }); + + editor.getEditorState().read(() => { + // Still exactly one paragraph, one char span — a newline in an attribute payload must never + // split the document the way it would in body content. + expect($getRoot().getChildren().filter($isParaNode)).toHaveLength(2); + expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="hi"a b'); + }); + }); + + it('paste "a\\n\\nb" (two consecutive newlines): each `\\n` becomes its OWN space — "...a b..." with TWO spaces, not one collapsed space', async () => { + // Pins that the replacement is per-newline (`text.replace(/\n/g, " ")`), not a run-collapsing + // one — there is no "multiple blank lines" concept to collapse INTO for a single-line + // attribute value; two newlines are two individually-typed line breaks, so they become two + // individually-typed spaces. + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "a\n\nb" }); + + editor.getEditorState().read(() => { + expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="hi"a b'); + }); + }); + + it("undo after the multi-line paste restores the exact pre-paste USJ in one step", async () => { + const { editor } = await testEnvironmentWithCharSyncAndHistory($charFixture); + const preUsj = usjOf(editor); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "a\nb" }); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); +}); + +describe("marker-bearing payload: literal value text, no strip, no chapter node", () => { + it('paste "\\c 5" into the attribute run: the bytes survive literally, no chapter node is created', async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + + editor.getEditorState().read(() => { + // Literal bytes intact in the run — the chapter/book-id strip + // ($stripPastedChapterAndBookId) must never run against attribute-context text. + expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="hi"\\c 5'); + }); + const usj = usjOf(editor); + expect( + usj.content.filter((item) => typeof item !== "string" && item.type === "chapter"), + ).toEqual([]); + expect(usj.content.filter((item): item is string => typeof item === "string")).toEqual([]); + }); +}); + +describe("CUT of a selection inside the attribute value", () => { + it('cutting "hi" out of |who="hi" removes the value text, keeps the structure, and the clipboard holds "hi"', async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + const { event, getData } = copyEvent(); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + const text = run.getTextContent(); + const valueStart = text.indexOf('"') + 1; + const valueEnd = text.lastIndexOf('"'); + run.select(valueStart, valueEnd); + editor.dispatchCommand(CUT_COMMAND, event); + }), + ); + + expect(getData("text/plain")).toBe("hi"); + editor.getEditorState().read(() => { + expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who=""'); + }); + + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + editor.getEditorState().read(() => { + // An EMPTY attribute value refuses the whole list and stays literal, matching Paratext 9 — + // `|who=""` is not a reading Paratext will ever agree with, so the engine does not invent an + // attribute the rest of the pipeline would disagree about. What the cut must guarantee is + // that the bytes the user did not remove survive: the span keeps its own marker and closer, + // and the emptied list rides on as ordinary content rather than being silently dropped. + expect($firstChar().getUnknownAttributes()).toBeUndefined(); + expect($firstChar().getTextContent().replaceAll(NBSP, " ")).toBe('\\nd asdf|who=""\\nd*'); + }); + }); +}); + +describe("mixed selection: spans BOTH attribute and non-attribute content", () => { + // Design choice (brief-permitted either way): a selection that only PARTLY sits inside + // attribute-display text now TAKES the attribute-context path (widened from an earlier + // both-ends-must-qualify check) rather than declining it — under paste ≡ typing, a user typing a + // character over this exact selection gets `selection.insertText`'s own removeText-then-insert + // behavior regardless of which node the selection's OTHER end sits on, so paste must take the + // identical path instead of falling through to a branch (the ordinary pipeline's + // `insertParagraph()` for a multi-line payload, or Lexical's rich-node paste for a + // same-namespace-flavored one) that CAN corrupt the attribute-run end of the range — reproduced + // in the two pins below this one, each of which used to reach exactly one of those two branches + // before the OR-widening. + function $selectRunThroughCloserStart(): void { + const char = $firstChar(); + const run = $charAttributeRun(char); + const closer = $charCloser(char); + run.select(0, 0); // anchor: start of the run (attribute context)... + const selection = $getSelection(); + if ($isRangeSelection(selection)) selection.focus.set(closer.getKey(), 0, "text"); // ...focus: start of the closer (NOT attribute-tagged) — spans the run's full text + } + + it("plain single-line payload: claims the attribute path, replacing the selected range with the pasted text", async () => { + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => editor.update($selectRunThroughCloserStart)); + + await pasteAndFlush(editor, { "text/plain": "X" }); + + // No crash, and the document stays structurally sane: still 2 paragraphs, closer still present + // (the mixed range covered only the run's own text, never reaching into the closer glyph). + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($getRoot().getChildren().filter($isParaNode)).toHaveLength(2); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + }); + }); + + it("mixed selection + same-namespace application/x-lexical-editor flavor: does not reach Lexical's rich-paste node insertion (regression for the corruption class this closes)", async () => { + // Before the OR-widening, this exact shape (one end in attribute context, one end not) failed + // the (then AND-based) attribute-context check, so the same-namespace-flavor decline above ran + // unconditionally and handed the paste to Lexical's default rich-paste node insertion — the + // SAME corruption class as the "root cause" describe block above, reached via a mixed selection + // instead of a fully-inside one. + const { editor } = await testEnvironmentWithCharSync(() => { + $charFixture(); + $getRoot().append( + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + $createTextNode('sid="things"'), + ), + ); + }); + let lexicalPayload = ""; + await act(async () => + editor.update(() => { + const source = $getRoot().getChildren().filter($isParaNode)[2].getLastChild(); + if (!$isTextNode(source)) throw new Error("copy source text missing"); + source.select(0, source.getTextContentSize()); + lexicalPayload = $getLexicalContent(editor) ?? ""; + }), + ); + expect(lexicalPayload).not.toBe(""); + + await act(async () => editor.update($selectRunThroughCloserStart)); + await pasteAndFlush(editor, { + "text/plain": 'sid="things"', + "application/x-lexical-editor": lexicalPayload, + }); + + // MUST-NOT: the closer must still exist, and no sibling paragraph text may have been swallowed + // into the char span — the exact corruption shape a decline into Lexical's rich-paste path + // produces. + editor.getEditorState().read(() => { + const char = $firstChar(); + expect($getRoot().getChildren().filter($isParaNode)).toHaveLength(3); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + expect(char.getTextContent()).not.toContain("body"); + }); + }); + + it('mixed selection + multi-line plain payload ("a\\nb"): collapses to a single space instead of splitting the paragraph', async () => { + // Before the OR-widening, this exact shape fell through to the ordinary external-paste + // pipeline, whose line replay dispatches INSERT_PARAGRAPH_COMMAND per newline — splitting the + // paragraph with the selection still anchored inside the char span's own attribute run, + // corrupting its structure. + const { editor } = await testEnvironmentWithCharSync($charFixture); + await act(async () => editor.update($selectRunThroughCloserStart)); + + await pasteAndFlush(editor, { "text/plain": "a\nb" }); + + editor.getEditorState().read(() => { + // Still exactly 2 paragraphs — no paragraph split — and the closer survived. + const char = $firstChar(); + expect($getRoot().getChildren().filter($isParaNode)).toHaveLength(2); + expect($charCloser(char).getTextContent()).toBe("\\nd*"); + }); + }); +}); + +// --------------------------------------------------------------------------------------------- +// Milestone attribute run: one paste case, matching milestoneAttributeSettle.test.tsx's idiom. +// --------------------------------------------------------------------------------------------- + +function $milestoneFixture(): void { + const [glyph, separator] = $createMarkerPrefix("p"); + const [glyph2, separator2] = $createMarkerPrefix("p"); + $getRoot().append( + $createParaNode("p").append( + glyph, + separator, + $createTextNode("before "), + $createMilestoneNode("qt-s", "q1"), + $createTextNode(" after"), + ), + $createParaNode("p").append(glyph2, separator2, $createTextNode("body")), + ); +} + +function $firstPara(): ParaNode { + return $getRoot().getChildren().filter($isParaNode)[0]; +} +function $milestoneInFirstPara(): MilestoneNode { + return requireDefined($firstPara().getChildren().find($isMilestoneNode), "milestone missing"); +} +function $milestoneAttributeRun(): TextNode { + const { attribute } = $milestoneAttributeRunPieces($milestoneInFirstPara()); + return requireDefined(attribute, "milestone attribute run missing"); +} +function $milestoneBodyText(): TextNode { + const body = $getRoot().getChildren().filter($isParaNode)[1]?.getLastChild(); + if (!$isTextNode(body)) throw new Error("body text node missing"); + return body; +} + +describe("milestone attribute run paste", () => { + it('paste \'who="ed"\' at the end of a \\qt-s sid="q1" run settles with sid unchanged and who added', async () => { + const { editor } = await testEnvironment($milestoneFixture); + await act(async () => + editor.update(() => { + const run = $milestoneAttributeRun(); + run.select(run.getTextContentSize(), run.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": 'who="ed"' }); + await departAndSettle(editor, () => $milestoneBodyText().select(0, 0)); + + editor.getEditorState().read(() => { + const milestone = $milestoneInFirstPara(); + expect(milestone.getSid()).toBe("q1"); + expect(milestone.getUnknownAttributes()).toEqual({ who: "ed" }); + }); + }); +}); + +// --------------------------------------------------------------------------------------------- +// Verse \va run: one paste case, matching verseAttributeSettle.test.tsx's idiom. +// --------------------------------------------------------------------------------------------- + +function $verseFixture(): void { + const verse = $createVerseNode( + "1", + getVisibleOpenMarkerText("v", "1"), + undefined, + "2", + undefined, + ); + $getRoot().append( + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + verse, + $createTextNode("In the beginning"), + ), + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + $createTextNode("body"), + ), + ); + $appendVerseAttributeRun(verse, "va", "2"); +} + +function $firstVerse(): VerseNode { + return requireDefined( + $getRoot().getChildren().filter($isParaNode)[0].getChildren().find($isVerseNode), + "verse missing", + ); +} +function $verseBodyText(): TextNode { + const body = $getRoot().getChildren().filter($isParaNode)[1]?.getLastChild(); + if (!$isTextNode(body)) throw new Error("body text node missing"); + return body; +} + +describe("verse \\va run paste", () => { + it('paste "3" at the end of a \\va 2 run settles altnumber to "23"', async () => { + const { editor } = await testEnvironmentWithSpacing($verseFixture); + await act(async () => + editor.update(() => { + const { value } = $verseAttributeRunPieces($firstVerse(), "va"); + const valueNode = requireDefined(value, "\\va value missing"); + valueNode.select(valueNode.getTextContentSize(), valueNode.getTextContentSize()); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "3" }); + await departAndSettle(editor, () => $verseBodyText().select(0, 0)); + + editor.getEditorState().read(() => { + expect($firstVerse().getAltnumber()).toBe("23"); + }); + }); +}); diff --git a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx new file mode 100644 index 000000000..075a87dbd --- /dev/null +++ b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx @@ -0,0 +1,420 @@ +/** + * Copy-fidelity pins for Standard-view `text/plain`: note callers (previously silently dropped), + * source-faithful NBSP handling around a collapsed note's internal markers (previously a blanket + * NBSP→space mapping produced phantom spaces the source USFM never had), multi-paragraph joining, + * and the copy→paste USJ round trip. Kept separate from `whitespaceDisplay.plugin.utils.test.tsx` + * (which already covers the NBSP display-run invariant and the plain payload-builder contract) so + * this file can stay focused on `$selectionToUsfmText`'s USFM-shape behavior. + */ +import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { + $appendVerseAttributeRun, + copyEvent, + findOnlyNote, + plainTextPasteEvent, + serializedState, + testEnvironment, + viewOptions, +} from "./markerEdit.test-helpers"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; +import { act } from "@testing-library/react"; +import { $dfs } from "@lexical/utils"; +import { + $createPoint, + $createRangeSelection, + $createTextNode, + $getRoot, + $setSelection, + $setState, + COPY_COMMAND, + CUT_COMMAND, + LexicalEditor, + PASTE_COMMAND, + TextNode, +} from "lexical"; +import { + $createMarkerNode, + $createParaNode, + $createVerseNode, + $isNoteNode, + getVisibleOpenMarkerText, + NBSP, + NoteNode, + textTypeState, +} from "shared"; +// Reaching inside only for tests. +// eslint-disable-next-line @nx/enforce-module-boundaries +import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; +import { Usj } from "@eten-tech-foundation/scripture-utilities"; + +// jsdom implements neither `ClipboardEvent` nor `DragEvent`, but Lexical's default (non-Standard- +// view-specific) paste path — reached once our copied text carries no NBSP for +// `$handlePasteForStandardView` to claim — checks `instanceof`/class-name against both to decide +// whether a paste carries files. Same stub as `markerEditDeletion.utils.test.tsx`'s round-trip +// paste test; only defined if not already present (a shared jsdom global, not per-file). +const globalStubs: { DragEvent?: unknown; ClipboardEvent?: unknown } = globalThis; +if (typeof globalStubs.DragEvent === "undefined") + globalStubs.DragEvent = class DragEvent extends Event {}; +if (typeof globalStubs.ClipboardEvent === "undefined") + globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; + +/** A source-faithful (real space after the marker glyph) `marker-trailing-space` separator, the + * shape `usj-editor.adaptor.ts`'s `createPara` builds after a paragraph's own opening marker. */ +function $trailingSpaceNode(): TextNode { + const spaceNode = $createTextNode(NBSP); + $setState(spaceNode, textTypeState, "marker-trailing-space"); + return spaceNode; +} + +/** Selects an entire (inline) node — its first descendant's start through its last descendant's + * end — the shape needed to select a whole `NoteNode` regardless of collapsed/expanded layout. */ +function $selectWholeNode(node: NoteNode): void { + const first = node.getFirstDescendant(); + const last = node.getLastDescendant(); + if (!first || !last) throw new Error("node has no descendants to select"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(first.getKey(), 0, "text"); + selection.focus = $createPoint(last.getKey(), last.getTextContentSize(), "text"); + $setSelection(selection); +} + +/** Selects the whole document (every top-level block) — used for whole-paragraph-selection pins + * against a single-paragraph fixture. */ +function $selectWholeDocument(): void { + const root = $getRoot(); + root.select(0, root.getChildrenSize()); +} + +/** Number of `NoteNode`s currently in the tree — used to pin cut's removal without asserting on + * node-tree shape. */ +function $countNotes(): number { + return $dfs($getRoot()).filter(({ node }) => $isNoteNode(node)).length; +} + +/** A `\p` paragraph: `\v 1 In the beginning` + a collapsed note with `caller` + ` God created.`, + * parameterized on the note's USJ caller. */ +function noteUsj(caller: string): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: [ + { type: "verse", marker: "v", number: "1" }, + "In the beginning", + { + type: "note", + marker: "f", + caller, + content: [ + { type: "char", marker: "fr", content: ["1.1 "] }, + { type: "char", marker: "ft", content: ["A note."] }, + ], + }, + " God created.", + ], + }, + ], + } as unknown as Usj; +} + +/** + * The 2026-08-07 live-repro footnote: `\f - \fr 1:1 \ft Caller test.\f*` — content chars carry + * `closed: "false"` (ParatextData's real shape for footnote content — `\fr`/`\ft` never get their + * own closer), matching the exact bytes the repro pinned. + */ +function footnoteReproUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: [ + { + type: "note", + marker: "f", + caller: "-", + content: [ + { type: "char", marker: "fr", content: ["1:1 "], closed: "false" }, + { type: "char", marker: "ft", content: ["Caller test."], closed: "false" }, + ], + }, + ], + }, + ], + } as unknown as Usj; +} + +/** + * The 2026-08-07 live-repro cross-reference: `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*` — content + * chars carry no `closed` flag (explicitly closed, real cross-reference shape), matching the exact + * bytes the repro pinned. + */ +function xrefReproUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: [ + { + type: "note", + marker: "x", + caller: "-", + content: [ + { type: "char", marker: "xo", content: ["1:3: "] }, + { type: "char", marker: "xt", content: ["2Cor 4:6"] }, + ], + }, + ], + }, + ], + } as unknown as Usj; +} + +/** Mounts a headless Standard-view editor (`MarkerEditPlugin`) with `usj` loaded. */ +async function renderUsjEditor(usj: Usj): Promise<{ editor: LexicalEditor }> { + return baseTestEnvironment(serializedState(usj), ); +} + +describe("note caller fidelity", () => { + it.each([ + ["+", "\\f + "], + ["a", "\\f a "], + ["-", "\\f - "], + ])( + "copies a collapsed note's %s caller into text/plain, correctly placed", + async (caller, expectedOpen) => { + const { editor } = await renderUsjEditor(noteUsj(caller)); + await act(async () => editor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + const textPlain = getData("text/plain"); + expect(textPlain).toContain(expectedOpen); + expect(textPlain).toContain("\\f*"); + }, + ); + + it("places the caller correctly relative to the surrounding verse text and the note's own closer", async () => { + const { editor } = await renderUsjEditor(noteUsj("+")); + await act(async () => editor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toMatch(/^\\p \\v 1 In the beginning\\f \+ /); + }); +}); + +describe("phantom-space live-repro pins (2026-08-07) — collapsed note, byte-identical to source", () => { + it("copies an unclosed footnote's internal markers with no phantom spaces (caller included)", async () => { + const { editor } = await renderUsjEditor(footnoteReproUsj()); + let note!: NoteNode; + editor.getEditorState().read(() => { + note = findOnlyNote($getRoot()); + }); + await act(async () => editor.update(() => $selectWholeNode(note))); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + // "What you copy" > "what you see": a COLLAPSED note's full hidden bytes are the payload. + expect(getData("text/plain")).toBe("\\f - \\fr 1:1 \\ft Caller test.\\f*"); + }); + + it("copies a closed cross-reference's internal markers with no phantom spaces after each closer", async () => { + const { editor } = await renderUsjEditor(xrefReproUsj()); + let note!: NoteNode; + editor.getEditorState().read(() => { + note = findOnlyNote($getRoot()); + }); + await act(async () => editor.update(() => $selectWholeNode(note))); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe("\\x - \\xo 1:3: \\xo*\\xt 2Cor 4:6\\xt*\\x*"); + }); +}); + +describe("multi-paragraph selections", () => { + it("joins a full multi-paragraph selection with a single \\n, each paragraph keeping its own \\marker", async () => { + let secondText: TextNode; + const { editor } = await testEnvironment(() => { + const firstText = $createTextNode("one"); + secondText = $createTextNode("two"); + $getRoot().append( + $createParaNode("p").append($createMarkerNode("p"), $trailingSpaceNode(), firstText), + $createParaNode("q1").append($createMarkerNode("q1"), $trailingSpaceNode(), secondText), + ); + }); + await act(async () => + editor.update(() => { + // A "full paragraph" selection anchors at the PARAGRAPH's own element start (before its + // \marker glyph), not at the first content TextNode — anchoring directly on the text + // node would exclude the marker/trailing-space siblings that precede it, exactly like + // the "starts mid-paragraph" test below (deliberately) does. + const firstPara = $getRoot().getFirstChildOrThrow(); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(firstPara.getKey(), 0, "element"); + selection.focus = $createPoint( + secondText.getKey(), + secondText.getTextContentSize(), + "text", + ); + $setSelection(selection); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe("\\p one\n\\q1 two"); + }); + + it("omits the first paragraph's own \\marker glyph when the selection starts mid-paragraph", async () => { + let firstText: TextNode; + let secondText: TextNode; + const { editor } = await testEnvironment(() => { + firstText = $createTextNode("before tail"); + secondText = $createTextNode("two"); + $getRoot().append( + $createParaNode("p").append($createMarkerNode("p"), $trailingSpaceNode(), firstText), + $createParaNode("q1").append($createMarkerNode("q1"), $trailingSpaceNode(), secondText), + ); + }); + await act(async () => + editor.update(() => { + const selection = $createRangeSelection(); + selection.anchor = $createPoint(firstText.getKey(), "before ".length, "text"); + selection.focus = $createPoint( + secondText.getKey(), + secondText.getTextContentSize(), + "text", + ); + $setSelection(selection); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe("tail\n\\q1 two"); + }); +}); + +describe("AttributeRunNode traversal", () => { + // The wrapper contributes no bytes of its own (`usj-editor.adaptor.ts`'s `addVerseAttributeRun` + // wraps a verse's `\va`/`\vp` triplet in one `AttributeRunNode`, the same "run lives inside a + // container" shape a milestone's attribute run gets) — a selection spanning it must still carry + // the wrapped opening marker, NBSP-prefixed value, and closing marker, with no extra separator + // contributed by the wrapper itself, and the plain text on either side must not be disturbed. + it("copies a selection spanning a wrapped verse \\va attribute run transparently, byte-exact", async () => { + const { editor } = await testEnvironment(() => { + // The verse's own `altnumber` must match the manually-built display run below (mirroring + // `attributeClass.utils.test.tsx`'s working pattern for this same helper): without it, the + // marker-edit engine's pend/settle machinery (active even in plain `testEnvironment`, + // registered on every `AttributeRunNode` mutation) treats the just-built wrapper as an + // unbacked run and clears its children, since nothing here re-derives them from `altnumber` + // the way `TextSpacingPlugin`'s self-healing sync would. + const verse = $createVerseNode("1", getVisibleOpenMarkerText("v", "1"), undefined, "2"); + $getRoot().append( + $createParaNode("p").append( + $createMarkerNode("p"), + $createTextNode(NBSP), + verse, + $createTextNode("In the beginning"), + ), + ); + $appendVerseAttributeRun(verse, "va", "2"); + }); + await act(async () => editor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + // "\p " + "\v 1 " (the verse's own baked-in glyph+number+space) + the wrapped "\va 2\va*" run + // (opening glyph, NBSP-prefixed value inverted to a plain space, closing glyph — no bytes from + // the AttributeRunNode wrapper itself) + the surrounding "In the beginning" text, undisturbed. + expect(getData("text/plain")).toBe("\\p \\v 1 \\va 2\\va*In the beginning"); + }); +}); + +describe("cut = copy + removeText", () => { + // Two independent editors (rather than one editor dispatching COPY then CUT in sequence): a + // second command dispatch on the same editor lets Lexical's own DOM-selection reconciliation + // (unrelated to this plugin) collapse the just-set programmatic selection under jsdom before the + // handler runs, which would make this a test of that reconciliation quirk instead of cut/copy + // parity. Each editor here does exactly one selection-set + one dispatch, matching every other + // test in this file. + it("cuts a collapsed note: clipboard matches copy's bytes, and the note is removed from the tree", async () => { + const usj = noteUsj("+"); + + const { editor: copyEditor } = await renderUsjEditor(usj); + let copyNote!: NoteNode; + copyEditor.getEditorState().read(() => { + copyNote = findOnlyNote($getRoot()); + }); + await act(async () => copyEditor.update(() => $selectWholeNode(copyNote))); + const copyStub = copyEvent(); + await act(async () => copyEditor.dispatchCommand(COPY_COMMAND, copyStub.event)); + const copiedText = copyStub.getData("text/plain"); + expect(copiedText).toContain("\\f + "); + + const { editor: cutEditor } = await renderUsjEditor(usj); + let cutNote!: NoteNode; + cutEditor.getEditorState().read(() => { + cutNote = findOnlyNote($getRoot()); + }); + await act(async () => cutEditor.update(() => $selectWholeNode(cutNote))); + const cutStub = copyEvent(); + await act(async () => cutEditor.dispatchCommand(CUT_COMMAND, cutStub.event)); + expect(cutStub.getData("text/plain")).toBe(copiedText); + + let notesRemaining = -1; + cutEditor.getEditorState().read(() => { + notesRemaining = $countNotes(); + }); + expect(notesRemaining).toBe(0); + }); +}); + +describe("copy → paste USJ round trip", () => { + // A whole-paragraph copy starts with its own "\p " literal (the paragraph's own marker rides + // along with a whole-block selection). Pasted at an existing "\p" host's content start, the + // fragment Tier 2 rebuilds from would otherwise carry BOTH the host's own glyph and the pasted + // literal's — two paragraph-marker occurrences with nothing between them, tokenizing into a + // stray empty leading paragraph (the host's, now with nothing to show for it) ahead of the real + // one. `$buildParaFragment`'s own-marker-wins rule (tier2Rebuild.utils.ts) drops the host's + // redundant glyph from the fragment in exactly this shape, closing that gap. + it("re-tokenizes a whole-paragraph copy back to the source USJ when pasted into a fresh editor", async () => { + initializeDeserialize(undefined); + const usj = noteUsj("+"); + const { editor: sourceEditor } = await renderUsjEditor(usj); + await act(async () => sourceEditor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, event)); + const copiedText = getData("text/plain"); + + let trailing: TextNode; + const { editor: targetEditor } = await testEnvironment(() => { + trailing = $trailingSpaceNode(); + $getRoot().append($createParaNode("p").append($createMarkerNode("p"), trailing)); + }); + await act(async () => + targetEditor.update(() => { + trailing.select(trailing.getTextContentSize(), trailing.getTextContentSize()); + targetEditor.dispatchCommand(PASTE_COMMAND, plainTextPasteEvent(copiedText)); + }), + ); + // Settle: flush any reconciliation the paste-triggered Tier 2 re-tokenization schedules + // beyond the synchronous update above. + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + const pastedUsj = deserializeSerializedEditorState( + targetEditor.getEditorState().toJSON(), + viewOptions, + ); + expect(pastedUsj).toEqual(usj); + }); +}); diff --git a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx new file mode 100644 index 000000000..283cb4fe1 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx @@ -0,0 +1,277 @@ +/** + * Corpus-style copy→paste round trip: for every fixture in the shared USJ round-trip corpus + * (`../adaptors/corpus/corpus-data.ts`, the same fixtures `corpus-round-trip.test.ts` uses for the + * plain load→save round trip), load it into a Standard-view editor, select the chapter's own + * content, dispatch `COPY_COMMAND`, paste the resulting `text/plain` into a FRESH editor holding + * the same book/chapter header, settle, export USJ, and expect it to deep-equal the source. This + * is the clipboard-specific sibling of that load→save sweep — it exercises `$selectionToUsfmText` + * (copy) and `$handlePasteForStandardView` + Tier 2's re-tokenizer (paste) together, over the same + * corpus, instead of the adaptors alone. + * + * The book/chapter header itself is never part of the copied SELECTION (Standard view always + * copies out of an already-open chapter, never the header) and paste normalization deliberately + * strips any `\c`/`\id` a paste DOES carry (see `whitespaceDisplay.plugin.utils.ts`), so the + * target editor is seeded with the SAME header up front and the paste only has to reproduce the + * chapter's CONTENT — matching how a real user would select and copy inside an already-open + * chapter. + * + * `"periph"` is a named `it.skip` rather than a swept case: it is book-level front matter with no + * chapter at all, so it does not fit "a single-chapter editor state" — there is no chapter-content + * selection for it to exercise. Every other fixture is either swept clean or recorded in + * `KNOWN_LOSSY` below with the exact byte-level divergence — none are silently dropped. All 21 + * fixtures run in well under a second; no sampling is needed. + * + * The OTHER USJ corpus in this repo, `libs/test-data/src/data/2sa.usj.ts` (141 paragraphs; IS + * single-chapter — one top-level chapter object, despite the size), is deliberately NOT included + * here. It was tried against this exact harness and does not round-trip clean: it hits the + * `"sidebar (esb)"` gap below three times (2SA's own sidebar coverage), PLUS several additional, + * unrelated fidelity gaps this sweep's one-construct-per-fixture design has no clean way to + * itemize as a single byte diff (an empty `\b` blank-line paragraph folds into literal text inside + * a `p` paragraph; a verse's derived `sid` attribute is dropped; a `\ref` cross-reference target + * degrades differently than the corpus's own dedicated `ref` fixture above). `tier2Rebuild.corpus + * .test.tsx` already exercises this exact fixture, but for a narrower, DIFFERENT property (an + * unedited `$rebuildParas` call refusing as a fixed point) — that coverage does not include the + * paste path (NBSP normalization, `\c`/`\id` strip, own-marker-wins dedup) this sweep does, so + * excluding 2sa here is a real coverage gap, not a redundant re-test. + */ +import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { + copyEvent, + plainTextPasteEvent, + serializedState, + viewOptions, +} from "./markerEdit.test-helpers"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; +import { corpusFixtures } from "../adaptors/corpus/corpus-data"; +import { act } from "@testing-library/react"; +// Reaching inside only for tests (same pattern as markerEdit.test-helpers). +// eslint-disable-next-line @nx/enforce-module-boundaries +import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; +import { MarkerObject, Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; +import { $getRoot, COPY_COMMAND, PASTE_COMMAND, RootNode } from "lexical"; +import { $isChapterNode, $isImmutableChapterNode } from "shared"; + +/** The index of the top-level child right after the document's (LAST) chapter node — every corpus + * fixture but `"periph"` has exactly one, but a document could in principle carry more than one + * top-level `\c` boundary marker, so this anchors on the last rather than the first. Throws for a + * document with no chapter at all (`"periph"`, deliberately excluded from the sweep before this + * ever runs). */ +function $contentStartIndex(root: RootNode): number { + const children = root.getChildren(); + let lastChapterIndex = -1; + children.forEach((child, index) => { + if ($isChapterNode(child) || $isImmutableChapterNode(child)) lastChapterIndex = index; + }); + if (lastChapterIndex === -1) throw new Error("no chapter node found at the top level"); + return lastChapterIndex + 1; +} + +/** Selects every top-level node from right after the chapter marker through the end of the + * document — the chapter's own CONTENT, matching what Standard view actually copies out of an + * open chapter (never the book/chapter header itself). Element-offset (`root.select`), not + * node-anchored: this is used ONLY on the source/copy side (the paste side below always targets a + * fresh EMPTY host via `selectEnd()`), so it never hits the "replacing a large non-collapsed + * selection on paste" doubling bug a node-anchored `"text"`-typed focus point was once written to + * dodge here — and the element-offset form is strictly more general: it does not require the + * chapter's last node be a `TextNode` (a `"text"`-typed focus point does, and crashed with + * `TypeError: focusNode.selectionTransform is not a function` against a real corpus document, + * `libs/test-data/src/data/2sa.usj.ts`, whose last node isn't one). */ +function $selectChapterContent(root: RootNode): void { + const start = $contentStartIndex(root); + root.select(start, root.getChildrenSize()); +} + +/** The fixture's book+chapter header, byte-identical, plus a single EMPTY `\p` paragraph as the + * paste insertion host — the same "empty host + Tier 2's own-marker-wins dedup absorbs the whole + * pasted fragment" shape `clipboardCopyFidelity.test.tsx`'s "copy → paste USJ round trip" test + * uses, generalized to any book+chapter header so the target starts from the SAME chapter context + * as the source without paste ever having to reconstruct `\c`/`\id` itself (it can't — paste + * normalization strips them). */ +function chapterHeaderSkeletonUsj(usj: Usj): Usj { + const content = usj.content; + let chapterIndex = -1; + content.forEach((item, index) => { + if (typeof item !== "string" && item.type === "chapter") chapterIndex = index; + }); + if (chapterIndex === -1) throw new Error("fixture has no chapter to build a skeleton from"); + const emptyHost: MarkerObject = { type: "para", marker: "p", content: [] } as MarkerObject; + return { ...usj, content: [...content.slice(0, chapterIndex + 1), emptyHost] }; +} + +/** + * Corpus fixtures the copy→paste round trip cannot currently reproduce byte-for-byte. Kept IN the + * sweep as `it.skip` (never silently omitted) so a future engine fix un-skips them instead of a + * blind gap going unnoticed; each reason is the short form of the diff, with the full byte-level + * mechanism recorded in this file's neighboring comments and in the semantics doc's accepted- + * asymmetries list. + * + * - **"cross-reference ref target"** — USJ's `` cross-reference-target wrapper carries no + * USFM byte representation anywhere (`unknownUsfm.utils.ts`'s own doc comment: "USJ invented + * this container, USFM never carried it... only its child text renders"). Source content + * `["See ", {type:"ref", loc:"GEN 1:1", content:["Genesis 1:1"]}, " for details."]` copies as + * plain `"See Genesis 1:1 for details."` with no marker bytes anywhere marking the wrapper's + * extent, so paste re-tokenizes it as ordinary prose: content + * `["See Genesis 1:1 for details."]`, the `ref` wrapper gone. Inherent to the construct — a raw + * USFM export of this same fixture has the identical gap, independent of clipboard mechanics. + * + * - **"sidebar (esb)"** — NOT a tokenizer gap: `usfmFragmentToUsj.ts` already implements the + * `\esb`/`\esbe` pairing (its `SIDEBAR_MARKER`/`SIDEBAR_END_MARKER` assembly case tracks an open + * sidebar across tokens and closes it on `\esbe`). The real mechanism is structural, upstream of + * tokenizing: a sidebar's nested `\p` child is a real `ParaNode`, and `$selectionToUsfmText` (the + * copy walker) inserts a `\n` before any non-inline `ElementNode` boundary it crosses — a + * `ParaNode` is non-inline, so a `\n` lands between `\esb \cat History\cat*` and the nested + * paragraph's own content, even though both came from ONE sidebar. On paste, the line replay + * dispatches an `INSERT_PARAGRAPH_COMMAND` for every `\n`, so that single `\n` turns into TWO + * sibling `ParaNode`s where the source had one sidebar wrapping one paragraph. Tier 2 then + * re-tokenizes strictly per paragraph — `$requestTier2ForNode` (tier2Rebuild.utils.ts), the only + * production call site, always invokes `$rebuildParas([current], context)` with a single-element + * array — so the tokenizer's "current open sidebar" state can never span the two separate + * `$rebuildParas` calls the two now-sibling paragraphs each trigger; each is tokenized alone, and + * the pairing that DOES exist in `usfmFragmentToUsj.ts` never gets the chance to run across both + * lines at once. Result: an UNCLOSED sidebar (`closed:"false"`, no content), the inner paragraph + * hoisted OUT to become a top-level sibling, and a stray EMPTY paragraph with marker `"esbe"`. + * Tables hit the identical mode — their rows and cells are real block-level `ElementNode`s + * (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the walker's + * non-inline-boundary rule fires for every one of them; see the "table with header and cells" + * entry below. A real fix means grouping a paste's newly-inserted SIBLING paragraphs that originated + * from one selection back into a single rebuild fragment before Tier 2 tokenizes — a Tier-2 + * architecture change (rebuild granularity, not a per-paragraph tweak), genuinely out of this + * work item's scope. The byte-level corruption stays pinned here rather than fixed. + * + * - **"closed=false body char span (implicit close, no closer)"** — a `closed="false"` char span + * has, by definition, no closing marker byte anywhere in its own USFM. When such a span is not + * the last thing in its paragraph (`Tell the Lord plainly.`), the + * copied text (`\nd Lord plainly.`) carries no byte marking where the span's content ends and + * the trailing prose resumes, so paste has nothing to stop at "Lord" on — it swallows the rest + * of the paragraph into the span (`{marker:"nd", content:["Lord plainly."]}`, the top-level + * `" plainly."` string gone). Inherent to the encoding, not a paste-path regression — the + * sibling `"unclosed note (closed=false)"` fixture, whose unclosed span IS the last thing in its + * paragraph, has no such trailing content to lose and round-trips clean. + * + * - **"table with header and cells"** — the SAME rebuild-scoping gap as the sidebar entry above, + * reaching tables now that a table's rows and cells are real block-level `ElementNode`s + * (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`) rather than the inline + * `UnknownNode`s they used to be. The copy walker inserts a `\n` at every non-inline element + * boundary it crosses, so one table copies out as eight lines (`\tr `, `\th1 Day`, `\th2 Tribe`, + * …); paste replays each line as its own paragraph, and Tier 2 re-tokenizes strictly per + * paragraph, so the `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never + * sees the whole table in one pass. Result: empty `table:row` wrappers and the cells stranded as + * sibling paragraphs. The fix is the same one the sidebar entry names — grouping a paste's + * newly-split sibling paragraphs back into one rebuild fragment — and is equally out of scope. + * + * - **"milestones (ts)"** — a COMPOUND of two deliberate behaviors, neither of them a byte-level + * copy or normalization defect: the copied text is byte-correct + * (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host with NO marker + * prefix of its own round-trips clean. A milestone whose rebuild would EJECT content deliberately + * does NOT settle inside the commit that produced it (`markerEditTier2Trigger.utils.ts`: + * rearranging the line under a caret the user is still on is worse than waiting) — it pends until + * caret departure. But the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, + * `tier2Rebuild.utils.ts`) is armed only for the PASTE's own update, so the later + * departure-triggered settle no longer knows the fragment came from a paste and takes the typed + * path instead: the host's own `\p ` glyph and the pasted line's `\p ` literal both survive as + * two paragraphs, the first of them empty. Closing it means giving the dedup's arm a lifetime + * that spans the deferred settle — a paste-provenance change, not a tokenizer one, and out of + * scope here. + * + * - **"paragraph-leading space (display rule)"** — ACCEPTED normalization, matching Paratext 9, + * not a bug: isolated with a minimal non-corpus repro, pasting the literal text `"\p X"` + * (marker, its own required separator, and a SECOND, real content-leading space) into a fresh + * empty `"\p"` host produces `"\p X"` — one space, not two. The mechanism is + * `consumeSeparator()` (`usfmFragmentToUsj.ts`), whose own comment says exactly this: "Consume + * the separator whitespace after an opening marker (PT9 skips it) — all leading whitespace, not + * just a single space." That mirrors Paratext 9's own `NormalizeUsfm` re-tokenization pass + * (documented above under "Paratext 9 Reference Behavior": "whitespace collapse, newlines + * inserted before paragraph/verse markers"), which likewise collapses a whitespace run after a + * marker during paste's reformat pipeline — P10 doing the same is parity, not a divergence. + * Corpus symptom: copying ` Leading space precedes this text.` (source + * content `" Leading space precedes this text."`) round-trips through paste to + * `"Leading space precedes this text."` — the leading space is gone, same as it would be in P9. + * Kept in `KNOWN_LOSSY` (the byte-level comparison genuinely differs) but NOT a fix candidate. + */ +const KNOWN_LOSSY: { name: string; reason: string }[] = [ + { + name: "cross-reference ref target", + reason: "USJ wrapper carries no USFM bytes; content survives as prose", + }, + { + name: "sidebar (esb)", + reason: + "copy's \\n before the sidebar's nested \\p splits one paste into two ParaNodes; Tier 2 rebuilds per-paragraph so the sidebar pairing usfmFragmentToUsj.ts already has never sees both", + }, + { + name: "closed=false body char span (implicit close, no closer)", + reason: "no closer byte to mark where an unclosed span's content ends before trailing prose", + }, + { + name: "paragraph-leading space (display rule)", + reason: + "consumeSeparator() eats the whole whitespace run after a marker, matching P9's NormalizeUsfm parity — accepted, not a bug", + }, + { + name: "table with header and cells", + reason: + "a table's rows and cells are real block-level ElementNodes, so copy puts each on its own line and paste splits one table into eight sibling paragraphs Tier 2 rebuilds one at a time — the same rebuild-scoping gap as the sidebar above", + }, + { + name: "milestones (ts)", + reason: + "a milestone that ejects content pends until caret departure, and the paste-scoped own-marker-prefix dedup is armed only for the paste's own commit — so the deferred settle leaves the host's now-redundant \\p glyph behind as an empty paragraph", + }, +]; + +describe("corpus copy/paste round trip (Standard view)", () => { + for (const fixture of corpusFixtures) { + if (fixture.name === "periph") { + // Book-level front matter (no chapter at all — periph is BookNode + an UnknownNode-shaped + // periph wrapper, never chapter-scoped content), so it has no "chapter content" for + // $selectChapterContent to select from — it does not fit "a single-chapter editor state". + // A named skip (rather than a silent loop `continue`) so it shows in test output the same + // way every other excluded fixture does. + it.skip('periph (book-level front matter, no chapter — outside "a single-chapter editor state")', () => + undefined); + continue; + } + const lossy = KNOWN_LOSSY.find((entry) => entry.name === fixture.name); + const run = lossy ? it.skip : it; + run(`${fixture.name}${lossy ? ` (${lossy.reason})` : ""}`, async () => { + initializeDeserialize(undefined); + const usj = usxStringToUsj(fixture.usx); + + const { editor: sourceEditor } = await baseTestEnvironment( + serializedState(usj), + , + ); + await act(async () => sourceEditor.update(() => $selectChapterContent($getRoot()))); + const { event, getData } = copyEvent(); + await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, event)); + const copiedText = getData("text/plain"); + + const { editor: targetEditor } = await baseTestEnvironment( + serializedState(chapterHeaderSkeletonUsj(usj)), + , + ); + await act(async () => + targetEditor.update(() => { + $getRoot().getLastChild()?.selectEnd(); + targetEditor.dispatchCommand(PASTE_COMMAND, plainTextPasteEvent(copiedText)); + }), + ); + // Settle: flush any reconciliation the paste-triggered Tier 2 re-tokenization schedules + // beyond the synchronous update above (the same double-microtask flush every paste-adjacent + // suite in this directory uses). + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + const pastedUsj = deserializeSerializedEditorState( + targetEditor.getEditorState().toJSON(), + viewOptions, + ); + expect(pastedUsj).toEqual(usj); + }); + } +}); diff --git a/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx b/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx index fe134ebf6..f32efe144 100644 --- a/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx +++ b/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx @@ -258,6 +258,25 @@ export async function testEnvironmentWithCharSync( ); } +/** + * Like `testEnvironmentWithCharSync` (app mount order only), but also mounts `HistoryPlugin` — for + * undo/redo pins on a char-attribute-run fixture, where both the self-healing sync AND undo + * availability are needed at once (`historyTestEnvironment` alone has no `CharNodePlugin`, and the + * sibling char-attribute suites that need the sync have no `HistoryPlugin`). + */ +export async function testEnvironmentWithCharSyncAndHistory($initialEditorState: () => void) { + initializeSerialize(undefined, undefined); + reset(); + return baseTestEnvironment( + $initialEditorState, + <> + + + + , + ); +} + /** * Like `testEnvironment`, but in Standard view with EXPANDED notes (`markerMode: "editable"`, * `noteMode: "expanded"`) — the combination that used to make `getViewMode` return undefined and @@ -501,3 +520,53 @@ export function usjNoteOf(editor: LexicalEditor): MarkerObject { export function usjNoteFromUsfm(usfm: string): MarkerObject { return findUsjNote(usfmFragmentToUsjContent(usfm)); } + +/** + * jsdom doesn't implement `ClipboardEvent`/`DataTransfer`; the copy/cut handlers under test only + * touch `clipboardData.getData`/`setData`/`preventDefault`, so a minimal stub covers both dispatch + * and direct-call sites. Shared by every suite that dispatches `COPY_COMMAND`/`CUT_COMMAND`. + */ +export function copyEvent(): { event: ClipboardEvent; getData: (type: string) => string } { + const store = new Map(); + const clipboardData = { + getData: (type: string) => store.get(type) ?? "", + setData: (type: string, data: string) => { + store.set(type, data); + }, + }; + return { + event: { clipboardData, preventDefault: vi.fn() } as unknown as ClipboardEvent, + getData: (type: string) => clipboardData.getData(type), + }; +} + +/** + * jsdom-safe paste-event stub carrying an arbitrary clipboard MIME payload. `types`/`files` are + * populated so Lexical's own default paste handling (reached whenever a Standard-view/protection + * handler declines and the dispatch falls through to a lower-priority `PASTE_COMMAND` listener) + * can duck-type it the same way a real `ClipboardEvent` would — jsdom implements neither + * `ClipboardEvent` nor `DataTransfer`. Shared by every suite that dispatches `PASTE_COMMAND` or + * calls a paste handler directly. + */ +export function pasteEvent(payload: { [key: string]: string }): { + event: ClipboardEvent; + prevented: () => boolean; +} { + const store = new Map(Object.entries(payload)); + const preventDefault = vi.fn(); + const clipboardData = { + types: [...store.keys()], + files: [], + getData: (type: string) => store.get(type) ?? "", + }; + return { + event: { clipboardData, preventDefault } as unknown as ClipboardEvent, + prevented: () => preventDefault.mock.calls.length > 0, + }; +} + +/** A paste event whose only payload is `text/plain` — what pasting from a plain-text source + * (terminal, text editor, address bar) delivers. */ +export function plainTextPasteEvent(text: string): ClipboardEvent { + return pasteEvent({ "text/plain": text }).event; +} diff --git a/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx index 8a3330897..3ba7d44c4 100644 --- a/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx @@ -1,21 +1,35 @@ import { serializeEditorState } from "../adaptors/usj-editor.adaptor"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; import { $appendCharPara, $appendVersePara, + copyEvent, + plainTextPasteEvent, + requireDefined, + serializedState, testEnvironment, viewOptions as standardViewOptions, } from "./markerEdit.test-helpers"; import { $createMarkerPrefix, $setParaMarkerWithPrefix } from "./markerEditDeletion.utils"; +import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { $selectionToUsfmText } from "./whitespaceDisplay.plugin.utils"; import { act } from "@testing-library/react"; -import { EMPTY_USJ, MarkerObject } from "@eten-tech-foundation/scripture-utilities"; +import { EMPTY_USJ, MarkerObject, Usj } from "@eten-tech-foundation/scripture-utilities"; import { + $createPoint, + $createRangeSelection, $createTextNode, $getRoot, $getSelection, $getState, $isRangeSelection, $isTextNode, + $setSelection, $setState, + CUT_COMMAND, INSERT_PARAGRAPH_COMMAND, LexicalEditor, NODE_STATE_KEY, @@ -42,6 +56,9 @@ import { textTypeState, VerseNode, } from "shared"; +// Reaching inside only for tests. +// eslint-disable-next-line @nx/enforce-module-boundaries +import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; // jsdom implements `getBoundingClientRect` on Element but not on Range. The Enter-split test // below seeds an initial selection, which gives the editor root DOM focus as soon as it mounts; @@ -899,17 +916,6 @@ describe("multi-line plain-text paste", () => { if (typeof globalStubs.ClipboardEvent === "undefined") globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; - /** A paste event whose only payload is `text/plain` — what pasting from a plain-text source - * (terminal, text editor, address bar) delivers. */ - function plainTextPasteEvent(text: string): ClipboardEvent { - const clipboardData = { - types: ["text/plain"], - files: [], - getData: (type: string) => (type === "text/plain" ? text : ""), - }; - return { clipboardData, preventDefault: () => undefined } as unknown as ClipboardEvent; - } - it("keeps every pasted line as its own prefixed paragraph, caret at the end of the paste", async () => { // @lexical/clipboard's text/plain path calls `selection.insertParagraph()` directly per // newline — never INSERT_PARAGRAPH_COMMAND — so the engine's Enter handler can't arm @@ -1039,3 +1045,118 @@ describe("load/engine para prefix drift pin", () => { expect(engine).toEqual(loaded); }); }); + +describe("selection-delete at a settled char-span boundary (WI-2 filed)", () => { + // Regression armor for a QA-filed live repro: after typing settled a `\wj …\wj*` char span, a + // PROGRAMMATIC DOM selection built with `range.setEndAfter(spanElement)` — anchor in the text + // BEFORE the span, focus set immediately after the span's whole DOM element — was deleted, and + // the deletion absorbed one character AFTER the selection's own end: the closing curly quote + // `”` that immediately follows the span. Mouse-drawn selections never reproduced it, so this + // mirrors the DOM shape at the Lexical level with an ELEMENT-point focus (offset = the + // CharNode's own index within its parent, plus one — the exact mapping `setEndAfter` produces), + // not a text point landing inside either neighboring text node. + + /** `\p “quote \wj words\wj*” after` — built via the real adaptor (USJ → Lexical) so the `\wj` + * span carries its genuine settled shape: opening MarkerNode, NBSP-prefixed content, closing + * MarkerNode, sandwiched between two plain TextNodes. */ + function quoteWjUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: ["“quote ", { type: "char", marker: "wj", content: ["words"] }, "” after"], + }, + ], + } as unknown as Usj; + } + + async function renderQuoteWjEditor() { + return baseTestEnvironment( + serializedState(quoteWjUsj()), + , + ); + } + + /** + * Selects from just after the opening curly quote (inside `“quote `, the text BEFORE the span) + * through an ELEMENT point immediately after the whole `\wj` span — a boundary between the + * CharNode and its next sibling, the exact Lexical shape a DOM `range.setEndAfter(spanElement)` + * maps to. + */ + function $selectQuoteThroughSpanBoundary(): void { + const para = requireDefined($getRoot().getChildren().filter($isParaNode)[0], "para missing"); + const before = requireDefined( + para + .getChildren() + .find((node) => $isTextNode(node) && node.getTextContent().includes("quote")), + "leading quote text not found", + ); + const char = requireDefined(para.getChildren().find($isCharNode), "wj span not found"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(before.getKey(), 1, "text"); // right after the opening “ + selection.focus = $createPoint(para.getKey(), char.getIndexWithinParent() + 1, "element"); + $setSelection(selection); + } + + /** Select-all, walk it through the copy-text walker, and return the USFM text — the same + * byte-exact check `$handleCopyForStandardView` performs on copy/cut. */ + function $selectAllUsfmText(): string { + const root = $getRoot(); + root.select(0, root.getChildrenSize()); + const selection = $getSelection(); + if (!$isRangeSelection(selection)) throw new Error("expected a range selection"); + return $selectionToUsfmText(selection); + } + + it("CUT_COMMAND: the closing curly quote and everything after it survive byte-exactly; clipboard holds exactly the selected content", async () => { + const { editor } = await renderQuoteWjEditor(); + await act(async () => editor.update($selectQuoteThroughSpanBoundary)); + + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(CUT_COMMAND, event)); + + // Clipboard: exactly the selected content (NBSP inverted to a plain space), nothing more — + // in particular no trailing "”" that would prove the cut over-selected. + expect(getData("text/plain")).toBe("quote \\wj words\\wj*"); + + // The copy-text walker's own read of the post-cut document: the closing curly quote and + // " after" must both still be there, adjacent, byte-exact. + let usfmSelectAll = ""; + await act(async () => editor.update(() => (usfmSelectAll = $selectAllUsfmText()))); + expect(usfmSelectAll).toBe("\\p “” after"); + + // The editor's own USJ export agrees — a second, independent read of the same post-cut state. + initializeDeserialize(undefined); + const usj = requireDefined( + deserializeSerializedEditorState(editor.getEditorState().toJSON(), standardViewOptions), + "expected a deserialized USJ document", + ); + expect(usj.content).toEqual([{ type: "para", marker: "p", content: ["“” after"] }]); + }); + + it("plain removeText(): same boundary shape — no character past the selection's end is absorbed", async () => { + const { editor } = await renderQuoteWjEditor(); + await act(async () => + editor.update(() => { + $selectQuoteThroughSpanBoundary(); + const selection = $getSelection(); + if (!$isRangeSelection(selection)) throw new Error("expected a range selection"); + selection.removeText(); + }), + ); + + let usfmSelectAll = ""; + await act(async () => editor.update(() => (usfmSelectAll = $selectAllUsfmText()))); + expect(usfmSelectAll).toBe("\\p “” after"); + + initializeDeserialize(undefined); + const usj = requireDefined( + deserializeSerializedEditorState(editor.getEditorState().toJSON(), standardViewOptions), + "expected a deserialized USJ document", + ); + expect(usj.content).toEqual([{ type: "para", marker: "p", content: ["“” after"] }]); + }); +}); diff --git a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx index 2b9c18cd4..c23dab953 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx @@ -928,7 +928,9 @@ describe("$resolvePendingMarkers attribute-run re-pend guard", () => { getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -1002,7 +1004,9 @@ describe("$settlePendedDisplayOwner AttributeRunNode husk arm (dual-read)", () = getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -1130,7 +1134,9 @@ describe("$resolvePendingMarkers folds a husk-only settle's mutation", () => { getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -1192,7 +1198,9 @@ describe("$settlePendedDisplayOwner verse migration + fallthrough interaction", getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -1382,7 +1390,9 @@ describe("$resolvePendingMarkers routes a pended run PIECE through its owner's g getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } diff --git a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts index 57b53a3db..d8d24d81e 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts @@ -74,6 +74,7 @@ import { textTypeState, VerseNode, } from "shared"; +import { StructureProtectionMode } from "shared-react"; /** * The engine's mutable per-editor state, threaded through every marker-edit transform and command @@ -110,6 +111,24 @@ export interface MarkerEditContext extends Tier2Context { * caret) stays untouched. Optional with the same never-reap safe default as its sibling. */ collapsedDeleteCaretParas?: Set; + /** + * Narrows `Tier2Context.pasteRebuildArmed` from optional to REQUIRED: the marker-edit engine + * (`MarkerEditPlugin.tsx`) always constructs and maintains this field, unlike a bare + * `Tier2Context` a test may build directly to exercise the tokenizer/rebuild machinery alone + * (where "not a paste rebuild" is simply the field's absence). See its doc comment on + * `Tier2Context` for what it does. + */ + pasteRebuildArmed: { current: boolean }; + /** + * Mirrors the host `Editor`'s `structureProtectionMode` option. Read by + * `$handlePasteForStandardView` (whitespaceDisplay.plugin.utils.ts), which must decline a + * `"protected"` document's paste so `StructureKeyboardPlugin`'s HTML sanitizer still governs it + * — both register at `COMMAND_PRIORITY_HIGH`, and the marker-edit engine mounts first, so + * without this check its unconditional external-paste claim would starve the sanitizer. Wiring, + * not engine state: refreshed every render alongside `viewOptions`/`getMarker`/`logger` rather + * than gating the registration effect, so toggling it doesn't tear down and reset the engine. + */ + structureProtectionMode: StructureProtectionMode; /** * Literal text already submitted to `$requestTier2ForNode` this commit. * `$rebuildParas` is deterministic (the degradation property): a paragraph @@ -135,8 +154,13 @@ export interface MarkerEditContext extends Tier2Context { /** Same-positional-kind rule for paragraph openers. Stylesheet-first: * a marker the effective sheet KNOWS classifies by its styleType; heuristics * cover only markers absent from the sheet. Unknown markers stay as typed - * (Tier-1 renames to unknown markers stay in place). */ -function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { + * (Tier-1 renames to unknown markers stay in place). Exported for + * `tier2Rebuild.utils.ts`'s own-marker-prefix dedup, which needs the SAME + * stylesheet-first/unknown-as-paragraph classification `$buildParaFragment` + * already uses for the paragraph's own marker — a second, narrower + * `type === MarkerType.Paragraph` check there disagreed with it for any + * unknown/custom.sty marker. */ +export function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { const clean = marker.replace(/^\+/, ""); if (clean === "v" || clean === "c") return false; const kind = getMarkerFn(clean)?.type; diff --git a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx index 796b9b97a..0f3bf80aa 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx @@ -823,7 +823,9 @@ describe("$textNodeTier2Transform on attribute-run text", () => { getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -890,7 +892,9 @@ describe("$textNodeTier2Transform on pipe-text in plain content", () => { getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } @@ -981,7 +985,9 @@ describe("$textNodeTier2Transform on `//` optbreak text in plain content", () => getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } diff --git a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx new file mode 100644 index 000000000..525fc5dcb --- /dev/null +++ b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx @@ -0,0 +1,526 @@ +/** + * Multi-line marker-bearing paste semantics (live-verified, 2026-08-07) and the `\c`/`\id` strip. + * A pasted line starting with its own paragraph-marker literal owns that marker instead of also + * getting the host paragraph's cloned prefix; a marker-free line inherits the host's. `\c`/`\id` + * never survive paste normalization — pasting either used to be able to reach an unsaveable + * editor state (a second chapter/book-id node the PDP rejects on save). Kept separate from + * `whitespaceDisplay.plugin.utils.test.tsx` (the NBSP/claim-policy contract) and + * `clipboardCopyFidelity.test.tsx` (copy-side byte fidelity) so this file stays focused on the + * paste-side STRUCTURAL outcome: which paragraphs/markers/nodes a marker-bearing paste produces. + */ +import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { + findOnlyNote, + historyTestEnvironment, + pasteEvent, + serializedState, + viewOptions, +} from "./markerEdit.test-helpers"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; +import { act } from "@testing-library/react"; +// Reaching inside only for tests (same pattern as markerEdit.test-helpers). +// eslint-disable-next-line @nx/enforce-module-boundaries +import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; +import { MarkerObject, Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; +import { $dfs } from "@lexical/utils"; +import { + $createTextNode, + $getRoot, + $isTextNode, + $setState, + LexicalEditor, + PASTE_COMMAND, + TextNode, + UNDO_COMMAND, +} from "lexical"; +import { + $createMarkerNode, + $createParaNode, + $createVerseNode, + $isCharNode, + getVisibleOpenMarkerText, + NBSP, + textTypeState, +} from "shared"; + +/** Paste `text` at the caret, then flush the double microtask Tier 2's post-paste reconciliation + * needs to settle — the same pattern every paste-adjacent suite in this directory uses. */ +async function pasteAndSettle( + editor: LexicalEditor, + $select: () => void, + text: string, +): Promise { + await act(async () => + editor.update(() => { + $select(); + editor.dispatchCommand(PASTE_COMMAND, pasteEvent({ "text/plain": text }).event); + }), + ); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** Like `pasteAndSettle`, but takes the full clipboard payload map rather than assuming a bare + * `text/plain` string — used by the S4 equivalence pins below to dispatch a "full" (plain+html) + * payload and compare it against a plain-only one. */ +async function pastePayloadAndSettle( + editor: LexicalEditor, + $select: () => void, + payload: { [key: string]: string }, +): Promise { + await act(async () => + editor.update(() => { + $select(); + editor.dispatchCommand(PASTE_COMMAND, pasteEvent(payload).event); + }), + ); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** One UNDO_COMMAND dispatch, flushed the same way a paste is. */ +async function undoAndSettle(editor: LexicalEditor): Promise { + await act(async () => editor.dispatchCommand(UNDO_COMMAND, undefined)); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** The current document as USJ, via the same `toJSON` → deserialize path every sibling suite + * reads settled state through. */ +function usjOf(editor: LexicalEditor): Usj { + const usj = editor + .getEditorState() + .read(() => deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions)); + if (!usj) throw new Error("editor state did not serialize to USJ"); + return usj; +} + +/** All plain-text content of a top-level `MarkerObject`, joined — ignores nested objects (verses, + * notes); sufficient for the pure-text paragraphs these pins build. */ +function textOf(content: MarkerObject): string { + return (content.content ?? []) + .filter((item): item is string => typeof item === "string") + .join(""); +} + +/** `[marker, textOf(paragraph)]` for every top-level paragraph in `usj`. Fails loudly (rather than + * silently reading `undefined` off a string via an unchecked cast) if any top-level entry is a + * bare STRING instead of a paragraph-shaped object — a chapter/verse/book token splitting the + * enclosing paragraph and stranding plain text outside it is exactly the poisoned-save shape a + * leaked `\c`/`\id` produces (see the `\c/\id strip on paste` describe below), and a helper that + * masked that shape instead of erroring could hide the very regression these pins exist to catch. */ +function paraMarkerText(usj: Usj): [string | undefined, string][] { + return usj.content.map((content, index) => { + if (typeof content === "string") + throw new Error( + `expected a paragraph-shaped object at usj.content[${index}], found a bare string: ${JSON.stringify(content)}`, + ); + return [content.marker, textOf(content)]; + }); +} + +/** A realistic book+chapter+paragraph document (`usxStringToUsj`, not the bare single-paragraph + * fixture `singleParaHost` builds) — needed for chapter/book-COUNT assertions, which are + * meaningless without a real chapter/book already present for a pasted `\c`/`\id` to (not) + * duplicate. Returns the editor and the paragraph's own text node ("before after"). */ +async function bookChapterParaHost(): Promise<{ editor: LexicalEditor; text: TextNode }> { + initializeDeserialize(undefined); + const usx = usxStringToUsj( + `Ruth` + + `before after`, + ); + const { editor } = await baseTestEnvironment( + serializedState(usx), + , + ); + let text: TextNode | undefined; + editor.getEditorState().read(() => { + text = $dfs($getRoot()) + .map(({ node }) => node) + .filter($isTextNode) + .find((node) => node.getTextContent().includes("before")); + }); + if (!text) throw new Error("host text node not found in the book/chapter/paragraph fixture"); + return { editor, text }; +} + +/** Every top-level bare STRING entry in `usj.content` — should always be empty. A non-empty + * result means a chapter/verse/book token split the enclosing paragraph and stranded plain text + * outside it: the poisoned-save shape a leaked `\c`/`\id` produces. */ +function topLevelBareStrings(usj: Usj): string[] { + return usj.content.filter((item): item is string => typeof item === "string"); +} + +/** A fresh single-paragraph `\p A` host, with `HistoryPlugin` mounted so undo is available. */ +async function singleParaHost(): Promise<{ editor: LexicalEditor; text: TextNode }> { + initializeDeserialize(undefined); + let text!: TextNode; + const { editor } = await historyTestEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("A"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + return { editor, text }; +} + +describe("multi-line marker-bearing paste semantics (live-verified 2026-08-07)", () => { + it('paste "\\p one\\n\\p two" at end of "\\p A": no doubled markers, no empty stray paragraph', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "\\p one\n\\p two"); + + expect(paraMarkerText(usjOf(editor))).toEqual([ + ["p", "A"], + ["p", "one"], + ["p", "two"], + ]); + }); + + it('undo after "\\p one\\n\\p two" restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await singleParaHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(1, 1), "\\p one\n\\p two"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + it('paste marker-free "one\\ntwo" at end of "\\p A": both lines inherit the host marker (existing behavior, re-pinned)', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "one\ntwo"); + + expect(paraMarkerText(usjOf(editor))).toEqual([ + ["p", "Aone"], + ["p", "two"], + ]); + }); + + it('undo after marker-free "one\\ntwo" restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await singleParaHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(1, 1), "one\ntwo"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + it('paste "tail\\n\\q1 line" at end of "\\p A": first line merges into the host, second owns its own marker', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "tail\n\\q1 line"); + + expect(paraMarkerText(usjOf(editor))).toEqual([ + ["p", "Atail"], + ["q1", "line"], + ]); + }); + + it('undo after "tail\\n\\q1 line" restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await singleParaHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(1, 1), "tail\n\\q1 line"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + /** A `\p` host with a verse, for the note/verse-materialization pins below. */ + async function versedHost(): Promise<{ editor: LexicalEditor; text: TextNode }> { + initializeDeserialize(undefined); + let text!: TextNode; + const { editor } = await historyTestEnvironment(() => { + const para = $createParaNode("p"); + const verse = $createVerseNode("1", getVisibleOpenMarkerText("v", "1")); + text = $createTextNode("In the beginning God created"); + $getRoot().append(para.append($createMarkerNode("p"), verse, text)); + }); + return { editor, text }; + } + + it('paste "\\f + \\ft note\\f*" mid-verse: a collapsed NoteNode materializes with USJ caller "+"', async () => { + const { editor, text } = await versedHost(); + await pasteAndSettle(editor, () => text.select(9, 9), "\\f + \\ft note\\f*"); // "In the be|ginning..." + + editor.getEditorState().read(() => { + const note = findOnlyNote($getRoot()); + expect(note.getMarker()).toBe("f"); + expect(note.getCaller()).toBe("+"); + expect(note.getIsCollapsed()).toBe(true); + }); + const para = (usjOf(editor).content as MarkerObject[])[0]; + const note = para.content?.find( + (item): item is MarkerObject => typeof item !== "string" && item.type === "note", + ); + expect(note).toBeDefined(); + expect(note?.marker).toBe("f"); + expect(note?.caller).toBe("+"); + }); + + it('undo after "\\f + \\ft note\\f*" mid-verse restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await versedHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(9, 9), "\\f + \\ft note\\f*"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + it('paste "\\v 2 rest" at paragraph end: a real VerseNode is created, the verse sequence stays sane', async () => { + const { editor, text } = await versedHost(); + const end = "In the beginning God created".length; + await pasteAndSettle(editor, () => text.select(end, end), "\\v 2 rest"); + + const para = (usjOf(editor).content as MarkerObject[])[0]; + const verses = (para.content ?? []).filter( + (item): item is MarkerObject => typeof item !== "string" && item.type === "verse", + ); + expect(verses.map((verse) => verse.number)).toEqual(["1", "2"]); + }); + + it('undo after "\\v 2 rest" restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await versedHost(); + const preUsj = usjOf(editor); + const end = "In the beginning God created".length; + + await pasteAndSettle(editor, () => text.select(end, end), "\\v 2 rest"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); +}); + +describe("\\c/\\id strip on paste", () => { + it('paste "\\c 5" on its own line mid-chapter: no chapter node created, no "\\c" survives, content unchanged', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "\\c 5"); + + const usj = usjOf(editor); + expect(paraMarkerText(usj)).toEqual([["p", "A"]]); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).not.toContain("\\c"); + }); + }); + + it('undo after pasting bare "\\c 5" (a no-op paste) leaves the USJ unchanged', async () => { + const { editor, text } = await singleParaHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(1, 1), "\\c 5"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + it('paste "\\id GEN" is stripped the same way as "\\c"', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "\\id GEN"); + + const usj = usjOf(editor); + expect(paraMarkerText(usj)).toEqual([["p", "A"]]); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).not.toContain("\\id"); + }); + }); + + it('paste "before\\n\\c 5\\nafter": the "\\c" line vanishes, "before"/"after" paste per the normal multi-line rules', async () => { + const { editor, text } = await singleParaHost(); + await pasteAndSettle(editor, () => text.select(1, 1), "before\n\\c 5\nafter"); + + expect(paraMarkerText(usjOf(editor))).toEqual([ + ["p", "Abefore"], + ["p", "after"], + ]); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).not.toContain("\\c"); + }); + }); + + it('undo after "before\\n\\c 5\\nafter" restores the exact pre-paste USJ in one step', async () => { + const { editor, text } = await singleParaHost(); + const preUsj = usjOf(editor); + + await pasteAndSettle(editor, () => text.select(1, 1), "before\n\\c 5\nafter"); + await undoAndSettle(editor); + + expect(usjOf(editor)).toEqual(preUsj); + }); + + it("a pasted \\c never leaves the editor in the unsaveable state the live repro produced: exactly one chapter survives, no bare top-level string strands outside the paragraph", async () => { + // Live repro (2026-08-07): pasting a bare `\c 2` mid-chapter put a second chapter node in a + // real book/chapter/paragraph document, and every subsequent save failed with the PDP's + // "Multiple chapter markers present" — the error surfaced only in the renderer log, so disk + // and other editors silently stopped updating. Reproduced here in a realistic book+chapter+ + // paragraph document (not the bare single-paragraph fixture the pins above use) so the + // chapter-count assertion means something. + const { editor, text } = await bookChapterParaHost(); + + await pasteAndSettle(editor, () => text.select(7, 7), "\\c 5"); + + const usj = usjOf(editor); + const chapters = usj.content.filter( + (item): item is MarkerObject => typeof item !== "string" && item.type === "chapter", + ); + expect(chapters).toHaveLength(1); + expect(chapters[0].number).toBe("1"); // the ORIGINAL chapter, untouched — not the pasted "5" + // A bare top-level string is the other half of the poisoned shape: a chapter token closes the + // enclosing paragraph the same way it does on a real load, stranding whatever text followed + // it outside any paragraph at all. + expect(topLevelBareStrings(usj)).toEqual([]); + }); + + it('paste "x \\c 5 y" mid-paragraph (the token is NOT at the start of its line): still exactly one chapter, no stranded top-level string', async () => { + // The `\c`/`\id` strip is not anchored to a line's start — a token can land mid-sentence + // (a paste that doesn't happen to fall on a line boundary), and an anchored strip would miss + // it entirely: the unstripped shape reproduces the SAME poisoning (a second chapter node) PLUS + // a bare top-level string for whatever followed the marker's payload, since a chapter token + // still closes the enclosing paragraph wherever it lands. + const { editor, text } = await bookChapterParaHost(); + + await pasteAndSettle(editor, () => text.select(7, 7), "x \\c 5 y"); + + const usj = usjOf(editor); + const chapters = usj.content.filter( + (item): item is MarkerObject => typeof item !== "string" && item.type === "chapter", + ); + expect(chapters).toHaveLength(1); + expect(chapters[0].number).toBe("1"); + // Not asserting `not.toContain("\\c")` on the whole document here: this fixture's OWN book + // legitimately carries a real "\c 1" glyph, unlike the bare single-paragraph fixture the + // earlier pins use — the chapter-count and paragraph-content assertions already cover what + // matters (the pasted "\c 5" left no trace, structural or textual, inside the paragraph). + expect(topLevelBareStrings(usj)).toEqual([]); + expect((usj.content[2] as MarkerObject).content).toEqual(["before x after"]); + }); + + it('paste "text \\id GEN more" mid-paragraph: still exactly one book id, one chapter, no stranded top-level string', async () => { + const { editor, text } = await bookChapterParaHost(); + + await pasteAndSettle(editor, () => text.select(7, 7), "text \\id GEN more"); + + const usj = usjOf(editor); + const books = usj.content.filter( + (item): item is MarkerObject => typeof item !== "string" && item.type === "book", + ); + const chapters = usj.content.filter( + (item): item is MarkerObject => typeof item !== "string" && item.type === "chapter", + ); + expect(books).toHaveLength(1); + expect(books[0].code).toBe("RUT"); // the ORIGINAL book, untouched — not the pasted "GEN" + expect(chapters).toHaveLength(1); + expect(topLevelBareStrings(usj)).toEqual([]); + expect((usj.content[2] as MarkerObject).content).toEqual(["before text after"]); + }); +}); + +describe("own-marker-prefix dedup: unknown/custom.sty markers", () => { + it('paste "\\zz one two" into an EMPTY "\\p" host: the unrecognized marker still owns the paragraph — no stray empty leading paragraph', async () => { + // The dedup's embedded-literal check must classify markers the SAME way `$buildParaFragment`'s + // own guard does (stylesheet-first, unknown-as-paragraph) — a narrower `type === + // MarkerType.Paragraph` comparison rejected every unknown/custom.sty marker and left this + // exact shape (a paste starting with its own marker literal into an already-prefixed empty + // paragraph) producing a stray empty host paragraph for any marker the bundled sheet doesn't + // recognize. + initializeDeserialize(undefined); + let sep!: TextNode; + const { editor } = await historyTestEnvironment(() => { + const para = $createParaNode("p"); + sep = $createTextNode(NBSP); + $setState(sep, textTypeState, "marker-trailing-space"); + $getRoot().append(para.append($createMarkerNode("p"), sep)); + }); + + await pasteAndSettle( + editor, + () => sep.select(sep.getTextContentSize(), sep.getTextContentSize()), + "\\zz one two", + ); + + expect(paraMarkerText(usjOf(editor))).toEqual([["zz", "one two"]]); + }); +}); + +describe("paste-as-plain-text equivalence (S4): no literal mode, plain always wins", () => { + // S4 (docs/superpowers/specs/2026-08-06-clipboard-semantics.md): Ctrl+Shift+V / "paste as plain + // text" narrows the clipboard payload down to `text/plain` only, but `$handlePasteForStandardView` + // reads `text/plain` unconditionally whenever it is present — the `text/html` leg only comes into + // play when `text/plain` is ABSENT (`htmlPasteText(html)` fallback). So a full (plain+html) paste + // and a plain-only paste of the SAME `text/plain` bytes must produce byte-identical final USJ: by + // construction, not by coincidence. Each pin below dispatches the same paste text twice, on two + // fresh hosts — once with a DELIBERATELY mismatched `text/html` alongside it (proving html is + // ignored outright, not merely equivalent to plain here) and once with `text/plain` alone — then + // asserts the two final documents are identical. There is no separate "paste literally, don't + // tokenize markers" code path in Standard view (matching P9, which has no Paste Special either): + // the second pin below additionally asserts a plain-only paste still tokenizes a marker pair into + // a real CharNode rather than leaving it as literal `\marker` text. + const MISMATCHED_HTML = "

this text must never appear in the pasted result

"; + + it('"\\p one\\n\\p two" (multi-line paragraph split): full (plain+html) payload and plain-only payload paste identically', async () => { + const hostA = await singleParaHost(); + const hostB = await singleParaHost(); + const text = "\\p one\n\\p two"; + + await pastePayloadAndSettle(hostA.editor, () => hostA.text.select(1, 1), { + "text/plain": text, + "text/html": MISMATCHED_HTML, + }); + await pastePayloadAndSettle(hostB.editor, () => hostB.text.select(1, 1), { + "text/plain": text, + }); + + const usjA = usjOf(hostA.editor); + const usjB = usjOf(hostB.editor); + expect(usjA).toEqual(usjB); + // Matches the multi-line marker-bearing paste pin above for this exact fixture: no doubled + // markers, no stray empty paragraph. + expect(paraMarkerText(usjA)).toEqual([ + ["p", "A"], + ["p", "one"], + ["p", "two"], + ]); + hostA.editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).not.toContain("must never appear"); + }); + }); + + it('"\\nd Lord\\nd* " (marker-bearing inline span): full (plain+html) payload and plain-only payload paste identically, and the plain-only payload still tokenizes — there is no literal mode', async () => { + const hostA = await singleParaHost(); + const hostB = await singleParaHost(); + const text = "\\nd Lord\\nd* "; + + await pastePayloadAndSettle(hostA.editor, () => hostA.text.select(1, 1), { + "text/plain": text, + "text/html": MISMATCHED_HTML, + }); + await pastePayloadAndSettle(hostB.editor, () => hostB.text.select(1, 1), { + "text/plain": text, + }); + + const usjA = usjOf(hostA.editor); + const usjB = usjOf(hostB.editor); + expect(usjA).toEqual(usjB); + hostA.editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).not.toContain("must never appear"); + }); + // "No literal mode": the plain-only payload's `\nd`…`\nd*` pair must be recognized by Tier 2 + // and rebuilt as a real CharNode, not survive as unrecognized literal marker text. + hostB.editor.getEditorState().read(() => { + const chars = $dfs($getRoot()) + .map(({ node }) => node) + .filter($isCharNode); + const ndChars = chars.filter((char) => char.getMarker() === "nd"); + expect(ndChars).toHaveLength(1); + expect(ndChars[0].getTextContent()).toContain("Lord"); + }); + }); +}); diff --git a/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx b/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx index ce94e51e6..86c7502bb 100644 --- a/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx +++ b/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx @@ -356,7 +356,9 @@ describe("collab-materialized milestone settles into a re-tokenizable run", () = getMarker: bundledGetMarker, pendingKeys: new Set(milestoneFirst ? [msKey, runKey] : [runKey, msKey]), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; editor.update( () => { diff --git a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx index 2472744ae..4dfd7b00b 100644 --- a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx +++ b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx @@ -9,6 +9,7 @@ import { $noteContentText, findOnlyNote, noteUsx, + plainTextPasteEvent, renderStandardEditorWithCollapsedNote, renderStandardEditorWithUnclosedNote, requireDefined, @@ -898,12 +899,6 @@ describe("multi-line plain-text paste inside note content", () => { return { clipboardData, preventDefault: () => undefined } as unknown as ClipboardEvent; } - /** A paste event whose only payload is `text/plain` — what pasting from a plain-text source - * (terminal, text editor, address bar) delivers. */ - function plainTextPasteEvent(text: string): ClipboardEvent { - return pasteEventWith({ "text/plain": text }); - } - /** Paste the given event with the selection set by `$select` — one update, like a real paste. */ async function pasteEventAt( editor: LexicalEditor, @@ -1154,7 +1149,7 @@ describe("multi-line plain-text paste inside note content", () => { }); it("outranks the structure-protection paste handler: in-note pastes break with \\fp even when protected", async () => { - // With structure protection ON (the Simple-mode shipping default) StructureProtectionPlugin + // With structure protection ON (the Simple-mode shipping default) StructureKeyboardPlugin // handles PASTE at COMMAND_PRIORITY_HIGH: any html-bearing paste is sanitize-inserted (or // blocked) before a lower-priority in-note claim can run, so pasted line breaks never became // `\fp` breaks inside the note. The in-note claim must outrank it — an `\fp` break edits @@ -1244,14 +1239,36 @@ describe("multi-line plain-text paste inside note content", () => { expect(note.getTextContent()).toContain("stuff"); }); }); + + it("strips \\c/\\id bytes from a multi-line paste — they never land in note content", async () => { + // Single-line in-note pastes decline to the main external-paste handler + // ($handlePasteForStandardView, whitespaceDisplay.plugin.utils.ts), which already strips + // `\c`/`\id`; this CRITICAL multi-line claim is a SEPARATE code path that did not share the + // strip until now. A `\c`/`\id` token landing in note content re-tokenizes through the same + // Tier 2 tokenizer a paragraph does, so it is just as reachable — and just as save-poisoning + // (a live-verified `\c` paste anywhere puts a second chapter node in the editor and fails + // every subsequent PDP save) — as one landing in body text. + const { editor } = await renderStandardEditorWithUnclosedNote(); + + await pasteAt(editor, "first\n\\c 5\nlast", $selectFtEnd); + + editor.getEditorState().read(() => { + const note = findOnlyNote($getRoot()); + expect(note.getTextContent()).not.toContain("\\c"); + }); + }); }); - it("normalizes a pasted data-NBSP to `~` like the single-line path (round-trips to NBSP in USJ)", async () => { - // This claim outranks the standard-view paste normalization at HIGH, so it must apply the - // same display mapping itself: a pasted data-NBSP lands as `~` (its display form). Inserted - // raw, the NBSP is indistinguishable from a display-NBSP (a plain space in a run) and - // serialization corrupts it into a plain space. A pasted literal `~` is untouched — it IS - // the display form and already round-trips to a data NBSP, exactly like typing `~`. + it("normalizes a pasted data-NBSP with no marker adjacency to `~` (round-trips to NBSP in USJ)", async () => { + // This claim outranks the standard-view paste normalization at HIGH, so it runs its own NBSP + // normalization — the SAME positional rule that path uses (`$normalizePastedNbsp`, + // whitespaceDisplay.plugin.utils.ts), not a divergent mapping of its own. This payload has no + // marker literal anywhere near its NBSP, so it lands as `~` (genuine data) under the + // positional rule exactly as it would under a blanket one — the positional-vs-blanket + // distinction only shows up for a marker-adjacent NBSP (see the next test). Inserted raw, the + // NBSP is indistinguishable from a display-NBSP (a plain space in a run) and serialization + // would corrupt it into a plain space; a pasted literal `~` is untouched — it IS the display + // form and already round-trips to a data NBSP, exactly like typing `~`. const { editor } = await renderStandardEditorWithUnclosedNote(); await pasteAt(editor, `tilde~data${NBSP}pair\nnext`, $selectFtEnd); @@ -1280,6 +1297,60 @@ describe("multi-line plain-text paste inside note content", () => { ).toBe(true); }); + it("normalizes a marker-adjacent pasted NBSP positionally, not into `~` (the same corruption class the main paste path was fixed for)", async () => { + // Before this was wired to the shared positional rule, this claim's OWN blanket NBSP→`~` + // mapping turned the required separator after a marker's opener into data, corrupting a + // recognizable `\nd`…`\nd*` pair the same way a same-editor footnote paste once corrupted + // `\f`/`\fr`/`\ft` on the main external-paste path (whitespaceDisplay.plugin.utils.test.tsx's + // "2026-08-07 live-repro" pin). Multi-line so this CRITICAL in-note claim (not the HIGH + // external-paste handler) is the one doing the normalization. + const { editor } = await renderStandardEditorWithUnclosedNote(); + + await pasteAt(editor, `\\nd${NBSP}light\\nd*\nsecond`, $selectFtEnd); + + const usj = requireDefined( + editorUsjAdaptor.deserializeEditorState(editor.getEditorState(), viewOptions), + "editor state did not serialize to USJ", + ); + const chars: MarkerContent[] = []; + const walk = (content: MarkerContent[] | undefined): void => { + content?.forEach((item) => { + if (typeof item !== "string") { + chars.push(item); + walk(item.content); + } + }); + }; + walk(usj.content); + // The pasted pair tokenized into a real `nd` char span (not literal, never-recognized text) — + // the same structural proof the main path's doubled-glyph pin uses. + expect(chars.some((item) => typeof item !== "string" && item.marker === "nd")).toBe(true); + editor.getEditorState().read(() => { + const note = findOnlyNote($getRoot()); + expect(note.getTextContent()).not.toContain("~"); + }); + }); + + it("normalizes a data-NBSP at the start of a note's SECOND \\fp line to a plain space, not `~` — the same leading-NBSP-per-line pass the main paste path uses", async () => { + // `$normalizePastedNbsp`'s leading-NBSP pass is `gm`-flagged: `^` matches right after every + // `\n`, not just at the very start of the whole paste (whitespaceDisplay.plugin.utils.test.tsx + // pins the identical outcome for the main external-paste path). A leading NBSP reads as a + // structural separator with nothing in front of it to match against, the same as a partial + // selection starting exactly at a char span's own leading separator. + const { editor } = await renderStandardEditorWithUnclosedNote(); + + await pasteAt(editor, `first\n${NBSP}second`, $selectFtEnd); + + editor.getEditorState().read(() => { + const note = findOnlyNote($getRoot()); + const chars = note.getChildren().filter($isCharNode); + expect(chars.map((c) => c.getMarker())).toEqual(["ft", "fp"]); + // Leading NBSP on the second line became a plain space, not data (`~`). + expect(chars[1].getTextContent()).toContain(" second"); + expect(chars[1].getTextContent()).not.toContain("~"); + }); + }); + it("serializes the two-\\fp note to USJ with no newline characters (the paragraph look is display-only)", async () => { // The editor renders each \fp as a paragraph start via a CSS-generated line break // (usj-nodes.css), so the pasted line breaks must live on ONLY as \fp char spans — diff --git a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx new file mode 100644 index 000000000..f99258b01 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx @@ -0,0 +1,483 @@ +/** + * Optbreak (`//`) copy/paste coverage (filed 2026-08-13). Live symptom (TJ, most likely a + * pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the + * clipboard correctly, but pasting that same clipboard back did NOT insert the optbreak, while + * pasting external plaintext `//` DID. + * + * Three hypotheses, verified below rather than assumed: + * + * - **H1** (the live symptom, on an OLD build): with the lexical flavor stripped by the async + * Ctrl+V clipboard read, Lexical's default paste prefers `text/html`; `//` renders in `text/html` + * as nothing at all (see "copy characterization" below — `UnknownNode.exportDOM` returns + * `{element: null}` unconditionally, so `@lexical/html`'s `$appendNodesToHTML` returns `false` + * before ever visiting the optbreak's display child, dropping it silently). This branch's + * `$handlePasteForStandardView` (`whitespaceDisplay.plugin.utils.ts`) prefers `text/plain` + * whenever present, and this branch's own copy always populates `text/plain` — H1 predicts the + * symptom is already fixed here for the plain path. Pinned, not assumed, in "paste round trip" + * below. + * - **H2**: `$normalizePastedNbsp`'s marker-token regexes (`whitespaceDisplay.plugin.utils.ts`) + * only match `\`-shaped tokens — an NBSP adjacent to `//` would not be recognized as display + * whitespace and would fall to the blanket data-`~` rule instead. Characterized below: this + * branch's own copy walker (`$selectionToUsfmText`) inverts every TextNode's NBSP to a plain + * space unconditionally (the note-internal-separator special case aside), so `text/plain` never + * carries an NBSP next to `//` to begin with — H2 does not bite the round trip this task is + * scoped to, and doubly so: this branch's own `text/html` never carries `//` at all for an + * optbreak either (see the `text/html` pin below), so there is no `//` in P10's OWN html for a + * foreign NBSP to even land beside. A synthetic html-only foreign payload that DOES carry such + * an NBSP is characterized (not "fixed" — unreachable via this branch's own copy, reachable only + * via a genuinely foreign clipboard source) for completeness: the NBSP is genuine data from that + * foreign source, and settles to the correct display form for real data-NBSP (`~`), just one + * position off from where a marker-adjacent rule would have recognized it as a separator. + * - **H3** (confirmed root cause, TWO independent gaps): the same-namespace + * `application/x-lexical-editor` fast path (S2 in the clipboard semantics doc) reconstructs + * nodes via `@lexical/clipboard`'s JSON generator/parser, not DOM `importDOM`. That generator + * (`$appendNodesToJSON`, inside `@lexical/clipboard`'s `LexicalClipboard.dev.js`) computes + * `shouldExclude` from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, + * hardcoded, for EVERY copy-out destination `$appendNodesToJSON` handles (`'clone'` is never + * passed by any Lexical-shipped code path in the installed version). `UnknownNode.excludeFromCopy` + * (`UnknownNode.ts`) returned `destination !== "clone"` — excluding itself unconditionally. When + * a node is excluded, `$appendNodesToJSON` does not drop it silently — it HOISTS the excluded + * node's own children into the parent's list in its place. For an optbreak, that stranded the + * `//` `ImmutableTypedTextNode` (a DecoratorNode) as a bare sibling with no owning `UnknownNode` + * wrapper: `$parseSerializedNode` on paste reconstructed a loose decorator, not a recognized + * optbreak — `$isUnknownNode` is false for it, and nothing re-tokenizes a decorator's text. + * `text/html` is UNAFFECTED by this whole mechanism: `$appendNodesToHTML` (`@lexical/html`) + * computes the same `excludeFromCopy('html')` value but returns early on `UnknownNode`'s + * unconditional `{element: null}` export BEFORE ever consulting it — so the fix below changes + * ONLY the lexical-JSON flavor. Fixed by narrowing `UnknownNode.excludeFromCopy` to leave a + * CHILD-BEARING "optbreak" out of the exclusion. A SECOND, independent gap surfaced while + * verifying that fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the + * optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked + * the wrapper "selected" under Lexical's default `isSelected` (key-membership in + * `selection.getNodes()`, blind to the node's own child count), producing a CHILDLESS + * `{type:"optbreak"}` placeholder in the lexical-JSON copy even though the live node has a + * child — disagreeing with `text/plain`, which correctly emits nothing for that same boundary. + * `excludeFromCopy` cannot see this (it has no visibility into which children a given selection + * will include), so a scoped `isSelected` override closes it directly, mirroring + * `$selectionToUsfmText`'s own `getNodes()`-based walk. Every OTHER `UnknownNode` kind (figure, + * table, sidebar, periph, ref) keeps the OLD, still-excluding behavior — not because their + * content differs meaningfully (it doesn't: every kind's marker/attribute bytes are the same + * content-free `ImmutableTypedTextNode` display decorators optbreak's `//` is), but pending + * per-kind paste verification this task did not do; the measured, more severe residual for those + * kinds (a lexical-flavor figure paste renders its full literal USFM bytes on screen while the + * USJ silently drops the figure node and its attributes) is recorded in the clipboard semantics + * doc's "Deferred / Out of Scope" list rather than fixed here. + */ + +import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { + copyEvent, + pasteEvent, + serializedState, + testEnvironment, + viewOptions, +} from "./markerEdit.test-helpers"; +import { $handlePasteForStandardView } from "./whitespaceDisplay.plugin.utils"; +import { + deserializeSerializedEditorState, + initialize as initializeDeserialize, +} from "../adaptors/editor-usj.adaptor"; +import { act } from "@testing-library/react"; +import { HistoryPlugin } from "@lexical/react/LexicalHistoryPlugin"; +// Reaching inside only for tests (same pattern as markerEdit.test-helpers). +// eslint-disable-next-line @nx/enforce-module-boundaries +import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; +import { MarkerObject, Usj } from "@eten-tech-foundation/scripture-utilities"; +import { + $createPoint, + $createRangeSelection, + $createTextNode, + $getRoot, + $isTextNode, + $setSelection, + COPY_COMMAND, + CUT_COMMAND, + LexicalEditor, + PASTE_COMMAND, + TextNode, + UNDO_COMMAND, +} from "lexical"; +import { + $createMarkerNode, + $createParaNode, + $isParaNode, + $isUnknownNode, + NBSP, + ParaNode, +} from "shared"; + +// jsdom implements neither `ClipboardEvent` nor `DragEvent`; the fallback path the "H3" round-trip +// pin exercises on purpose (Lexical's own default rich-paste handling, reached whenever +// `$handlePasteForStandardView` declines) checks `instanceof`/class-name against both under the +// hood (`objectKlassEquals`). Same stub as `clipboardCopyFidelity.test.tsx` / +// `attributeContextPasteFidelity.test.tsx`; only defined if not already present. +const globalStubs: { DragEvent?: unknown; ClipboardEvent?: unknown } = globalThis; +if (typeof globalStubs.DragEvent === "undefined") + globalStubs.DragEvent = class DragEvent extends Event {}; +if (typeof globalStubs.ClipboardEvent === "undefined") + globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; + +/** A minimal book+chapter+two-paragraph USJ document — the first paragraph carries `content`, the + * second is a fixed "depart here" paragraph (the shape `settledGetUsj.test.tsx`'s `twoParaUsj` + * helper builds). A typed or pasted literal `//` PENDS rather than settling immediately, no matter + * where the caret lands after insertion (`$textNodeTier2Transform`, `markerEditTier2Trigger.utils + * .ts`: "without pending here a typed `//` would delete its key and stay literal text forever") — + * it re-tokenizes into an optbreak only once the caret actually DEPARTS to a different node, via + * `$resolvePendingMarkers`. The second paragraph is the departure target every paste round-trip + * pin below moves the caret into after pasting. Spaces flank the optbreak on both sides by default + * (PT9 preserves optbreak whitespace byte-for-byte — pinned in `editor-usj-adaptor.test.tsx`'s + * "round-trips optbreak spacing variant" cases). */ +function optbreakUsj( + content: MarkerObject["content"] = ["before ", { type: "optbreak" }, " after"], +): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { type: "book", marker: "id", code: "RUT", content: ["RUT"] }, + { type: "chapter", marker: "c", number: "1" }, + { type: "para", marker: "p", content }, + { type: "para", marker: "p", content: ["depart here"] }, + ], + } as unknown as Usj; +} + +/** Mounts a headless Standard-view editor (`MarkerEditPlugin`, optionally `HistoryPlugin`) with + * `usj` loaded. */ +async function renderUsjEditor(usj: Usj, withHistory = false): Promise<{ editor: LexicalEditor }> { + return baseTestEnvironment( + serializedState(usj), + withHistory ? ( + <> + + + + ) : ( + + ), + ); +} + +/** Selects the document's one paragraph in full (element offsets, start to end) — includes the + * paragraph's own `\p` marker and trailing separator in the walk, matching how + * `clipboardCopyFidelity.test.tsx`'s "joins a full multi-paragraph selection" test selects a whole + * block. Skips the book/chapter header, matching how a user actually selects inside an open + * chapter. Used only for the copy-characterization pins below, where the realistic "\p " prefix is + * part of what is being pinned. */ +function $selectWholePara(): ParaNode { + const para = $getRoot().getChildren().find($isParaNode); + if (!para) throw new Error("expected a ParaNode"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(para.getKey(), 0, "element"); + selection.focus = $createPoint(para.getKey(), para.getChildrenSize(), "element"); + $setSelection(selection); + return para; +} + +/** Selects the paragraph's CONTENT only — its own `\p` marker and trailing-separator token (the + * first two children `usj-editor.adaptor.ts`'s `createPara` always prepends in editable Standard + * view) are excluded. Used for the round-trip/cut pins below: a real Ctrl+C selecting the words + * around an optbreak never captures the host paragraph's own marker glyph, and excluding it here + * keeps each pin's target-side paste an inline-content insertion rather than a block-level one — + * the block-splitting behavior a whole-paragraph node paste triggers is Lexical's own generic + * rich-paste mechanic, orthogonal to what this file is pinning. */ +function $selectParaContent(): ParaNode { + const para = $getRoot().getChildren().find($isParaNode); + if (!para) throw new Error("expected a ParaNode"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(para.getKey(), 2, "element"); + selection.focus = $createPoint(para.getKey(), para.getChildrenSize(), "element"); + $setSelection(selection); + return para; +} + +/** The document's own USJ, via the same `toJSON` -> deserialize path every sibling clipboard suite + * reads settled state through. */ +function usjOf(editor: LexicalEditor): Usj { + initializeDeserialize(undefined); + const usj = editor + .getEditorState() + .read(() => deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions)); + if (!usj) throw new Error("editor state did not serialize to USJ"); + return usj; +} + +/** Flushes the double-microtask paste/settle window every paste-adjacent suite in this directory + * uses, after the `act` block containing the triggering update. */ +async function settle(): Promise { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe("copy characterization: what the walker actually emits around an optbreak", () => { + it("text/plain carries the exact `//` token with its significant flanking spaces — no NBSP anywhere near it", async () => { + const { editor } = await renderUsjEditor(optbreakUsj()); + await act(async () => editor.update($selectWholePara)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + // "\p" (marker glyph) + " " (the paragraph's own trailing-NBSP separator, inverted to a plain + // space) + "before " + "//" (the optbreak's decorator child, contributed byte-for-byte) + + // " after". No NBSP (U+00A0) anywhere in the payload — `$selectionToUsfmText` inverts every + // TextNode's NBSP to a plain space unconditionally, so the flanking spaces next to `//` are + // ordinary ASCII spaces, not display artifacts requiring positional normalization on paste. + const plain = getData("text/plain"); + expect(plain).toBe("\\p before // after"); + expect(plain).not.toContain(NBSP); + }); + + it("text/html drops the optbreak's `//` bytes entirely — UnknownNode.exportDOM() always returns a null element, so @lexical/html's node walk short-circuits before ever visiting the display child (a documented gap, harmless here because text/plain always wins when present)", async () => { + const { editor } = await renderUsjEditor(optbreakUsj()); + await act(async () => editor.update($selectWholePara)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + const html = getData("text/html"); + expect(html).not.toContain("//"); + expect(html).toContain("before"); + expect(html).toContain("after"); + }); + + it("carrier agreement at a childless boundary: a selection ending exactly at the optbreak's own start (touching the wrapper, not its `//` child) excludes it from BOTH carriers, not just text/plain", async () => { + // Before the `isSelected` override (`UnknownNode.ts`), this selection shape — a focus point + // resolving as an ELEMENT-type point ON the optbreak at offset 0 — still marked the WRAPPER + // "selected" under Lexical's default `isSelected` (key-membership in `selection.getNodes()`, + // blind to whether any of the wrapper's own children are actually covered), even though the + // `//` child itself was not selected. `$appendNodesToJSON` then serialized a CHILDLESS + // `{type:"unknown", tag:"optbreak", children:[]}` placeholder into the lexical-JSON payload — + // a carrier disagreement (`text/plain`, walking the same `getNodes()` list via + // `$selectionToUsfmText`, correctly emitted nothing for this boundary). Measured directly + // against this exact selection shape before writing the fix. + const { editor } = await renderUsjEditor(optbreakUsj()); + let optbreakKey = ""; + editor.getEditorState().read(() => { + const optbreak = $getRoot() + .getChildren() + .filter($isParaNode)[0] + .getChildren() + .find($isUnknownNode); + if (!optbreak) throw new Error("expected an optbreak UnknownNode"); + optbreakKey = optbreak.getKey(); + }); + await act(async () => + editor.update(() => { + const para = $getRoot().getChildren().filter($isParaNode)[0]; + const selection = $createRangeSelection(); + selection.anchor = $createPoint(para.getKey(), 2, "element"); // right before "before " + selection.focus = $createPoint(optbreakKey, 0, "element"); // touches the wrapper, covers none of its content + $setSelection(selection); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).not.toContain("//"); + expect(getData("application/x-lexical-editor")).not.toContain("optbreak"); + }); + + it("characterization only (not fixed — unreachable via this branch's own copy): a synthetic html-only foreign payload with an NBSP directly before `//` still settles to a real optbreak node, with the foreign source's own NBSP kept as data `~` rather than folded into a display space", async () => { + // $normalizePastedNbsp's marker-token regexes only match `\`-shaped tokens (AFTER_MARKER_NBSP / + // BEFORE_MARKER_NBSP, whitespaceDisplay.plugin.utils.ts) — `//` has no backslash, so neither + // pass recognizes an NBSP next to it, and the final blanket `.replaceAll(NBSP, "~")` converts + // it to a literal data tilde — the correct display form for genuine data-NBSP, just one + // position off from where a marker-adjacent rule would have folded it to a space instead. The + // optbreak TOKEN itself still tokenizes correctly on caret departure (the tokenizer's `//` + // split, usfmFragmentToUsj.ts, is a plain string `.split("//")`, unaffected by an adjacent + // `~`) — asserted below by settling and checking the resulting USJ, not just the pre-settle + // display text. This shape is reachable only via a foreign clipboard source supplying + // `text/html` with no `text/plain` at all — this branch's own copy never produces it (the two + // pins above). + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + const departurePara = $createParaNode("p"); + const departureText = $createTextNode("depart here"); + $getRoot().append( + para.append($createMarkerNode("p"), text), + departurePara.append($createMarkerNode("p"), departureText), + ); + }); + await act(async () => editor.update(() => text.select(0, 0))); + const { event } = pasteEvent({ "text/html": `

before${NBSP}// after

` }); + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(event); + }), + ); + expect(handled).toBe(true); + editor.getEditorState().read(() => { + // Pre-settle display text: the literal, un-tokenized bytes the paste inserted. + expect($getRoot().getTextContent()).toContain("before~// after"); + }); + // Depart to the second paragraph and settle — the same caret-departure requirement + // `optbreakUsj`'s doc comment (above) explains for every other pasted-literal-`//` pin here. + await act(async () => + editor.update(() => { + const secondPara = $getRoot().getChildren().filter($isParaNode)[1]; + const departureText = secondPara?.getLastChild(); + if (!departureText || !$isTextNode(departureText)) + throw new Error("expected the 'depart here' paragraph's text node"); + departureText.select(0, 0); + }), + ); + await settle(); + editor.getEditorState().read(() => { + const firstPara = $getRoot().getChildren().filter($isParaNode)[0]; + expect(firstPara.getChildren().some($isUnknownNode)).toBe(true); + expect(firstPara.getTextContent().replaceAll(NBSP, " ")).toBe("\\p before~// afterbody"); + }); + }); +}); + +describe("paste round trip: all three payload shapes carry a copied optbreak back", () => { + /** Harvests the genuine text/plain, text/html, and application/x-lexical-editor bytes this + * branch's own copy produces for `optbreakUsj()`'s paragraph CONTENT (marker excluded, see + * `$selectParaContent`'s doc comment) — the same `COPY_COMMAND` path `$handleCopyForStandardView` + * (`whitespaceDisplay.plugin.utils.ts`) runs for a real Ctrl+C. */ + async function copyOptbreakContentPayload(): Promise<{ [key: string]: string }> { + const { editor } = await renderUsjEditor(optbreakUsj()); + await act(async () => editor.update($selectParaContent)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + return { + "text/plain": getData("text/plain"), + "text/html": getData("text/html"), + "application/x-lexical-editor": getData("application/x-lexical-editor"), + }; + } + + /** A fresh target editor with the SAME book+chapter header and one EMPTY `\p` host paragraph — + * the paste insertion point, matching `clipboardCorpusRoundTrip.test.tsx`'s + * `chapterHeaderSkeletonUsj` pattern (own header, empty `\p` host). */ + async function freshEmptyHost(withHistory = false): Promise<{ editor: LexicalEditor }> { + return renderUsjEditor(optbreakUsj([]), withHistory); + } + + /** Places the caret at the empty host paragraph's own content start (right after its `\p` marker + * and trailing separator — the paragraph has nothing else) and dispatches `PASTE_COMMAND` with + * `payload` as the clipboard's full MIME map, settles, then DEPARTS the caret to the fixture's + * second paragraph and settles again. Dispatching the COMMAND (not calling + * `$handlePasteForStandardView` directly) is required for the lexical-flavor shape: only a real + * command dispatch reaches Lexical's own fallback `PASTE_COMMAND` handler + * (`@lexical/react/LexicalRichTextPlugin`, `COMMAND_PRIORITY_EDITOR`) once + * `$handlePasteForStandardView` declines. The departure step is required for the plain-text + * shapes (a pasted literal `//` pends until caret departure — see `optbreakUsj`'s doc comment); + * harmless for the lexical-flavor shape, which inserts an already-structural node tree with + * nothing left pending. */ + async function pasteAndSettle( + editor: LexicalEditor, + payload: { [key: string]: string }, + ): Promise { + await act(async () => + editor.update(() => { + $getRoot().getChildren().filter($isParaNode)[0]?.selectEnd(); + editor.dispatchCommand(PASTE_COMMAND, pasteEvent(payload).event); + }), + ); + await settle(); + await act(async () => + editor.update(() => { + const secondPara = $getRoot().getChildren().filter($isParaNode)[1]; + const departureText = secondPara?.getLastChild(); // the "depart here" content TextNode + if (!departureText || !$isTextNode(departureText)) + throw new Error("expected the 'depart here' paragraph's text node"); + departureText.select(0, 0); + }), + ); + await settle(); + } + + it("plain-only payload — TJ's already-working external-plaintext case; pinned so it stays working", async () => { + const payload = await copyOptbreakContentPayload(); + const { editor } = await freshEmptyHost(); + await pasteAndSettle(editor, { "text/plain": payload["text/plain"] }); + expect(usjOf(editor)).toEqual(optbreakUsj()); + }); + + it("plain+html payload, no lexical flavor (the real async Ctrl+V shape) — text/plain wins over the optbreak-dropping html (H1: already fixed on this branch)", async () => { + const payload = await copyOptbreakContentPayload(); + const { editor } = await freshEmptyHost(); + await pasteAndSettle(editor, { + "text/plain": payload["text/plain"], + "text/html": payload["text/html"], + }); + expect(usjOf(editor)).toEqual(optbreakUsj()); + }); + + it("plain+html+lexical payload (sync/native shape carrying the same-namespace flavor) — the lexical fast path reconstructs the optbreak node tree instead of stranding a loose decorator (H3 fix)", async () => { + const payload = await copyOptbreakContentPayload(); + expect(payload["application/x-lexical-editor"]).not.toBe(""); + const { editor } = await freshEmptyHost(); + await pasteAndSettle(editor, payload); + expect(usjOf(editor)).toEqual(optbreakUsj()); + }); + + it("undo after the lexical-flavor paste restores the pre-paste (empty host) USJ in one step", async () => { + const payload = await copyOptbreakContentPayload(); + const { editor } = await freshEmptyHost(true); // withHistory + // Captured live rather than reconstructed via `optbreakUsj([])`: an empty `content: []` array + // and an omitted `content` key both mean "no content" to the adaptor, but `toEqual` treats + // them as different objects — comparing against the editor's OWN pre-paste serialization + // sidesteps that immaterial shape difference entirely. + const beforePaste = usjOf(editor); + await pasteAndSettle(editor, payload); + expect(usjOf(editor)).toEqual(optbreakUsj()); + await act(async () => editor.dispatchCommand(UNDO_COMMAND, undefined)); + await settle(); + expect(usjOf(editor)).toEqual(beforePaste); + }); + + it("undo after a PLAIN-payload paste is ONE step: the departure settle rides along with the paste it completes", async () => { + // A plain payload takes the longer route to the same node tree: the literal `//` inserts via + // `selection.insertText` in the paste's own update, and only re-tokenizes into an optbreak + // node in the SEPARATE update the caret-departure step triggers (see `optbreakUsj`'s doc + // comment — a pasted `//` pends until departure, identically to a TYPED `//`). Two commits, + // but ONE undo: a settle is never its own history entry, so the entry it merges into is the + // paste's, and a single press lands on the true pre-paste state rather than stranding the + // user on a half-settled literal they never typed. Measured, not assumed — and it matches the + // lexical-flavor path above, so which carrier the clipboard happened to provide is invisible + // to undo. + const payload = await copyOptbreakContentPayload(); + const { editor } = await freshEmptyHost(true); // withHistory + const beforePaste = usjOf(editor); + await pasteAndSettle(editor, { "text/plain": payload["text/plain"] }); + expect(usjOf(editor)).toEqual(optbreakUsj()); + await act(async () => editor.dispatchCommand(UNDO_COMMAND, undefined)); + await settle(); + expect(usjOf(editor)).toEqual(beforePaste); + }); +}); + +describe("cut across an optbreak", () => { + it("cuts the optbreak (and its flanking text) out: the clipboard holds the same bytes a copy of the identical selection would, and no optbreak node remains", async () => { + // Two independent editors (rather than one editor dispatching COPY then CUT in sequence) — + // the same pattern `clipboardCopyFidelity.test.tsx`'s "cut = copy + removeText" describe uses, + // for the same reason: a second command dispatch on the SAME editor lets Lexical's own DOM- + // selection reconciliation collapse the just-set programmatic selection under jsdom before the + // handler runs, which would make this a test of that reconciliation quirk instead of cut/copy + // byte parity. + const { editor: copyEditor } = await renderUsjEditor(optbreakUsj()); + await act(async () => copyEditor.update($selectParaContent)); + const copyStub = copyEvent(); + await act(async () => copyEditor.dispatchCommand(COPY_COMMAND, copyStub.event)); + const copiedPlainText = copyStub.getData("text/plain"); + expect(copiedPlainText).toBe("before // after"); // sanity: the byte-equality check below is + // only meaningful if this is actually the optbreak's own bytes, not an accidental empty match + + const { editor } = await renderUsjEditor(optbreakUsj()); + await act(async () => editor.update($selectParaContent)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(CUT_COMMAND, event)); + expect(getData("text/plain")).toBe(copiedPlainText); + const usj = usjOf(editor); + const para = usj.content[2]; + if (typeof para === "string") throw new Error("paragraph corrupted into a bare string"); + expect(para.content ?? []).toEqual([]); + editor.getEditorState().read(() => { + const paraNode = $getRoot().getChildren().find($isParaNode); + if (!paraNode) throw new Error("expected a ParaNode"); + expect(paraNode.getChildren().some($isUnknownNode)).toBe(false); + }); + }); +}); diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.test.tsx b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.test.tsx index 7658901e0..a2800a5da 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.test.tsx @@ -81,6 +81,16 @@ function usjFromUsx(paraContent: string) { ); } +/** Like `usjFromUsx`, but for a paragraph marker OTHER than `\p` — needed for the own-marker- + * prefix dedup's paste-vs-typed scoping pins below, which must start from a NON-`\p` host to show + * the marker actually being retagged (a `\p`-into-`\p` case can't distinguish "retagged" from + * "left alone"). */ +function usjFromUsxPara(marker: string, paraContent: string) { + return usxStringToUsj( + `T${paraContent}`, + ); +} + /** Load `usj` into a fresh headless editor in standard view; returns the editor. */ function loadEditor(usj: ReturnType) { initializeSerialize(undefined, undefined); @@ -819,6 +829,92 @@ describe("$rebuildParas", () => { }); }); +describe("own-marker-prefix dedup: paste-only scoping", () => { + // $buildParaFragment's own-marker-prefix dedup (tier2Rebuild.utils.ts) recognizes a PASTE + // shape — a whole-paragraph copy's own glyph riding along with the pasted text landing at an + // existing paragraph's content start — and must NOT also apply when a user simply TYPES the + // same byte sequence there: retagging (or silently no-op-ing) an existing paragraph's marker on + // a typed keystroke is a product decision this module must not make on its own, and the engine's + // pre-existing, P9-parity behavior for that case is a normal split with an EMPTY predecessor + // paragraph (the marker-deletion merge/retag machinery, markerEditDeletion.utils.ts, handles the + // empty paragraph from there). `context` (module-level, above) never sets `pasteRebuildArmed`, so + // `$rebuildParas`/`$buildParaFragment` default to `isPasteRebuild: false` here — exactly the + // "typed input" condition. + it('typing "\\q1 " at the content start of an "\\s1" paragraph still splits with an empty predecessor — the marker is NOT silently retagged/deleted', () => { + const editor = loadEditor(usjFromUsxPara("s1", "God Make Da World")); + editor.update( + () => { + const para = $lastPara(); + const text = requireDefined( + para + .getChildren() + .filter($isTextNode) + .find((node) => node.getTextContent().includes("God")), + "text node containing 'God' not found", + ); + // simulate the user having typed "\q1 " at content start + text.setTextContent("\\q1 God Make Da World"); + expect($rebuildParas([para], context)).toBe(true); + }, + { discrete: true }, + ); + const usj = deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions); + const paras = (usj?.content ?? []).filter((c) => typeof c !== "string" && c.type === "para"); + expect(paras).toEqual([ + { type: "para", marker: "s1" }, + { type: "para", marker: "q1", content: ["God Make Da World"] }, + ]); + }); + + it('typing "\\p " at the content start of a "\\p" paragraph still splits with an empty predecessor — not an invisible no-op', () => { + const editor = loadEditor(usjFromUsxPara("p", "Alpha")); + editor.update( + () => { + const para = $lastPara(); + const text = requireDefined( + para + .getChildren() + .filter($isTextNode) + .find((node) => node.getTextContent().includes("Alpha")), + "text node containing 'Alpha' not found", + ); + text.setTextContent("\\p Alpha"); + expect($rebuildParas([para], context)).toBe(true); + }, + { discrete: true }, + ); + const usj = deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions); + const paras = (usj?.content ?? []).filter((c) => typeof c !== "string" && c.type === "para"); + expect(paras).toEqual([ + { type: "para", marker: "p" }, + { type: "para", marker: "p", content: ["Alpha"] }, + ]); + }); + + it('the SAME "\\p " retype dedups down to one paragraph when `pasteRebuildArmed` says this rebuild IS a paste — the flag is the only thing that changes the outcome', () => { + const editor = loadEditor(usjFromUsxPara("p", "Alpha")); + const pasteContext: Tier2Context = { ...context, pasteRebuildArmed: { current: true } }; + editor.update( + () => { + const para = $lastPara(); + const text = requireDefined( + para + .getChildren() + .filter($isTextNode) + .find((node) => node.getTextContent().includes("Alpha")), + "text node containing 'Alpha' not found", + ); + text.setTextContent("\\p Alpha"); + expect($rebuildParas([para], pasteContext)).toBe(true); + }, + { discrete: true }, + ); + const usj = deserializeSerializedEditorState(editor.getEditorState().toJSON(), viewOptions); + const paras = (usj?.content ?? []).filter((c) => typeof c !== "string" && c.type === "para"); + expect(paras).toEqual([{ type: "para", marker: "p", content: ["Alpha"] }]); + }); +}); + describe("unknown-para rebuild round-trip", () => { it("rebuilds a paragraph whose marker is unknown to the sheet (no more guard refusal)", () => { const editor = loadEditor( diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index ba4f2a9a4..d0fc4e332 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -11,6 +11,7 @@ */ import usjEditorAdaptor from "../adaptors/usj-editor.adaptor"; +import { isParaKindMarker } from "./markerEditTier1.utils"; import { UNTERMINATED_MARKER_TAIL } from "./markerName.pattern"; import { $serializeExpandedNoteContent, @@ -49,6 +50,7 @@ import { $isMilestoneNode, $isNoteNode, $isParaNode, + $isSynthesizedMarkerNode, $isUnknownNode, $isVerseNode, $isMarkerTrailingSeparator, @@ -87,6 +89,21 @@ export interface Tier2Context { viewOptions: ViewOptions; getMarker: MarkerLookup; logger?: LoggerBasic; + /** + * Armed by `$handlePasteForStandardView` (whitespaceDisplay.plugin.utils.ts), alongside + * `MarkerEditContext.splitExpected`, for the duration of one external paste's update — reset by + * the same per-commit update listener that resets `splitExpected`. `$rebuildParas` reads it to + * decide whether `$buildParaFragment`'s own-marker-prefix dedup (`$withoutRedundantOwnPrefix`) + * may run: that dedup is a PASTE-shape recognition ("a whole-paragraph copy's own glyph rides + * along with the pasted text"), not a general typed-retag rule — applying it unconditionally + * made typing a paragraph-kind marker literal at an existing paragraph's content start silently + * DELETE or no-op the paragraph's prior marker instead of the engine's existing (P9-parity) + * split-with-empty behavior, a product decision this task must not make for typed input. + * Optional so bare `Tier2Context` objects built directly (tests exercising the tokenizer/rebuild + * machinery without the full marker-edit engine) default to "not a paste rebuild" — the + * pre-existing, dedup-free behavior. + */ + pasteRebuildArmed?: { current: boolean }; } /** @@ -1014,6 +1031,64 @@ function $appendNodesFragment( } } +/** A paragraph-kind marker literal at the very start of a text run — the same terminated-marker + * shape `TERMINATED_MARKER_IN_TEXT_REGEX` (markerEditTier2Trigger.utils.ts) recognizes anywhere + * in a run, anchored here to the run's first character. */ +const LEADING_MARKER_LITERAL = /^\\\+?([\w-]+)(?:\*|[ \u00A0])/; + +/** + * "Own marker wins": when `para` already carries its own visible marker prefix (a real glyph, not + * the prefix-less shape a freshly split paragraph starts in) and the fragment text right after + * that glyph and its one-character separator is ITSELF a paragraph-kind marker literal — the + * shape a whole-paragraph copy (which includes the source paragraph's own `\p ` glyph, see + * `$selectionToUsfmText`) produces when pasted at an existing paragraph's content start — the + * host's now-redundant glyph+separator is dropped from `out` so the pasted/typed literal supplies + * the paragraph's only marker occurrence. Left as `out` unchanged (a no-op) whenever the check + * doesn't hold, including a paragraph still prefix-less (a fresh multi-line-paste split, which + * settles through a different route before any prefix is ever injected onto it — see + * `$paraMarkerDeletionTransform`, markerEditDeletion.utils.ts) or one with real content between + * its own glyph and any embedded marker. ONLY called for a paste-triggered rebuild — see + * `$buildParaFragment`'s `isPasteRebuild` parameter; typing the same shape keeps the engine's + * existing split-with-empty behavior, which this function never runs for. + * + * The embedded literal's marker-kind check uses `isParaKindMarker` (markerEditTier1.utils.ts) — + * stylesheet-first, UNKNOWN-AS-PARAGRAPH, the same classification `$buildParaFragment`'s own guard + * below applies to the paragraph's own marker — rather than a bare `type === MarkerType.Paragraph` + * comparison, which rejected every unknown/custom.sty marker and left the stray-empty-paragraph + * bug reachable for any of them (e.g. a pasted `\zz one two`, unrecognized by the bundled sheet). + * + * Operates on the ALREADY-BUILT fragment (offsets into `out.text`/`out.spans`) rather than + * pre-filtering which child nodes contribute, because the glyph's one-character trailing + * separator and the paragraph's first real content do not reliably land in separate nodes — a + * non-token-mode separator lets Lexical's own text-splice absorb an insertion landing at its + * boundary into the SAME node, so "the paragraph's content" is not always a cleanly, separately + * indexable child. Slicing the built text instead works identically either way: the glyph's own + * span (`out.spans[0]`, always the first contribution when a prefix is present) locates the + * boundary regardless of which node(s) the separator and content bytes actually live in. + */ +function $withoutRedundantOwnPrefix( + out: FragmentAccumulator, + para: ParaNode, + getMarkerFn: MarkerLookup, +): FragmentAccumulator { + if (!$isSynthesizedMarkerNode(para.getFirstChild()) || out.spans.length === 0) return out; + const sliceAt = out.spans[0].end + 1; // the glyph, plus its one-character separator + const rest = out.text.slice(sliceAt); + const match = LEADING_MARKER_LITERAL.exec(rest); + if (!match || !isParaKindMarker(match[1], getMarkerFn)) return out; + return { + text: rest, + spans: out.spans + .filter((span) => span.end > sliceAt) // drop spans wholly inside the stripped prefix + .map((span) => ({ + ...span, + start: Math.max(0, span.start - sliceAt), + end: span.end - sliceAt, + })), + sentinels: out.sentinels, + }; +} + /** * Exported for two callers outside this module's own rebuild path. The read-only settle * (virtualSettle.utils.ts) builds the SAME fragment a mutating rebuild would, which is what makes @@ -1022,11 +1097,20 @@ function $appendNodesFragment( * against its hand-built wrapped-shape equivalent for byte-for-byte equality * (`tier2Rebuild.utils.test.tsx`) — the direct evidence that wrapping a run changes nothing about * what gets tokenized. + * + * `isPasteRebuild` gates `$withoutRedundantOwnPrefix` (default `false`, matching every direct test + * call site, every TYPED-input rebuild, and the read-only virtual settle): the own-marker-prefix + * dedup recognizes a PASTE shape (a whole-paragraph copy's own glyph riding along with the pasted + * text) and must not also apply to a user typing the same byte sequence — see that function's doc + * comment for why. `$rebuildParas` is the only caller that ever passes `true`, and only when + * `Tier2Context.pasteRebuildArmed` says the triggering rebuild is inside an external paste's own + * update. */ export function $buildParaFragment( para: ParaNode, getMarkerFn: MarkerLookup, viewOptions: ViewOptions | undefined, + isPasteRebuild = false, ): FragmentAccumulator | undefined { // Guard rails (preserve-or-refuse): a paragraph the engine cannot fully // re-derive from its text is never rebuilt — edits inside it stay literal text. @@ -1046,7 +1130,7 @@ export function $buildParaFragment( if ($isUnknownNode(parent)) return undefined; const out: FragmentAccumulator = { text: "", spans: [], sentinels: [] }; $appendChildrenFragment(para, out, getMarkerFn, viewOptions); - return out; + return isPasteRebuild ? $withoutRedundantOwnPrefix(out, para, getMarkerFn) : out; } /** Replace each U+FFFC in the rebuilt tree with the next preserved node run. */ @@ -1495,11 +1579,12 @@ function $restoreSelectionInNoteContent( */ export function $rebuildParas(paras: ParaNode[], context: Tier2Context): boolean { if (paras.length === 0) return false; - const { viewOptions, getMarker: getMarkerFn, logger } = context; + const { viewOptions, getMarker: getMarkerFn, logger, pasteRebuildArmed } = context; + const isPasteRebuild = pasteRebuildArmed?.current ?? false; const combined: FragmentAccumulator = { text: "", spans: [], sentinels: [] }; for (const para of paras) { - const fragment = $buildParaFragment(para, getMarkerFn, viewOptions); + const fragment = $buildParaFragment(para, getMarkerFn, viewOptions, isPasteRebuild); if (!fragment) { logger?.debug("[MarkerEdit] Tier 2 skipped: paragraph excluded by guard rails"); return false; diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index d88e3d021..89d5d101b 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -1,5 +1,8 @@ import { MarkerEditPlugin } from "./MarkerEditPlugin"; import { + copyEvent, + findOnlyNote, + pasteEvent, serializedState, testEnvironment, testEnvironmentWithDisplaySyncs, @@ -17,7 +20,7 @@ import { LexicalClipboardData } from "@lexical/clipboard"; // Reaching inside only for tests. // eslint-disable-next-line @nx/enforce-module-boundaries import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; -import { usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; +import { Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; import { $dfs } from "@lexical/utils"; import { $createPoint, @@ -37,11 +40,15 @@ import { $createCharNode, $createMarkerNode, $createParaNode, + $isCharNode, + $isMarkerNode, $isParaNode, NBSP, + NoteNode, ParaNode, textTypeState, } from "shared"; +import { StructureKeyboardPlugin } from "shared-react"; /** * Null-event leg: ClipboardPlugin/ContextMenuPlugin/EditorRef dispatch COPY_COMMAND/ @@ -70,25 +77,6 @@ vi.mock("@lexical/clipboard", async (importOriginal) => { return { ...actual, copyToClipboard: copyToClipboardSpy }; }); -/** - * jsdom (see StructureKeyboardPlugin.test.tsx's `htmlPasteEvent`) doesn't implement - * `ClipboardEvent`/`DataTransfer`; the handler under test only touches - * `clipboardData.setData`/`preventDefault`, so a minimal stub covers it. - */ -function copyEvent(): { event: ClipboardEvent; getData: (type: string) => string } { - const store = new Map(); - const clipboardData = { - getData: (type: string) => store.get(type) ?? "", - setData: (type: string, data: string) => { - store.set(type, data); - }, - }; - return { - event: { clipboardData, preventDefault: vi.fn() } as unknown as ClipboardEvent, - getData: (type: string) => clipboardData.getData(type), - }; -} - /** * Builds `

` + a marker-trailing-space NBSP + `text` as siblings. The trailing-space node's * `textType` state (matching the real adaptor's `createText(NBSP, "marker-trailing-space")`) @@ -617,22 +605,47 @@ describe("clipboard normalization — null-event leg (ClipboardPlugin/ContextMen }); }); -describe("paste normalization ($handlePasteForStandardView)", () => { - function pasteEvent(payload: { [key: string]: string }): { - event: ClipboardEvent; - prevented: () => boolean; - } { - let prevented = false; - const clipboardData = { getData: (type: string) => payload[type] ?? "" }; - const event = { - clipboardData, - preventDefault: () => { - prevented = true; - }, - } as unknown as ClipboardEvent; - return { event, prevented: () => prevented }; - } +/** + * Pastes `payload` at the current selection, settles Tier 2's structural rebuild, then selects + * the whole document and copies it back out, returning the round-tripped `text/plain`. A raw + * literal marker pair like `\nd`…`\nd*` only becomes its canonical node shape once Tier 2 + * rebuilds it — and that rebuild re-derives its OWN structural separator byte regardless of + * whether this handler inserted a plain space or left an NBSP, so asserting on raw post-paste + * text content pins Tier 2's rebuild timing, not this handler's contract. Comparing USFM strings + * before and after a full round trip is what the live repro and this task actually care about: + * whether the marker was recognized and the data survived — not which byte an about-to-be- + * rebuilt separator held for one commit. `$selectionToUsfmText` (the copy leg) inverts every + * STRUCTURAL NBSP back to a plain space regardless of which one Tier 2 chose, so the round trip + * is meaningful however Tier 2 shapes the interim tree. + */ +async function pasteAndCopyBack( + editor: LexicalEditor, + payload: { [key: string]: string }, +): Promise { + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(pasteEvent(payload).event); + }), + ); + expect(handled).toBe(true); + // Settle Tier 2 the same way neighboring suites do (a double microtask flush). + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + await act(async () => + editor.update(() => { + const root = $getRoot(); + root.select(0, root.getChildrenSize()); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + return getData("text/plain"); +} +describe("paste normalization ($handlePasteForStandardView)", () => { it("rewrites a pasted data-NBSP to the `~` display form (data round-trips to a real NBSP)", async () => { let text: TextNode; const { editor } = await testEnvironment(() => { @@ -660,7 +673,7 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); }); - it("passes through internal pastes (lexical payload) and NBSP-free plain text", async () => { + it("declines internal pastes (a same-namespace `application/x-lexical-editor` payload is present)", async () => { let text: TextNode; const { editor } = await testEnvironment(() => { const para = $createParaNode("p"); @@ -673,15 +686,41 @@ describe("paste normalization ($handlePasteForStandardView)", () => { "application/x-lexical-editor": "{}", "text/plain": `x${NBSP}y`, }); - const plain = pasteEvent({ "text/plain": "no nbsp here" }); + let handled = true; await act(async () => editor.update(() => { - expect($handlePasteForStandardView(internal.event)).toBe(false); - expect($handlePasteForStandardView(plain.event)).toBe(false); + handled = $handlePasteForStandardView(internal.event); }), ); + expect(handled).toBe(false); expect(internal.prevented()).toBe(false); - expect(plain.prevented()).toBe(false); + }); + + it("claims an external plain-text paste with no NBSP at all, inserting it unchanged", async () => { + // Previously this handler only claimed NBSP-bearing pastes; every OTHER external paste fell + // through to Lexical's own HTML/plain-text handling. It now claims every non-lexical paste — + // Standard view has no fidelity carrier but plain text, so an NBSP-free paste re-tokenizes + // the same way an NBSP-bearing one does. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const { event, prevented } = pasteEvent({ "text/plain": "no nbsp here" }); + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(event); + }), + ); + expect(handled).toBe(true); + expect(prevented()).toBe(true); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).toContain("no nbsp here"); + }); }); it("rewrites an NBSP found only in a `text/html` payload (word-processor ` `) to `~`, with no `text/plain` data", async () => { @@ -720,19 +759,21 @@ describe("paste normalization ($handlePasteForStandardView)", () => { text = $createTextNode("before after"); $getRoot().append(para.append($createMarkerNode("p"), sep, text)); }); - await act(async () => editor.update(() => text.select(7, 7))); // between "before " and "after" - const { event } = pasteEvent({ - "text/html": "

one two

three
four

", - }); - let handled = false; + // Dispatched through PASTE_COMMAND (not a direct handler call): a multi-line insert only + // gets its fresh paragraphs correctly prefixed when it goes through MarkerEditPlugin's own + // registration, which arms `context.splitExpected` before inserting — see "multi-line paste + // interplay" below. await act(async () => editor.update(() => { - handled = $handlePasteForStandardView(event); + text.select(7, 7); // between "before " and "after" + editor.dispatchCommand( + PASTE_COMMAND, + pasteEvent({ "text/html": "

one two

three
four

" }).event, + ); }), ); - expect(handled).toBe(true); editor.getEditorState().read(() => { // Block boundaries and
become newlines, so "two"/"three" don't fuse into one word — // and each newline is then replayed as a real paragraph split (see the line-replay @@ -803,7 +844,10 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); }); - it("falls through to default html handling when `text/html` carries no NBSP (raw or decoded)", async () => { + it("claims Word-style external HTML, inserting plain text with no formatting nodes", async () => { + // Previously this fell through to Lexical's own HTML import whenever text/html carried no + // NBSP, which would have created a bold-formatted TextNode from the ``. It now claims + // every non-lexical paste and never imports the html at all. let text: TextNode; const { editor } = await testEnvironment(() => { const para = $createParaNode("p"); @@ -812,16 +856,28 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); await act(async () => editor.update(() => text.select(0, 0))); - const { event, prevented } = pasteEvent({ "text/html": "

bold text

" }); - let handled = true; + const { event, prevented } = pasteEvent({ + "text/html": "

bold text

", + "text/plain": "bold text", + }); + let handled = false; await act(async () => editor.update(() => { handled = $handlePasteForStandardView(event); }), ); - expect(handled).toBe(false); - expect(prevented()).toBe(false); + expect(handled).toBe(true); + expect(prevented()).toBe(true); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).toContain("bold text"); + // No formatting survived the plain-text carrier: every text node is unformatted, unlike a + // real Lexical HTML import of ``, which would set the bold flag on a new TextNode. + const textNodes = $dfs($getRoot()) + .map(({ node }) => node) + .filter($isTextNode); + textNodes.forEach((node) => expect(node.getFormat()).toBe(0)); + }); }); it("keeps current behavior (declines) when an `application/x-lexical-editor` payload is present, even if `text/html` carries NBSP", async () => { @@ -847,6 +903,493 @@ describe("paste normalization ($handlePasteForStandardView)", () => { expect(handled).toBe(false); expect(prevented()).toBe(false); }); + + it("does not double a Standard-view-shaped `` glyph — the paste never reaches Lexical's HTML import", async () => { + // Shaped like Standard view's own exported markup (a data-marker span, marker glyph as real + // text) — the shape that would, if this ever reached Lexical's `$generateNodesFromDOM` + // import, produce a structural CharNode from the span AND a second, literal "\nd" from the + // marker glyph's own text (MarkerNode has no importDOM). Both text/html and text/plain carry + // the same bytes, matching a real Standard-view copy. Round-tripped through copy (rather than + // read as raw post-paste text) so an exact-equality check is meaningful: a doubled glyph + // would show up as extra "\nd"/"\nd*" occurrences in the copied USFM. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const source = `\\nd${NBSP}light\\nd*`; + const roundTripped = await pasteAndCopyBack(editor, { + "text/html": `${source}`, + "text/plain": source, + }); + + expect(roundTripped).toContain("\\nd light\\nd*body"); + // "\nd" appears exactly twice: the opener itself, and the "\nd" prefix of "\nd*". A doubled + // glyph (the HTML-import path this handler must never reach) would show four. + expect(roundTripped.split("\\nd").length - 1).toBe(2); + // Round-tripped bytes alone can't distinguish "tokenized into a real CharNode" from "survived + // as literal, never-recognized text" (both would copy back the same bytes) — so check the + // node shape directly: Tier 2 (settled inside `pasteAndCopyBack`) must have produced exactly + // one real `nd` CharNode, not left the pasted span as literal text. + editor.getEditorState().read(() => { + const chars = $dfs($getRoot()) + .map(({ node }) => node) + .filter($isCharNode); + expect(chars.filter((char) => char.getMarker() === "nd")).toHaveLength(1); + }); + }); + + describe("positional NBSP normalization", () => { + // These replace the old blanket NBSP→`~` pins above (the ones with no marker in the pasted + // text stay unaffected — every existing NBSP-preservation pin above still holds byte-for-byte + // under the positional rule; there was simply no marker adjacency in them to normalize). + it("an NBSP immediately after a marker token is treated as a display artifact, not corrupted into `~` (the 2026-08-07 live-repro shape, fixed)", async () => { + // Live repro: a same-editor paste of a copied footnote turned every display-NBSP into a + // literal `~`, corrupting `\f`/`\fr`/`\ft` into unknown-marker soup (see the round-trip + // regression below for the full footnote). This is the minimal reproduction of one such + // position — the required separator right after an opening glyph. Round-tripped through + // copy (see `pasteAndCopyBack`'s doc comment): once Tier 2 recognizes the full `\nd`…`\nd*` + // pair it rebuilds its own canonical structural separator regardless of which byte this + // handler inserted, so the meaningful check is that the marker was recognized at all + // (round-trips clean) rather than which byte survives for one commit pre-rebuild. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/plain": `\\nd${NBSP}light\\nd*`, + }); + + expect(roundTripped).toContain("\\nd light\\nd*"); + expect(roundTripped).not.toContain("~"); + }); + + it("a double NBSP after a marker token: only the first is marker-adjacent — the second is interior data (`\\nd ~light`)", async () => { + // Only ONE NBSP per marker adjacency is treated as a display artifact — a SECOND, + // unexplained NBSP right after it is preserved as data rather than silently dropped, since + // losing user data is worse than the reverse. `\nd` + NBSP + NBSP + `light`: the + // after-marker pass matches `\nd` + the FIRST NBSP only (consuming that pair), so the + // second NBSP is left with nothing recognized immediately before or after it and falls + // through to the interior/data pass. Round-tripped through copy (see `pasteAndCopyBack`'s + // doc comment): `\nd` alone (no closer) is still a real opener Tier 1 recognizes and + // re-derives its own canonical separator for, discarding whichever byte this handler + // inserted there — the same interim-shape caveat the closed-pair positional tests above + // sidestep the same way. The SECOND NBSP is untouched by any such rebuild (it is already a + // literal `~` character by the time Tier 1/2 sees it), so it survives the round trip as-is. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/plain": `\\nd${NBSP}${NBSP}light`, + }); + + expect(roundTripped).toContain("\\nd ~light"); + }); + + it("an NBSP with no adjacent marker token is genuine data and stays `~`", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const { event } = pasteEvent({ "text/plain": `word${NBSP}word` }); + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(event); + }), + ); + + expect(handled).toBe(true); + editor.getEditorState().read(() => { + expect($getRoot().getTextContent()).toContain("word~word"); + }); + }); + + it("an NBSP sitting before a closing marker (not following one) is dropped — a structural spacer with no source counterpart", async () => { + // Browser-hop shape: `\nd` + NBSP + `Lord` + NBSP + `\nd*`. The first NBSP follows the + // opening marker (display artifact → space). The second precedes the closer instead of + // following one: `createNote` (usj-editor.adaptor.ts) proves a note-level spacer sits in + // exactly this position (before `\ft`/`\f*`, not just after an opener), so this is a + // structural artifact too — but with no source USFM byte to become (`\nd Lord\nd*` needs no + // space before its closer at all) — so it is DROPPED entirely, not spaced and not kept as + // `~`. Round-tripped through copy (see `pasteAndCopyBack`) to sidestep asserting on Tier 2's + // pre-rebuild interim text. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/plain": `\\nd${NBSP}Lord${NBSP}\\nd*`, + }); + + expect(roundTripped).toContain("\\nd Lord\\nd*"); + expect(roundTripped).not.toContain("~"); + }); + + it("normalizes a leading NBSP with no preceding marker literal (a partial selection starting at a char span's structural separator)", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const { event } = pasteEvent({ "text/plain": `${NBSP}Lord` }); + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(event); + }), + ); + + expect(handled).toBe(true); + editor.getEditorState().read(() => { + const content = $getRoot().getTextContent(); + expect(content).toContain(" Lord"); + expect(content).not.toContain("~Lord"); + }); + }); + + it("html-only payload (no text/plain): text derived via `htmlPasteText` is positionally normalized the same way", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/html": `

\\nd${NBSP}light\\nd*

`, + }); + + expect(roundTripped).toContain("\\nd light\\nd*"); + }); + + it("recognizes a nested char marker (`\\+nd`) as a marker token — the NBSP after it does not gain `~`", async () => { + // Round-tripped through copy (see `pasteAndCopyBack`'s doc comment): `\wj`/`\+nd` are real + // recognized char markers, so Tier 2 rebuilds both the outer and nested span, re-deriving + // its own canonical structural leading separator for each (discarding whatever byte this + // handler inserted) — the same interim-shape caveat the simple single-marker positional + // tests above sidestep the same way. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/plain": `\\wj li\\+nd${NBSP}g\\+nd*ht\\wj*`, + }); + + expect(roundTripped).toContain("\\wj li\\+nd g\\+nd*ht\\wj*"); + expect(roundTripped).not.toContain("~"); + }); + + it("recognizes a milestone's anonymous self-closer (`\\*`) as a marker token — the NBSP after it does not gain `~`", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const { event } = pasteEvent({ "text/plain": `\\qt-s\\*${NBSP}after` }); + let handled = false; + await act(async () => + editor.update(() => { + handled = $handlePasteForStandardView(event); + }), + ); + + expect(handled).toBe(true); + editor.getEditorState().read(() => { + const content = $getRoot().getTextContent(); + expect(content).toContain("\\qt-s\\* after"); + expect(content).not.toContain("~"); + }); + }); + }); + + describe("multi-line paste interplay (splitExpected arming)", () => { + it("an NBSP-bearing multi-line paste both splits into prefixed paragraphs AND normalizes NBSPs positionally", async () => { + // Latent bug this closes: the old NBSP-only gate called `selection.insertText` on the + // WHOLE multi-line string with no paragraph-splitting logic at all, so a pasted "\n" + // landed as a literal character inside one text run instead of a paragraph break — an + // NBSP-bearing multi-line paste was never actually split. Dispatched via PASTE_COMMAND (not + // a direct handler call) so MarkerEditPlugin's own registration arms + // `context.splitExpected` before inserting; a direct call defaults that callback to a + // no-op and the freshly split paragraphs would be merged back as "marker deleted". + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + const sep = $createTextNode(NBSP); + $setState(sep, textTypeState, "marker-trailing-space"); + text = $createTextNode("before after"); + $getRoot().append(para.append($createMarkerNode("p"), sep, text)); + }); + + await act(async () => + editor.update(() => { + text.select(7, 7); // between "before " and "after" + editor.dispatchCommand( + PASTE_COMMAND, + pasteEvent({ "text/plain": `one${NBSP}two\nthree${NBSP}four` }).event, + ); + }), + ); + + editor.getEditorState().read(() => { + const paras = $getRoot().getChildren().filter($isParaNode); + expect(paras).toHaveLength(2); + expect(paras.map((p) => p.getMarker())).toEqual(["p", "p"]); + paras.forEach((p) => expect($isMarkerNode(p.getFirstChild())).toBe(true)); + // Neither NBSP is marker-adjacent (no `\marker` anywhere in this paste), so both stay + // data (`~`) across the paragraph split. + expect(paras[0].getTextContent()).toContain("before one~two"); + expect(paras[1].getTextContent()).toContain("three~four"); + }); + }); + + it("a leading NBSP on a SECOND line (right after an internal `\\n`, not string-start) also normalizes to a space, under the `gm`-flagged leading-NBSP pass", async () => { + // `$normalizePastedNbsp`'s leading-NBSP pass uses the `gm` flags, so `^` matches after + // every `\n`, not just at the very start of the whole paste — a later paragraph of a + // multi-line paste can itself start mid-span (a partial selection spanning a paragraph + // boundary) and reads as the same structural separator the string-start case does. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + const sep = $createTextNode(NBSP); + $setState(sep, textTypeState, "marker-trailing-space"); + text = $createTextNode("before after"); + $getRoot().append(para.append($createMarkerNode("p"), sep, text)); + }); + + await act(async () => + editor.update(() => { + text.select(7, 7); // between "before " and "after" + editor.dispatchCommand( + PASTE_COMMAND, + pasteEvent({ "text/plain": `one\n${NBSP}two` }).event, + ); + }), + ); + + editor.getEditorState().read(() => { + const paras = $getRoot().getChildren().filter($isParaNode); + expect(paras).toHaveLength(2); + expect(paras[0].getTextContent()).toContain("before one"); + // The second line's leading NBSP became a plain space, not data — no `~` anywhere. + expect(paras[1].getTextContent()).toContain(" twoafter"); + expect(paras[1].getTextContent()).not.toContain("~"); + }); + }); + + it("normalizes a bare `\\r` (no `\\n`) to a paragraph break instead of inserting a control character", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + + await act(async () => + editor.update(() => { + text.select(0, 0); + editor.dispatchCommand( + PASTE_COMMAND, + pasteEvent({ "text/plain": "first\rsecond" }).event, + ); + }), + ); + + editor.getEditorState().read(() => { + const paras = $getRoot().getChildren().filter($isParaNode); + expect(paras).toHaveLength(2); + expect(paras[0].getTextContent()).toContain("first"); + expect(paras[1].getTextContent()).toContain("second"); + expect($getRoot().getTextContent()).not.toContain("\r"); + }); + }); + }); + + describe("structure protection", () => { + async function protectedTestEnvironment($initialEditorState: () => void) { + return baseTestEnvironment( + $initialEditorState, + <> + + + , + ); + } + + it("declines when the document is structure-protected, letting StructureKeyboardPlugin's sanitizer govern the paste", async () => { + // Both plugins register PASTE_COMMAND at COMMAND_PRIORITY_HIGH, and MarkerEditPlugin mounts + // first (matching Editor.tsx's real order) — so without an explicit decline here, this + // handler would claim the paste before StructureKeyboardPlugin's sanitizer ever runs. + let t1: TextNode; + const { editor } = await protectedTestEnvironment(() => { + const sep = $createTextNode(NBSP); + $setState(sep, textTypeState, "marker-trailing-space"); + t1 = $createTextNode("hello world"); + $getRoot().append($createParaNode("p").append($createMarkerNode("p"), sep, t1)); + }); + await act(async () => editor.update(() => t1.select(5, 5))); + + // Same verse-marker HTML shape StructureKeyboardPlugin.test.tsx pins against alone. + const verseHtml = + '

' + + '2pasted

'; + const { event } = pasteEvent({ "text/html": verseHtml }); + + await act(async () => + editor.update(() => { + editor.dispatchCommand(PASTE_COMMAND, event); + }), + ); + + editor.getEditorState().read(() => { + // If `$handlePasteForStandardView` had wrongly claimed this instead of declining, it + // would extract the verse span's own visible "2" as literal text via `htmlPasteText` and + // insert "2pasted" — StructureKeyboardPlugin's sanitizer instead strips the verse node + // entirely, so no such digit leak survives. + const content = $getRoot().getTextContent(); + expect(content).not.toContain("2pasted"); + expect(content).toContain("pasted"); + }); + }); + }); + + describe("tilde-corruption regression (2026-08-07 live repro)", () => { + /** The exact live-repro footnote: `\f - \fr 1:1 \ft Caller test.\f*`. */ + function footnoteUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: [ + { + type: "note", + marker: "f", + caller: "-", + content: [ + { type: "char", marker: "fr", content: ["1:1 "], closed: "false" }, + { type: "char", marker: "ft", content: ["Caller test."], closed: "false" }, + ], + }, + ], + }, + ], + } as unknown as Usj; + } + + it("a same-editor paste of a just-copied footnote round-trips byte-exact — no NBSP is corrupted into `~`", async () => { + // `pasteSelection` (clipboard.utils.ts) rebuilds its `DataTransfer` from Chromium's async + // `navigator.clipboard.read()`, which exposes only standard MIME types — the private + // `application/x-lexical-editor` flavor written on copy never comes back on a real Ctrl+V, + // so even a same-editor paste rides text/html and text/plain like any external source. This + // reproduces exactly that: copy captures all three flavors below, but the paste payload + // deliberately omits the lexical one, matching what a real Ctrl+V actually delivers. + const { editor: sourceEditor } = await baseTestEnvironment( + serializedState(footnoteUsj()), + , + ); + let note: NoteNode | undefined; + sourceEditor.getEditorState().read(() => { + note = findOnlyNote($getRoot()); + }); + if (!note) throw new Error("expected exactly one note"); + const noteNode = note; + await act(async () => + sourceEditor.update(() => { + const first = noteNode.getFirstDescendant(); + const last = noteNode.getLastDescendant(); + if (!first || !last) throw new Error("note has no descendants"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(first.getKey(), 0, "text"); + selection.focus = $createPoint(last.getKey(), last.getTextContentSize(), "text"); + $setSelection(selection); + }), + ); + const copyStub = copyEvent(); + await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, copyStub.event)); + const sourceText = copyStub.getData("text/plain"); + // Ground truth: no NBSP at all in a correct copy (matches + // clipboardCopyFidelity.test.tsx's pin for this exact live-repro footnote). + expect(sourceText).toBe("\\f - \\fr 1:1 \\ft Caller test.\\f*"); + + let text: TextNode; + const { editor: targetEditor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => targetEditor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(targetEditor, { + "text/plain": sourceText, + "text/html": copyStub.getData("text/html"), + }); + + expect(roundTripped).not.toContain("~"); + expect(roundTripped).toContain("\\f - \\fr 1:1 \\ft Caller test.\\f*"); + }); + + it("a browser-hop/html-derived collapsed-footnote shape (structural NBSPs before `\\ft` and `\\f*`, not only after `\\f`/`\\fr`) round-trips clean", async () => { + // `createNote` (usj-editor.adaptor.ts) appends a spacer NBSP after EVERY child, not just + // the first — confirmed by inspecting this exact fixture's real `$getHtmlContent` export: + // `…\fr 1:1
 \ft…` + // (a bare ` ` span sits between `\fr`'s content and `\ft`, and another one between + // `\ft`'s content and `\f*`). So a browser-hop paste of this shape carries structural NBSPs + // on BOTH sides of its interior markers, not only after an opener. Hand-built (rather than + // a live html round-trip) so this pins the positional-normalization rule specifically — + // `ImmutableNoteCallerNode`'s own DOM export carries the caller only as a `data-caller` + // attribute, never as visible text, so a REAL html-only round-trip of this fixture loses + // the caller entirely, an unrelated, pre-existing gap this test isn't about. + let text: TextNode; + const { editor } = await testEnvironment(() => { + const para = $createParaNode("p"); + text = $createTextNode("body"); + $getRoot().append(para.append($createMarkerNode("p"), text)); + }); + await act(async () => editor.update(() => text.select(0, 0))); + + const roundTripped = await pasteAndCopyBack(editor, { + "text/plain": `\\f - \\fr 1:1 ${NBSP}\\ft Caller test.${NBSP}\\f*`, + }); + + expect(roundTripped).not.toContain("~"); + expect(roundTripped).toContain("\\f - \\fr 1:1 \\ft Caller test.\\f*"); + }); + }); }); /** diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 67ae6a1d8..4f4e41b9a 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -19,24 +19,35 @@ import { LexicalClipboardData, } from "@lexical/clipboard"; import { + $getCharacterOffsets, $getEditor, $getSelection, $getState, + $isDecoratorNode, + $isElementNode, + $isLineBreakNode, $isRangeSelection, + $isTextNode, INSERT_PARAGRAPH_COMMAND, LexicalEditor, + LexicalNode, PasteCommandType, + RangeSelection, TextNode, } from "lexical"; import { + $isAttributeRunNode, $isBookNode, $isChapterNode, $isCharNode, + $isNoteNode, $isUnknownNode, + GENERATOR_NOTE_CALLER, MARKER_TRAILING_SPACE_TEXT_TYPE, NBSP, textTypeState, } from "shared"; +import { $isImmutableNoteCallerNode } from "shared-react"; /** Spaces in runs display as NBSP so they are visible while typing. */ export function $displayWhitespaceTransform(node: TextNode): void { @@ -151,58 +162,285 @@ export function getPastePayload( } /** - * Standard-view PASTE normalization: a pasted data-NBSP must appear on screen as `~` (the - * display form; serialization inverts `~` back to a real NBSP, so the DATA stays an NBSP). - * Without this, a pasted NBSP landed as a raw NBSP — indistinguishable from a display-NBSP - * (which represents a plain space inside a run) — so nothing showed on screen live, and - * serialization then corrupted it into a plain space; the `~` only appeared after an app - * reload re-ran the load-time mapping. Internal pastes (application/x-lexical-editor payload) - * are already in display form and pass through untouched. For the rare NBSP-bearing external - * paste this inserts the normalized PLAIN text (foreign `text/html` formatting is dropped — - * preserving the NBSP data beats preserving formatting the sanitizer would mostly strip - * anyway). The same NBSP-bearing check also covers `text/html` (word-processor copies carry the - * space as a literal ` `): some sources omit `text/plain` entirely, or their browser- - * generated `text/plain` has already collapsed ` ` to a plain space, losing the marker - * before it ever reaches this handler — so `text/html`, and the pasted text it decodes to - * (`htmlPasteText` above), are checked too, falling back to that decoded text when it's the - * only place the NBSP survives. + * A marker token this handler recognizes for positional NBSP normalization: a plain or + * nested-char marker (`\nd`, `\+nd`), either one's closer (`\nd*`, `\+nd*`), or a milestone's + * anonymous self-closer (`\*`). + */ +const AFTER_MARKER_NBSP = /(\\(?:\+?[a-z0-9-]+\*?|\*))\u00A0/gi; +const BEFORE_MARKER_NBSP = /\u00A0(?=\\(?:\+?[a-z0-9-]+\*?|\*))/gi; + +/** + * Positional NBSP normalization for an external paste's resolved text. Standard view has no + * `text/html` fidelity carrier for foreign sources (`$handlePasteForStandardView` below drops + * formatting entirely and re-tokenizes the plain text), so a `text/html` payload's NBSPs are the + * only clue to which spaces were meaningful markup vs. plain content — and the browser's own + * clipboard round-trip (and a same-editor paste, whose private Lexical flavor does not survive + * `navigator.clipboard.read()` — see `$handlePasteForStandardView`'s doc comment) both carry a + * DISPLAY-NBSP (a Standard-view run space, a marker's own trailing separator, or a note's + * internal spacer — `createNote` in `usj-editor.adaptor.ts` appends one after EVERY child, not + * just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller) + * as a real NBSP, indistinguishable at this point from a genuine data-NBSP (PT9's `~` glyph). + * + * The two are told apart POSITIONALLY, mirroring PT9's `PostprocessUsfm`, in three passes: + * + * 1. A leading NBSP — at the very start of the text, or right after a newline (a later paragraph + * of a multi-line paste can itself start mid-span) — reads as a structural separator with + * nothing in front of it to match against (a partial selection starting exactly at a char + * span's structural leading NBSP) and becomes a plain space. + * 2. An NBSP immediately FOLLOWING a marker token is the required opener/closer separator and + * becomes a plain space (e.g. the mandatory space after `\f`/`\fr`, or a char span's own + * leading separator when the marker literal IS present in the pasted text). + * 3. An NBSP immediately PRECEDING a marker token is a structural spacer with no source + * counterpart — `createNote`'s inter-child spacer sits exactly here — and is DROPPED entirely + * (neither spaced nor kept as data): `\nd Lord\u00A0\nd*` settles to `\nd Lord\nd*`, matching + * the source USFM, which needs no byte there at all. + * + * Passes 2 and 3 both match against the SAME `AFTER_MARKER_NBSP`/`BEFORE_MARKER_NBSP` token set, + * so a marker recognized by one is recognized by the other. Every remaining NBSP is genuine data + * and is preserved as `~`, the same display form typed data-NBSP takes, so serialization + * round-trips it to a real NBSP instead of silently collapsing it to a plain space or dropping it. + */ +export function $normalizePastedNbsp(text: string): string { + return text + .replace(/^\u00A0/gm, " ") + .replace(AFTER_MARKER_NBSP, "$1 ") + .replace(BEFORE_MARKER_NBSP, "") + .replaceAll(NBSP, "~"); +} + +/** A `\c`/`\id` marker token ANYWHERE in a line, capturing its payload up to — but not including — + * the next marker or line end. Not anchored to the line's start: a chapter/book-id token can sit + * mid-line (`x \c 5 y`, a paste landing mid-sentence), and an anchor there would silently miss it + * — the exact live-repro shape (see `$stripPastedChapterAndBookId`'s doc comment). Global so more + * than one occurrence on the same line is fully swept, not just the first. */ +const CHAPTER_OR_BOOK_ID_TOKEN = /\\(?:c|id)(?![\w-])[^\n\\]*/g; + +/** + * Drops every pasted `\c`/`\id` token and its payload (the chapter number / book code, up to the + * next marker or newline) before insertion. Both create a document-structural node PT9 allows + * only once per book (a `ChapterNode`/`BookNode`, materialized from the marker name alone — + * `usj-editor.adaptor.ts` — same as a real load), and Standard view has no per-paste "am I the + * only one" check the way an initial document load does. Live repro (2026-08-07): pasting a bare + * `\c 2` mid-chapter created a second chapter node in the editor; every subsequent save then + * failed with the PDP's "Multiple chapter markers present" (the error surfaces only in the + * renderer log — disk and other editors silently stop updating). `\id` is the book-level twin of + * the same hazard and is stripped identically. A token need not be its own line — `x \c 5 y` + * mid-paragraph reaches the same tokenizer branch (chapter/book-id tokens are recognized wherever + * they occur, not just at a fragment's start) and left unstripped produces the identical poisoned + * shape: a second chapter node PLUS the trailing bytes (`y`) stranding as a bare top-level USJ + * string outside any paragraph, since a chapter token closes the enclosing paragraph the same way + * it does on a real load. + * + * Splits on lines and strips per line (not one global pass over the whole text) so a token that + * consumes an ENTIRE line can cleanly take that line's own newline with it too (no stray empty + * paragraph left behind). A token sharing a line with a LATER marker — `\c 5\v 1 In the + * beginning` — only loses its own bytes: the token regex stops at the next `\`, leaving `\v 1 In + * the beginning` to paste normally. But `[^\n\\]*` has no such stop when nothing marker-shaped + * follows on the line: the mid-line `x \c 5 y` shape above loses the token's trailing payload TOO, + * all the way to the newline — `x \c 5 y` strips down to `x ` alone, the trailing `y` dropped + * along with the marker (pinned in `markerPasteFidelity.test.tsx`). A line that already carried no + * other content becomes empty after stripping and is dropped from the output entirely, rather than + * surviving as a blank paragraph; a line that was ALREADY blank in the source paste (nothing to do + * with `\c`/`\id`) is left alone. + * + * Exported for the in-note CRITICAL multi-line paste claim (`MarkerEditPlugin.tsx`), which shares + * this strip the same way it shares `$normalizePastedNbsp` — a `\c`/`\id` token pasted into note + * content is just as reachable (the note-content Tier 2 rebuild tokenizes literal text the same + * way a paragraph rebuild does) and just as harmful there. + */ +export function $stripPastedChapterAndBookId(text: string): string { + return text + .split("\n") + .map((line) => { + const stripped = line.replace(CHAPTER_OR_BOOK_ID_TOKEN, ""); + return stripped === "" && line !== "" ? undefined : stripped; + }) + .filter((line): line is string => line !== undefined) + .join("\n"); +} + +/** + * True when `node` is part of an attribute display run: a TextNode tagged textType "attribute" + * (a char span's bare `|…` run, a verse's `\va`/`\vp` value, or a milestone's attribute text — all + * three share this one tag), or `node` itself / one of its ancestors is an `AttributeRunNode` — the + * structural wrapper a verse/milestone run rides in, reachable by an element-point selection even + * when no value text exists yet (an empty `\qt-s \*` run with nothing typed between the glyphs). A + * char span's run has no such wrapper (it rides as a bare child of the `CharNode`), so the + * textType tag alone covers it. + * + * The `AttributeRunNode`-ancestor branch also matches a selection landing on the wrapper's OWN + * opening/self-closing `MarkerNode` glyphs (`\va`, `\va*`, a milestone's `\*`), not only its + * textType-"attribute" value text — deliberately: a `MarkerNode` is itself an ordinary `TextNode`, + * so `selection.insertText` at a glyph point behaves exactly as it would for typing there (the + * same paste-≡-typing guarantee this whole file exists to uphold), and it is harmless the same way + * inserting into any other plain text is. + */ +function $isNodeInAttributeContext(node: LexicalNode): boolean { + if ($isTextNode(node) && $getState(node, textTypeState) === "attribute") return true; + for (let ancestor: LexicalNode | null = node; ancestor; ancestor = ancestor.getParent()) + if ($isAttributeRunNode(ancestor)) return true; + return false; +} + +/** + * True when `selection` touches attribute-display text at EITHER end — its anchor OR its focus + * satisfies {@link $isNodeInAttributeContext}. A collapsed caret trivially qualifies when its one + * point does. A real (non-collapsed) selection qualifies even when only ONE end is inside an + * attribute run and the other reaches into ordinary content: under the paste-≡-typing principle, a + * user typing a character over such a mixed selection gets exactly `selection.insertText`'s own + * behavior — remove the whole selected range (whatever nodes it spans, attribute or not), then + * insert at the resulting collapsed point — so paste must take the identical path rather than + * declining into a branch (Lexical's rich-node paste, or this file's own newline-splitting + * external-paste pipeline) that can corrupt the attribute-run end of the range. See + * `$handlePasteForStandardView`'s doc comment for the two corruption shapes this closes. + */ +function $isSelectionInAttributeContext(selection: RangeSelection): boolean { + return ( + $isNodeInAttributeContext(selection.anchor.getNode()) || + $isNodeInAttributeContext(selection.focus.getNode()) + ); +} + +/** + * Paste normalization for a selection touching attribute-display text — the binding design + * principle: paste ≡ typing the same characters at the same caret. Each `\n` in `text` becomes a + * single space (a PER-NEWLINE replacement, not a run-collapsing one: `"a\n\nb"` becomes `"a b"`, + * two spaces, matching what two individual Enter-less keystrokes over an attribute run would + * produce — attribute values are single-line, so there is no multi-line attribute byte shape to + * collapse INTO). `text` arrives with bare `\n` line endings ({@link getPastePayload} normalizes + * them for every paste claim) — no `\r` reaches this function. The result is inserted via the exact `selection.insertText` a keystroke + * uses — no chapter/book-id strip, no positional NBSP mapping, no marker tokenization, all of which + * exist for BODY content's marker-recognition and whitespace-display invariants and apply to none + * of it: a pasted `\c 5` or `\p` here is literal value text, not a structural marker, and + * `$displayWhitespaceTransform` (this file) already skips textType "attribute" nodes outright, so a + * literal NBSP a user TYPES into an attribute run is never touched — a pasted one must not be + * treated any differently. For a non-collapsed (mixed or fully-inside) selection, + * `selection.insertText` removes the selected range before inserting, the same as it does for a + * typed keystroke over that selection — no separate removal step is needed here. The existing + * attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, + * `$resolvePendingMarkers`) takes over identically whether the text arrived by typing or this call. + */ +function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: string): void { + selection.insertText(text.replace(/\n/g, " ")); +} + +/** + * Standard-view PASTE normalization: every external paste (no same-namespace + * `application/x-lexical-editor` payload) is routed through the plain-text USFM carrier instead + * of Lexical's HTML import — Standard view has markers as real text, so re-tokenizing pasted + * text is the SAME mechanism that recognizes typed markers, and it is the only carrier that + * survives `navigator.clipboard.read()` at all: Chromium's async clipboard-read API exposes only + * a fixed sanctioned MIME allow-list (`text/plain`, `text/html`, and a short list of others) — + * the private `application/x-lexical-editor` flavor Lexical writes on copy is not one of them, so + * the `DataTransfer` `pasteSelection` (`clipboard.utils.ts`) rebuilds from `navigator.clipboard. + * read()` can never contain it. A real Ctrl+V — even a same-editor paste of the editor's own + * copy — therefore always rides `text/html`/`text/plain` like any external source. Previously + * this only claimed NBSP-bearing pastes and inserted the text with a BLANKET NBSP→`~` mapping; + * live repro (2026-08-07) showed that corrupts a same-editor paste of its own copy — every + * display-NBSP (the separator after `\f`/`\fr`/`\ft`) became a literal `~`, turning recognized + * markers into unknown-marker soup — and a browser-hop `\nd …\nd*` paste came back with an + * unmatched closer. `$normalizePastedNbsp` above replaces that blanket mapping with the + * positional rule. * * A MULTI-LINE payload is replayed line by line with an `INSERT_PARAGRAPH_COMMAND` dispatch - * between lines, because no USFM line can carry a newline: inserting the whole payload at once - * left literal `\n` bytes sitting inside a text node, a byte on screen the file cannot represent. - * This claim runs at HIGH ahead of every other paste claim, so an NBSP-carrying paste never - * reaches the multi-line claims that would otherwise split it — the split has to happen here. - * Going through the COMMAND (rather than `selection.insertParagraph()`) is what makes a paste - * into a character-style stack close and reopen that stack per line, the same way Enter does. - * Line endings are already normalized by {@link getPastePayload}, so `\r\n` and bare-`\r` - * clipboards break correctly and no `\r` ever lands in content. A single-line payload keeps the - * exact one-`insertText` behavior it always had. + * between lines, because no USFM line can carry a newline. Going through the COMMAND (rather than + * `selection.insertParagraph()`) is what makes a paste into a character-style stack close and + * reopen that stack per line, the same way Enter does. This claim runs at HIGH ahead of every + * other paste claim, so an external paste never reaches the multi-line claims that would + * otherwise split it — the split has to happen here. A single-line payload is one `insertText`. + * + * Declines (returns `false`, lets Lexical's own paste handling run) when: the payload carries a + * same-namespace Lexical flavor (the sync `ClipboardEvent` path — a null-payload dispatch or a + * live native paste event that still has it — keeps the exact node-tree fast path); the document + * is structure-protected (`StructureKeyboardPlugin` must sanitize the HTML payload instead — + * this handler runs at the same `COMMAND_PRIORITY_HIGH` but registers earlier, so without this + * check it would claim the paste first and starve that sanitizer — a recorded trade-off: a + * protected editor's plain-text pastes get NO NBSP normalization at all, since this handler + * never runs for them); or no text can be resolved. * - * Mutating: call inside `editor.update()` — dispatched from `MarkerEditPlugin`'s - * `PASTE_COMMAND` registration. + * The same-namespace-flavor decline is SUSPENDED whenever the selection TOUCHES attribute-display + * text at either end ({@link $isSelectionInAttributeContext}). TJ's live repro (2026-08-11, filed + * against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the + * `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing + * `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the + * saved file diverged from the editor. `sid="things"` carries no NBSP, and that pre-branch build's + * `$handlePasteForStandardView` still had its OLD gate (documented above: "previously this only + * claimed NBSP-bearing pastes") — so the most plausible mechanism is that OLD gate declining an + * NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion, + * not the same-namespace-flavor path below (this branch's OWN copy already carries `text/plain` + * unconditionally, per the doc comment above, and the "private Lexical flavor is dead on Ctrl+V" + * fact recorded in the semantics doc's S3 still holds for the reconstructed-`DataTransfer` paste + * path this repro most likely used). What both mechanisms — the old NBSP gate and this file's own + * same-namespace-flavor decline — share is the SAME failure shape once either one declines: Lexical's + * default rich-paste node insertion has no notion that an attribute run's text must stay inside its + * one tagged TextNode, and (reproduced directly on THIS branch, see + * `attributeContextPasteFidelity.test.tsx`'s "root cause" describe) merges the run, the closing + * glyph, and even the FOLLOWING paragraph's own sibling text into one plain node: the attribute + * display and the closing marker both vanish, and the pasted bytes end up loose in body content. + * Confirmed regression classes on THIS branch, closed by this suspension: (1) a live native paste + * event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own + * documented case for when that CAN still reach a handler, unlike the reconstructed-`DataTransfer` + * path); (2) a multi-line plain-text payload, which the ordinary external-paste pipeline below + * would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND` — inside an attribute run that + * is exactly as destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`), which + * the ordinary pipeline's `$stripPastedChapterAndBookId` would eat bytes out of an attribute VALUE + * that were never a chapter token to begin with; (4) a MIXED selection (one end inside the run, one + * end outside) combined with either (1) or (2) above — the selection touches attribute context, so + * it must not be allowed to reach either risky branch merely because its OTHER end sits outside. + * Attribute value bytes are never rich content — a user cannot "type formatting" into one either — + * so this handler must always claim a paste touching one and insert it as plain text + * ({@link $insertPastedTextIntoAttributeContext}), regardless of what other MIME flavors the + * clipboard also carries. Structure protection still takes precedence (checked first, below): an + * attribute run inside a protected document defers to the same protection contract as everything + * else. One further pre-existing precedence is UNCHANGED by this suspension: the CRITICAL-priority + * in-note multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs BEFORE this handler + * and still wins for a multi-line payload whose selection touches EXPANDED note content — an + * attribute run that happens to sit inside an expanded note's content is reached by this handler + * (and this suspension) only when that in-note claim itself declines. */ -export function $handlePasteForStandardView(event: ClipboardEvent | null | undefined): boolean { +export function $handlePasteForStandardView( + event: ClipboardEvent | null | undefined, + isStructureProtected = false, + armSplitExpected: () => void = () => undefined, + armPasteRebuildDedup: () => void = () => undefined, +): boolean { const payload = getPastePayload(event); - if (!payload || payload.isInternal) return false; - const { plainText, html, htmlText } = payload; - // Which source to replay is decided by where the NBSP survived, not by the usual - // plain-then-html preference: a browser-generated `text/plain` may have collapsed the - // ` ` to a plain space, losing the very thing this claim exists to preserve. - const text = plainText.includes(NBSP) - ? plainText - : html.includes(NBSP) || htmlText.includes(NBSP) - ? htmlText - : undefined; - if (!text) return false; + if (!payload) return false; const selection = $getSelection(); + const inAttributeContext = + $isRangeSelection(selection) && $isSelectionInAttributeContext(selection); + if (!inAttributeContext && payload.isInternal) return false; + if (isStructureProtected) return false; + // `text` is `text/plain` when the clipboard carries any, else the decoded `text/html` — the one + // preference every paste claim in this editor shares, with line endings already normalized to + // bare `\n` ({@link getPastePayload}), so no `\r` ever reaches content and a `\r\n` clipboard + // breaks into lines correctly. + const { text } = payload; + if (!text) return false; if (!$isRangeSelection(selection)) return false; event?.preventDefault(); - const displayText = text.replaceAll(NBSP, "~"); - const lines = displayText.split("\n"); + if (inAttributeContext) { + $insertPastedTextIntoAttributeContext(selection, text); + return true; + } + // Arms `Tier2Context.pasteRebuildArmed` for this paste's own update, BEFORE inserting — + // unconditionally, unlike `armSplitExpected` below, because a SINGLE-line paste (no newline at + // all) can just as easily trigger the immediate own-marker-prefix dedup rebuild (`\p one` pasted + // right after an existing `\p` host's prefix) as a multi-line one can. + armPasteRebuildDedup(); + const normalized = $normalizePastedNbsp($stripPastedChapterAndBookId(text)); + const lines = normalized.split("\n"); if (lines.length < 2) { - selection.insertText(displayText); + selection.insertText(normalized); return true; } + // The engine's own INSERT_PARAGRAPH_COMMAND handler arms `splitExpected` for each dispatch + // below, but the FIRST line is inserted before any of them run, so the flag is armed up front + // too: every fresh prefix-less paragraph this paste creates needs its marker prefix injected + // rather than being read as marker-deleted and merged back into the paragraph above. The + // LOW-priority PASTE_COMMAND arm that covers the pastes reaching it never runs once this + // HIGH-priority claim consumes the command. + armSplitExpected(); // Replayed as the two steps the user would have performed by hand: text, then the command. // The selection is removed up front rather than relying on the first `insertText` to replace // it, since a payload whose first line is empty (a leading newline) inserts no text at all @@ -218,6 +456,96 @@ export function $handlePasteForStandardView(event: ClipboardEvent | null | undef return true; } +/** + * Whether `node` is a collapsed note's internal display separator: a bare single-NBSP text node + * `createNote` (`usj-editor.adaptor.ts`) inserts purely so a caller/content children render apart + * on screen. USFM never needs a byte to separate a closing marker from the next `\marker` token, + * so only ONE placement of this family has a real source counterpart — the separator directly + * after the caller, which is the mandatory space between a note's caller and its first content + * marker (`\f + \fr …`). Every other placement (between content children, before the note's own + * closing marker) has nothing in the source USFM to reproduce and must contribute nothing. + */ +function $isNoteInternalDisplaySeparator(node: TextNode): boolean { + if (node.getTextContent() !== NBSP) return false; + const parent = node.getParent(); + if (!$isNoteNode(parent)) return false; + return !$isImmutableNoteCallerNode(node.getPreviousSibling()); +} + +/** The nearest enclosing `NoteNode`'s USJ caller value, falling back to the auto-generated-caller + * marker (`+`) when the note has none set. */ +function $noteCallerText(callerNode: LexicalNode): string { + const noteNode = callerNode.getParent(); + const caller = $isNoteNode(noteNode) ? noteNode.getCaller() : undefined; + return caller || GENERATOR_NOTE_CALLER; +} + +/** + * Source-faithful USFM text of `selection` — the `text/plain` leg of Standard-view copy/cut. Walks + * `selection.getNodes()` the same way `RangeSelection.getTextContent()` does (single `\n` between + * non-inline block boundaries, anchor/focus offsets respected on the boundary text nodes, + * `DecoratorNode`s contributing their own text; an inline element like `AttributeRunNode` or + * `NoteNode` contributes nothing itself, its children being walked as their own list entries), with + * two USFM-specific corrections: + * + * 1. An `ImmutableNoteCallerNode` — which renders as `""` on screen for a collapsed note with an + * auto-generated caller — contributes the enclosing note's real USJ caller (`+`, `-`, or a + * literal) plus its own leading separating space (the mandatory space after `\f`/`\x`). The node + * itself is left untouched (`getTextContent()` still returns `""`): it serves every view mode, + * and formatted-view prose copy depends on staying caller-free. + * 2. NBSP inverts to a plain space per node instead of via a blanket `replaceAll` — a note's + * internal display-only separators (`$isNoteInternalDisplaySeparator`) contribute nothing + * instead of becoming phantom spaces; every other NBSP (marker-trailing spaces, a char span's + * structural leading separator, a verse's own marker-to-number gap) represents a real source + * space and still maps to one. Data-NBSP (displayed as `~`) is untouched either way. + */ +export function $selectionToUsfmText(selection: RangeSelection): string { + const nodes = selection.getNodes(); + if (nodes.length === 0) return ""; + const firstNode = nodes[0]; + const lastNode = nodes[nodes.length - 1]; + const { anchor, focus } = selection; + const isBefore = anchor.isBefore(focus); + const [anchorOffset, focusOffset] = $getCharacterOffsets(selection); + let text = ""; + let prevWasElement = true; + for (const node of nodes) { + if ($isElementNode(node) && !node.isInline()) { + if (!prevWasElement) text += "\n"; + prevWasElement = !node.isEmpty(); + continue; + } + prevWasElement = false; + if ($isImmutableNoteCallerNode(node)) { + if (node !== lastNode || !selection.isCollapsed()) text += ` ${$noteCallerText(node)}`; + } else if ($isTextNode(node)) { + let nodeText = node.getTextContent(); + if (node === firstNode) { + if (node === lastNode) { + if ( + anchor.type !== "element" || + focus.type !== "element" || + focus.offset === anchor.offset + ) + nodeText = + anchorOffset < focusOffset + ? nodeText.slice(anchorOffset, focusOffset) + : nodeText.slice(focusOffset, anchorOffset); + } else nodeText = isBefore ? nodeText.slice(anchorOffset) : nodeText.slice(focusOffset); + } else if (node === lastNode) { + nodeText = isBefore ? nodeText.slice(0, focusOffset) : nodeText.slice(0, anchorOffset); + } + text += $isNoteInternalDisplaySeparator(node) ? "" : nodeText.replaceAll(NBSP, " "); + } else if ( + ($isDecoratorNode(node) || $isLineBreakNode(node)) && + (node !== lastNode || !selection.isCollapsed()) + ) { + text += node.getTextContent(); + } + } + return text; +} + /** * Collapse-aware display-NBSP inversion for the `text/html` clipboard flavor. Every display-NBSP * stands for a PLAIN space in the document data (marker-trailing separator spaces, char spans' @@ -268,9 +596,10 @@ export function invertDisplayNbspInHtml(html: string): string { } /** - * Payload builder: the currently-selected content, normalized so `text/plain` carries plain - * spaces where the display shows NBSP and `text/html` keeps NBSP only where a plain space would - * not survive a rich-text consumer ({@link invertDisplayNbspInHtml}). The internal + * Payload builder: the currently-selected content, normalized per flavor. `text/plain` is the + * source-faithful USFM of the selection ({@link $selectionToUsfmText}), which carries plain spaces + * where the display shows NBSP. `text/html` keeps NBSP only where a plain space would not survive + * a rich-text consumer ({@link invertDisplayNbspInHtml}). The internal * `application/x-lexical-editor` flavor keeps the display form untouched so a paste back into a * Standard-view editor round-trips exactly. Shared by both the real-event and null-event * branches of `$handleCopyForStandardView` below so they stay byte-for-byte consistent. @@ -281,7 +610,7 @@ export function $getStandardViewClipboardData( const selection = $getSelection(); if (!$isRangeSelection(selection) || selection.isCollapsed()) return undefined; const data: LexicalClipboardData = { - "text/plain": selection.getTextContent().replaceAll(NBSP, " "), + "text/plain": $selectionToUsfmText(selection), }; const html = $getHtmlContent(editor); const lexical = $getLexicalContent(editor); From 5363bb257f8d83dfb405d8755cbea360c22069e2 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 26 Aug 2026 22:57:18 -0500 Subject: [PATCH 02/44] fix: a copy with nothing selected must leave the clipboard untouched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A copy that does not come from a real browser clipboard event — Ctrl+C/Ctrl+X, the context menu's Cut/Copy, EditorRef.copy()/.cut() — dispatches the command with a null payload, and @lexical/clipboard has to synthesize an event before anything can be written: it appends a hidden placeholder element to the editor, points the DOM selection at it, and runs document.execCommand("copy"). Filling that synthesized event declines an empty selection BEFORE suppressing the browser's own copy, so the browser copies the placeholder and the user's real clipboard contents are replaced by a character that was never in the document. A read-only construct makes it likelier, not special: an UnknownNode renders contentEditable=false and its marker glyphs are decorators that take no keyboard selection, so trying to select a figure's marker text leaves the caret in the prose beside it. A plain collapsed caret does the same, in every view, and cut takes the identical path. Two layers. copySelection/cutSelection (shared-react) do not synthesize a copy for a selection that cannot produce content, and leave the key unclaimed so the browser's own no-op copy runs — that is what "untouched" means. And the Standard-view handler claims a null-payload dispatch it cannot build data for rather than declining it into @lexical/rich-text's fallback. A real clipboard event at a collapsed caret is still declined, and a node selection still goes to Lexical, which copies it correctly. Copying a read-only construct works through a selection that contains it — the shape a browser resolves a drag over one into — which reaches its display glyphs through the same walker as prose and yields the construct's real USFM bytes. Co-Authored-By: Claude Fable 5 --- .../specs/2026-08-06-clipboard-semantics.md | 5 + .../src/plugins/usj/ClipboardPlugin.test.tsx | 165 ++++++++++++++++++ .../src/plugins/usj/ClipboardPlugin.tsx | 18 +- .../src/plugins/usj/ContextMenuPlugin.tsx | 16 +- .../src/plugins/usj/clipboard.utils.ts | 45 ++++- .../editor/CommandMenuPlugin.gate.test.tsx | 56 +++++- packages/platform/src/editor/Editor.tsx | 10 +- .../markerEdit/markerEdit.test-helpers.tsx | 28 +++ .../whitespaceDisplay.plugin.utils.test.tsx | 149 ++++++++++++++-- .../whitespaceDisplay.plugin.utils.ts | 24 ++- 10 files changed, 487 insertions(+), 29 deletions(-) create mode 100644 libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index d8fa8ac4a..34e146fb7 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -22,6 +22,8 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Multi-block selections:** One `\n` between blocks; a selection starting mid-paragraph omits that paragraph's own `\p ` glyph (matches P9's rendered-text copy). Pinned, not changed. +- **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection *before* calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it *likelier*: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers: `ClipboardPlugin`/`ContextMenuPlugin`/`EditorRef` (via `copySelection`/`cutSelection`, `clipboard.utils.ts`, shared-react) do not synthesize a copy for a selection that cannot produce content, and do not `preventDefault` the key either — the browser's own copy of an empty selection is itself a no-op, and letting it run is what "untouched" means; and `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into `@lexical/rich-text`'s fallback. A real clipboard event is still declined at a collapsed caret (the browser already writes nothing). A NodeSelection is still declined too — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it — the shape a browser resolves a drag or click over a `contentEditable=false` block into, since it will not place a selection endpoint inside one. Such a selection reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view, where no Standard-view handler is registered at all), and `ClipboardPlugin.test.tsx` (shared-react). + - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). ### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) @@ -83,6 +85,8 @@ Under S3 this is semantically identical to a normal external paste in Standard v ### S6. Cut = copy + `removeText()` (existing) +Including the empty case: a cut with nothing selected writes nothing and removes nothing, by the same rule and the same two enforcement layers as copy (see S1's "Copying nothing leaves the clipboard untouched"). + The WI-2 filed quirk — a selection-delete ending exactly at a just-settled char-span boundary absorbing one adjacent character — gets a targeted regression pin. **Outcome (2026-08-07):** does NOT reproduce at the Lexical selection level. Pinned as regression armor for both `CUT_COMMAND` and a plain `removeText()`, each given a selection whose focus is an element point mirroring DOM `range.setEndAfter(spanElement)` — the exact boundary shape the live repro's programmatic DOM selection used. Both variants produced byte-exact, correctly-scoped results, narrowing the live repro's root cause to the DOM Range → Lexical selection-resolution layer (`applyDOMRange`/`$internalResolveSelectionPoints`), which is not exercised by a Lexical-level `RangeSelection` and so remains covered only by the E2E selection steps (Task 8). ### S7. Undo — every paste, including rebuilds it triggers, is one undo step @@ -283,6 +287,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; six known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"` fixture (tight spacing, no flanking spaces) is swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | +| **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | | **E2E** Real-browser clipboard round trip (Standard view, written, not yet run in this task) | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | --- diff --git a/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx b/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx new file mode 100644 index 000000000..6151b3f7a --- /dev/null +++ b/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx @@ -0,0 +1,165 @@ +import { ClipboardPlugin } from "./ClipboardPlugin"; +import { baseTestEnvironment } from "./react-test.utils"; +import { act } from "@testing-library/react"; +import { + $createTextNode, + $getRoot, + COMMAND_PRIORITY_CRITICAL, + COPY_COMMAND, + CUT_COMMAND, + LexicalEditor, + TextNode, +} from "lexical"; +import { $createParaNode } from "shared"; + +/** + * `document.execCommand` is the mechanism a clipboard write ultimately runs through when no real + * clipboard event exists: `@lexical/clipboard` points the DOM selection at a hidden placeholder it + * appends to the editor and calls `execCommand("copy")` to provoke one. jsdom implements no + * `execCommand` at all, so these tests install a spy in its place — a call to it means a clipboard + * write reached the browser, and no call means the clipboard was left exactly as the user left it. + * That is the observable these tests pin: what does or does not get written, never the placeholder + * itself, which lives in `@lexical/clipboard` and is not ours to depend on. + */ +let execCommand: ReturnType; + +beforeEach(() => { + execCommand = vi.fn(() => true); + Object.defineProperty(document, "execCommand", { + configurable: true, + writable: true, + value: execCommand, + }); + // `pasteSelection`/`pasteSelectionAsPlainText` read the async clipboard API, which jsdom also + // does not implement. A read that never settles is enough for these tests: they assert only that + // the paste keys still reach it, not what a paste does with what it finds. + vi.stubGlobal("navigator", { + ...navigator, + clipboard: { read: vi.fn(() => new Promise(() => undefined)) }, + }); +}); + +afterEach(() => { + Reflect.deleteProperty(document, "execCommand"); + vi.unstubAllGlobals(); +}); + +/** An editor holding one paragraph of text, with the clipboard key handling under test mounted. */ +async function clipboardEnvironment(): Promise<{ editor: LexicalEditor; text: TextNode }> { + let text: TextNode | undefined; + const { editor } = await baseTestEnvironment( + () => { + text = $createTextNode("In the beginning"); + $getRoot().append($createParaNode("p").append(text)); + }, + , + ); + if (!text) throw new Error("expected the initial text node to exist"); + return { editor, text }; +} + +/** Presses a clipboard shortcut on the editor's root element, where the plugin listens. */ +async function pressShortcut( + editor: LexicalEditor, + key: string, + shiftKey = false, +): Promise { + const rootElement = editor.getRootElement(); + if (!rootElement) throw new Error("editor has no root element to press a key on"); + const event = new KeyboardEvent("keydown", { + key, + ctrlKey: true, + shiftKey, + bubbles: true, + cancelable: true, + }); + await act(async () => { + rootElement.dispatchEvent(event); + }); + return event; +} + +/** + * Watches a clipboard command without claiming it, so a regression runs the whole real chain — + * `@lexical/rich-text`'s fallback and the synthesized-copy machinery behind it — and trips the + * `execCommand` assertion instead of being masked by the watcher itself. + */ +function watchCommand(editor: LexicalEditor, command: typeof COPY_COMMAND) { + const seen = vi.fn(() => false); + const unregister = editor.registerCommand(command, seen, COMMAND_PRIORITY_CRITICAL); + return { seen, unregister }; +} + +describe("ClipboardPlugin — copy/cut with nothing selected", () => { + it("leaves the clipboard untouched on Ctrl+C at a collapsed caret", async () => { + const { editor, text } = await clipboardEnvironment(); + await act(async () => editor.update(() => text.select(3, 3))); + const { seen, unregister } = watchCommand(editor, COPY_COMMAND); + + const event = await pressShortcut(editor, "c"); + unregister(); + + // No copy is synthesized at all, so nothing can be written over what the clipboard holds. + expect(execCommand).not.toHaveBeenCalled(); + expect(seen).not.toHaveBeenCalled(); + // The browser's own copy is left to run: for an empty selection it writes nothing, which is + // the behavior this plugin's `preventDefault` would otherwise have suppressed. + expect(event.defaultPrevented).toBe(false); + }); + + it("leaves the clipboard untouched on Ctrl+X at a collapsed caret, and removes nothing", async () => { + const { editor, text } = await clipboardEnvironment(); + await act(async () => editor.update(() => text.select(3, 3))); + const { seen, unregister } = watchCommand(editor, CUT_COMMAND); + + const event = await pressShortcut(editor, "x"); + unregister(); + + expect(execCommand).not.toHaveBeenCalled(); + expect(seen).not.toHaveBeenCalled(); + expect(event.defaultPrevented).toBe(false); + editor.getEditorState().read(() => expect(text.getTextContent()).toBe("In the beginning")); + }); +}); + +describe("ClipboardPlugin — copy/cut with a selection", () => { + it("copies on Ctrl+C and claims the key", async () => { + const { editor, text } = await clipboardEnvironment(); + await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); + // Claimed here so the copy stops at this assertion instead of running the real clipboard + // machinery, whose synthesized-event timer would outlive the test. + const seen = vi.fn(() => true); + const unregister = editor.registerCommand(COPY_COMMAND, seen, COMMAND_PRIORITY_CRITICAL); + + const event = await pressShortcut(editor, "c"); + unregister(); + + expect(seen).toHaveBeenCalledTimes(1); + expect(event.defaultPrevented).toBe(true); + }); + + it("cuts on Ctrl+X and claims the key", async () => { + const { editor, text } = await clipboardEnvironment(); + await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); + const seen = vi.fn(() => true); + const unregister = editor.registerCommand(CUT_COMMAND, seen, COMMAND_PRIORITY_CRITICAL); + + const event = await pressShortcut(editor, "x"); + unregister(); + + expect(seen).toHaveBeenCalledTimes(1); + expect(event.defaultPrevented).toBe(true); + }); +}); + +describe("ClipboardPlugin — paste keys are unaffected", () => { + it("claims Ctrl+V and Ctrl+Shift+V regardless of the selection", async () => { + const { editor, text } = await clipboardEnvironment(); + await act(async () => editor.update(() => text.select(3, 3))); + + // A collapsed caret is exactly where a paste belongs, so the empty-selection rule copy and cut + // now follow must not reach these. + expect((await pressShortcut(editor, "v")).defaultPrevented).toBe(true); + expect((await pressShortcut(editor, "v", true)).defaultPrevented).toBe(true); + }); +}); diff --git a/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx b/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx index 9158db69e..f77f95f4c 100644 --- a/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx @@ -1,7 +1,11 @@ -import { pasteSelection, pasteSelectionAsPlainText } from "./clipboard.utils"; +import { + copySelection, + cutSelection, + pasteSelection, + pasteSelectionAsPlainText, +} from "./clipboard.utils"; import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext"; import { IS_APPLE } from "@lexical/utils"; -import { COPY_COMMAND, CUT_COMMAND } from "lexical"; import { useEffect } from "react"; export function ClipboardPlugin(): null { @@ -12,12 +16,14 @@ export function ClipboardPlugin(): null { const { key, shiftKey, metaKey, ctrlKey, altKey } = event; if (!(IS_APPLE ? metaKey : ctrlKey) || altKey) return; + // Copy/cut suppress the browser's own handling only when this plugin actually replaces it. + // With nothing selected there is nothing to replace, and leaving the browser to it is what + // keeps the clipboard untouched — see `copySelection` for what happens when a copy with no + // content behind it is synthesized anyway. if (!shiftKey && key.toLowerCase() === "c") { - event.preventDefault(); - editor.dispatchCommand(COPY_COMMAND, null); + if (copySelection(editor)) event.preventDefault(); } else if (!shiftKey && key.toLowerCase() === "x") { - event.preventDefault(); - editor.dispatchCommand(CUT_COMMAND, null); + if (cutSelection(editor)) event.preventDefault(); } else if (key.toLowerCase() === "v") { event.preventDefault(); if (shiftKey) pasteSelectionAsPlainText(editor); diff --git a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx index 7d8efd25e..7428bdb02 100644 --- a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx @@ -2,9 +2,13 @@ * Adapted from https://github.com/facebook/lexical/blob/main/packages/lexical-playground/src/plugins/ContextMenuPlugin/index.tsx */ -import { pasteSelection, pasteSelectionAsPlainText } from "./clipboard.utils"; +import { + copySelection, + cutSelection, + pasteSelection, + pasteSelectionAsPlainText, +} from "./clipboard.utils"; import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext"; -import { COPY_COMMAND, CUT_COMMAND } from "lexical"; import { ReactElement, useCallback, @@ -134,15 +138,19 @@ export function ContextMenuPlugin({ const options = useMemo(() => { const builtIn = [ + // Cut/Copy are no-ops with nothing selected rather than dispatches the browser would have to + // synthesize a clipboard event for — see `copySelection`. They are not disabled instead, + // because this option list is built once per editor rather than per menu opening, so its + // `isDisabled` flags cannot track the live selection. new ContextMenuOption(`Cut`, { onSelect: () => { - editor.dispatchCommand(CUT_COMMAND, null); + cutSelection(editor); }, isDisabled: isReadonly, }), new ContextMenuOption(`Copy`, { onSelect: () => { - editor.dispatchCommand(COPY_COMMAND, null); + copySelection(editor); }, }), new ContextMenuOption(`Paste`, { diff --git a/libs/shared-react/src/plugins/usj/clipboard.utils.ts b/libs/shared-react/src/plugins/usj/clipboard.utils.ts index e5b451e5d..a723b873c 100644 --- a/libs/shared-react/src/plugins/usj/clipboard.utils.ts +++ b/libs/shared-react/src/plugins/usj/clipboard.utils.ts @@ -1,9 +1,52 @@ -import { LexicalEditor, PASTE_COMMAND } from "lexical"; +import { $getSelection, COPY_COMMAND, CUT_COMMAND, LexicalEditor, PASTE_COMMAND } from "lexical"; function cleanupText(text: string): string { return text.replaceAll("\t", " "); } +/** + * Whether the editor's selection holds content a copy could put on the clipboard: a selection + * exists and covers something. A collapsed caret — or no selection at all — does not. + * + * This is the precondition for SYNTHESIZING a copy (see {@link copySelection}), not a general + * "can the user copy" predicate: a real, browser-generated copy needs no such check because the + * browser already knows an empty selection copies nothing. + */ +export function canCopySelection(editor: LexicalEditor): boolean { + return editor.getEditorState().read(() => { + const selection = $getSelection(); + return !!selection && !selection.isCollapsed(); + }); +} + +/** + * Copies the selection by dispatching `COPY_COMMAND` with no event of its own, and reports whether + * it did. Callers that suppressed the browser's own copy to get here should suppress it only when + * this returns `true`. + * + * Nothing is dispatched for a selection that cannot produce clipboard content. Without an event, + * the copy has to be synthesized: `@lexical/clipboard` selects a hidden placeholder element it + * appends to the editor and runs `document.execCommand("copy")` to provoke a real clipboard event + * it can fill in. That filling step declines an empty selection — and declines it BEFORE + * suppressing the browser's own copy — so the browser copies what it was pointed at, the + * placeholder, and the user loses whatever the clipboard already held to a character that was + * never in the document. Copying nothing must leave the clipboard alone, which is exactly what + * not dispatching achieves. + */ +export const copySelection = (editor: LexicalEditor): boolean => { + if (!canCopySelection(editor)) return false; + editor.dispatchCommand(COPY_COMMAND, null); + return true; +}; + +/** Cuts the selection, under the same "nothing selected, nothing dispatched" rule as + * {@link copySelection} — a cut copies first, so it reaches the identical synthesized-copy path. */ +export const cutSelection = (editor: LexicalEditor): boolean => { + if (!canCopySelection(editor)) return false; + editor.dispatchCommand(CUT_COMMAND, null); + return true; +}; + export const pasteSelection = (editor: LexicalEditor) => { navigator.clipboard.read().then(async (items) => { const permission = await navigator.permissions.query({ diff --git a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx index f80f0a939..5e3d44c48 100644 --- a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx +++ b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx @@ -1,5 +1,6 @@ import Editor from "./Editor"; import { EditorRef } from "./editor.model"; +import { execCommandSpy } from "./markerEdit/markerEdit.test-helpers"; import { act, render } from "@testing-library/react"; import { Usj } from "@eten-tech-foundation/scripture-utilities"; import { createRef } from "react"; @@ -15,7 +16,7 @@ import { } from "lexical"; import { EditorRefPlugin } from "@lexical/react/LexicalEditorRefPlugin"; import { FORMATTED_VIEW_MODE, getViewOptions, STANDARD_VIEW_MODE } from "shared-react"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // The Editor-level gate controls shared-react's `CommandMenuPlugin`, which `preventDefault`s a // typed or pasted `\` and `/` so those characters never land in the document. Editable marker @@ -277,3 +278,56 @@ describe("MarkerEditPlugin's Standard-view clipboard handlers do not leak into F }); }); }); + +/** + * The empty-copy rule is not Standard view's. It lives in `ClipboardPlugin` (shared-react), which + * every view mounts, and this view has no Standard-view copy handler registered at all — so a + * regression here would leak the same way the reported one did, in a view where nothing else is + * watching. `Ctrl+C` is pressed on the editor's root element, where that plugin listens, rather + * than through a command dispatch, so what runs is the whole real path from key to clipboard. + */ +describe("copying an empty selection leaves the clipboard alone in a hidden-marker view", () => { + // `execCommandSpy` supplies the `document.execCommand` jsdom lacks and reports whether anything + // was written. `ClipboardEvent` is stubbed for the same reason `formattedViewCopyEvent` stubs + // it: without it a regression would die on Lexical's own bare reference to the missing global + // before reaching the clipboard, hiding what actually got written. + const execCommand = execCommandSpy(); + + beforeEach(() => { + vi.stubGlobal("ClipboardEvent", StubClipboardEvent); + }); + + afterEach(() => vi.unstubAllGlobals()); + + it("Ctrl+C at a collapsed caret in Formatted view writes nothing and leaves the key unclaimed", async () => { + const editor = await renderEditor(FORMATTED_VIEW_MODE); + await act(async () => + editor.update(() => { + const last = $getRoot().getLastDescendant(); + if (!last) throw new Error("expected the loaded document to have content"); + const lastSize = "getTextContentSize" in last ? last.getTextContentSize() : 0; + const point = $createPoint(last.getKey(), lastSize, "text"); + const selection = $createRangeSelection(); + selection.anchor = point; + selection.focus = point; + $setSelection(selection); + }), + ); + + const rootElement = editor.getRootElement(); + if (!rootElement) throw new Error("editor has no root element to press a key on"); + const event = new KeyboardEvent("keydown", { + key: "c", + ctrlKey: true, + bubbles: true, + cancelable: true, + }); + await act(async () => { + rootElement.dispatchEvent(event); + }); + + expect(execCommand()).not.toHaveBeenCalled(); + // Unclaimed, so the browser's own copy of an empty selection runs — and writes nothing. + expect(event.defaultPrevented).toBe(false); + }); +}); diff --git a/packages/platform/src/editor/Editor.tsx b/packages/platform/src/editor/Editor.tsx index 88cf0cdec..74cfd0415 100644 --- a/packages/platform/src/editor/Editor.tsx +++ b/packages/platform/src/editor/Editor.tsx @@ -52,8 +52,6 @@ import { $isRangeSelection, $isTextNode, $setSelection, - COPY_COMMAND, - CUT_COMMAND, EditorState, LexicalEditor, HISTORIC_TAG, @@ -107,6 +105,8 @@ import { ClipboardPlugin, CommandMenuPlugin, ContextMenuPlugin, + copySelection, + cutSelection, DeltaOnChangePlugin, DeltaOp, DisableHistoryShortcutsPlugin, @@ -491,12 +491,14 @@ const Editor = forwardRef(function Editor( redo() { editorRef.current?.dispatchCommand(REDO_COMMAND, undefined); }, + // Both leave the clipboard untouched when nothing is selected, rather than asking the browser + // to synthesize a clipboard event for an empty copy — see `copySelection` (shared-react). cut() { assertEditable("cut"); - editorRef.current?.dispatchCommand(CUT_COMMAND, null); + if (editorRef.current) cutSelection(editorRef.current); }, copy() { - editorRef.current?.dispatchCommand(COPY_COMMAND, null); + if (editorRef.current) copySelection(editorRef.current); }, paste() { assertEditable("paste"); diff --git a/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx b/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx index f32efe144..4c8eb8e3e 100644 --- a/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx +++ b/packages/platform/src/editor/markerEdit/markerEdit.test-helpers.tsx @@ -540,6 +540,34 @@ export function copyEvent(): { event: ClipboardEvent; getData: (type: string) => }; } +/** + * Puts a spy where `document.execCommand` would be, for the duration of each test in the calling + * describe, and returns a getter for it (a getter, not the spy itself, because a fresh one is + * installed per test). + * + * `execCommand("copy")` is how a clipboard write reaches the browser when there is no real + * clipboard event to fill in — `@lexical/clipboard` runs it to provoke one. jsdom implements no + * `execCommand` at all, so a suite that needs to know whether anything was written has to supply + * it: called means a write reached the browser, not called means the clipboard is untouched. + * + * Registers its own `beforeEach`/`afterEach`; call it at describe scope. + */ +export function execCommandSpy(): () => ReturnType { + let spy: ReturnType; + beforeEach(() => { + spy = vi.fn(() => true); + Object.defineProperty(document, "execCommand", { + configurable: true, + writable: true, + value: spy, + }); + }); + afterEach(() => { + Reflect.deleteProperty(document, "execCommand"); + }); + return () => spy; +} + /** * jsdom-safe paste-event stub carrying an arbitrary clipboard MIME payload. `types`/`files` are * populated so Lexical's own default paste handling (reached whenever a Standard-view/protection diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index 89d5d101b..c7c6ab763 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -1,6 +1,7 @@ import { MarkerEditPlugin } from "./MarkerEditPlugin"; import { copyEvent, + execCommandSpy, findOnlyNote, pasteEvent, serializedState, @@ -43,6 +44,7 @@ import { $isCharNode, $isMarkerNode, $isParaNode, + $isUnknownNode, NBSP, NoteNode, ParaNode, @@ -240,12 +242,36 @@ function $selectAcrossFigure(): void { $setSelection(selection); } -async function copyAcrossFigure() { - const { editor } = await baseTestEnvironment( +/** + * Selects the figure and nothing else, as the two element points on its parent paragraph that sit + * either side of it — the shape a browser resolves a drag or click over a `contentEditable=false` + * block into, since it will not put a selection endpoint inside one. + * + * Mutating: call inside `editor.update()`. + */ +function $selectOnlyTheFigure(): void { + const para = $getRoot() + .getChildren() + .find((node) => $isParaNode(node)); + if (!para) throw new Error("Expected the loaded document to have a paragraph"); + const figureIndex = para.getChildren().findIndex((child) => $isUnknownNode(child)); + if (figureIndex < 0) throw new Error("Expected the paragraph to contain the figure"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(para.getKey(), figureIndex, "element"); + selection.focus = $createPoint(para.getKey(), figureIndex + 1, "element"); + $setSelection(selection); +} + +async function copyEnvironmentWithFigure() { + return baseTestEnvironment( serializedState(usxStringToUsj(FIGURE_USX)), , ); - await act(async () => editor.update(() => $selectAcrossFigure())); +} + +async function copyFigureWith($select: () => void) { + const { editor } = await copyEnvironmentWithFigure(); + await act(async () => editor.update($select)); const { event, getData } = copyEvent(); await act(async () => { editor.dispatchCommand(COPY_COMMAND, event); @@ -253,6 +279,10 @@ async function copyAcrossFigure() { return getData; } +async function copyAcrossFigure() { + return copyFigureWith($selectAcrossFigure); +} + describe("copy across an UnknownNode (figure) — full USFM byte display", () => { it("copies the figure's exact USFM: opening marker, caption, then attributes and closer", async () => { const getData = await copyAcrossFigure(); @@ -287,6 +317,19 @@ describe("copy across an UnknownNode (figure) — full USFM byte display", () => expect(html).not.toContain("fig"); expect(html).not.toContain("caption"); }); + + it("copies the figure's own bytes when the selection covers the figure alone", async () => { + const getData = await copyFigureWith($selectOnlyTheFigure); + + // The answer to "how does a user copy a read-only construct's marker text": by selecting the + // construct, from outside it. Its glyphs cannot take a caret or a keyboard selection of their + // own, but a selection that CONTAINS the block reaches every one of them through the same + // walker the surrounding prose goes through, so the copied bytes are the figure's real USFM + // with nothing of the neighbouring paragraph attached. + expect(getData("text/plain")).toBe( + '\\fig caption|src="cn01617.jpg" size="span" ref="1.18"\\fig*', + ); + }); }); describe("$getStandardViewClipboardData", () => { @@ -580,28 +623,112 @@ describe("clipboard normalization — null-event leg (ClipboardPlugin/ContextMen editor.getEditorState().read(() => expect(text.getTextContent()).toBe(`a${NBSP}${NBSP}b`)); }); - it("returns false (does not call copyToClipboard) for a collapsed selection", async () => { - // Calls the handler directly rather than via editor.dispatchCommand: a `false` return here - // falls through to Lexical's own RichText copy fallback, which — in real browsers — is fine, - // but under jsdom crashes on a bare `ClipboardEvent` reference (jsdom doesn't implement the - // class; verified above) regardless of this plugin's code. That's a pre-existing jsdom gap - // in Lexical's own fallback, orthogonal to what's under test here (the collapsed-selection - // decline), so it's sidestepped by unit-testing the handler in isolation. + it("declines a real clipboard event at a collapsed caret, writing nothing and suppressing nothing", async () => { + // A native copy event needs no help with an empty selection: the browser's own copy of an + // empty DOM selection writes nothing, so declining — and NOT calling preventDefault — is what + // leaves the clipboard exactly as the user left it. let text: TextNode; const { editor } = await testEnvironment(() => { text = $createTextNode(`a${NBSP}b`); $appendMarkerAndText(text); }); await act(async () => editor.update(() => text.select(1, 1))); + const { event, getData } = copyEvent(); copyToClipboardSpy.mockClear(); let handled: boolean | undefined; await act(async () => editor.update(() => { - handled = $handleCopyForStandardView(null, editor, false); + handled = $handleCopyForStandardView(event, editor, false); }), ); expect(handled).toBe(false); + expect(getData("text/plain")).toBe(""); + expect(event.preventDefault).not.toHaveBeenCalled(); + expect(copyToClipboardSpy).not.toHaveBeenCalled(); + }); +}); + +/** + * A copy with nothing selected must leave the clipboard holding whatever it already held. + * + * These dispatch the real command rather than calling the handler directly, because what matters + * is what happens when the handler DECLINES: the command carries on to `@lexical/rich-text`, which + * asks `@lexical/clipboard` to synthesize the clipboard event a null-payload dispatch never had — + * appending a hidden placeholder element to the editor, pointing the DOM selection at it, and + * running `document.execCommand("copy")` to provoke a real event to fill in. With nothing + * selected, that filling step bails before it can suppress the browser's own copy, so the browser + * copies the placeholder and whatever the user had on the clipboard is gone. Live report + * (2026-08-26): trying to copy the marker text of a read-only construct put a single stray + * character on the clipboard; a plain collapsed caret anywhere in the view does the same, and so + * does cut. + * + * `document.execCommand` — which jsdom does not implement, so these tests supply it — is the + * observable: called means a write reached the browser, not called means the clipboard is + * untouched. The placeholder's own content belongs to `@lexical/clipboard` and is not pinned here. + */ +describe("copying an empty selection leaves the clipboard alone", () => { + const execCommand = execCommandSpy(); + + it("COPY_COMMAND(null) at a collapsed caret is claimed and writes nothing", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + text = $createTextNode(`a${NBSP}b`); + $appendMarkerAndText(text); + }); + await act(async () => editor.update(() => text.select(1, 1))); + copyToClipboardSpy.mockClear(); + let handled: boolean | undefined; + await act(async () => { + handled = editor.dispatchCommand(COPY_COMMAND, null); + }); + expect(execCommand()).not.toHaveBeenCalled(); expect(copyToClipboardSpy).not.toHaveBeenCalled(); + // Claimed, so nothing downstream gets the chance to synthesize the copy this declined to make. + expect(handled).toBe(true); + editor.getEditorState().read(() => expect(text.getTextContent()).toBe(`a${NBSP}b`)); + }); + + it("CUT_COMMAND(null) at a collapsed caret is claimed, writes nothing and removes nothing", async () => { + let text: TextNode; + const { editor } = await testEnvironment(() => { + text = $createTextNode(`a${NBSP}b`); + $appendMarkerAndText(text); + }); + await act(async () => editor.update(() => text.select(1, 1))); + copyToClipboardSpy.mockClear(); + let handled: boolean | undefined; + await act(async () => { + handled = editor.dispatchCommand(CUT_COMMAND, null); + }); + expect(execCommand()).not.toHaveBeenCalled(); + expect(copyToClipboardSpy).not.toHaveBeenCalled(); + expect(handled).toBe(true); + editor.getEditorState().read(() => expect(text.getTextContent()).toBe(`a${NBSP}b`)); + }); + + it("COPY_COMMAND(null) with the caret beside a read-only figure is claimed and writes nothing", async () => { + // The reported shape: a figure renders `contentEditable="false"` and its marker/attribute + // glyphs are display decorators that are not keyboard-selectable, so an attempt to select the + // marker text inside one leaves the caret in the prose beside it rather than in the figure. + const { editor } = await copyEnvironmentWithFigure(); + await act(async () => + editor.update(() => { + const before = $dfs($getRoot()) + .map(({ node }) => node) + .filter($isTextNode) + .find((node) => node.getTextContent() === "Before "); + if (!before) throw new Error("expected the text before the figure to exist"); + before.select(before.getTextContentSize(), before.getTextContentSize()); + }), + ); + copyToClipboardSpy.mockClear(); + let handled: boolean | undefined; + await act(async () => { + handled = editor.dispatchCommand(COPY_COMMAND, null); + }); + expect(execCommand()).not.toHaveBeenCalled(); + expect(copyToClipboardSpy).not.toHaveBeenCalled(); + expect(handled).toBe(true); }); }); diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 4f4e41b9a..c3e6150e9 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -619,14 +619,34 @@ export function $getStandardViewClipboardData( return data; } -/** Clipboard text carries plain spaces where the display shows NBSP. */ +/** + * Clipboard text carries plain spaces where the display shows NBSP. + * + * Also the last line of defense for an EMPTY copy. A dispatch with no clipboard event of its own + * cannot simply be declined: `@lexical/rich-text` picks it up and has `@lexical/clipboard` + * synthesize an event — a hidden placeholder element is appended to the editor, the DOM selection + * pointed at it, and `document.execCommand("copy")` run to provoke a real clipboard event to fill + * in. With nothing selected, the filling step bails BEFORE it suppresses the browser's own copy, + * and the browser copies the placeholder, replacing the clipboard's real contents with a character + * that was never in the document. Standard view is where a caret most often has nothing copyable + * around it — a click on a read-only construct (a figure, a table) leaves the caret beside it + * rather than inside — so a null-payload dispatch with no content behind it is CLAIMED here and + * does nothing, which is what copying an empty selection means. Callers that can decline earlier + * do (`copySelection`, shared-react); this covers the ones that reach the command anyway. + * + * A selection that is merely not a RANGE (a node selection) is still declined: it has real content, + * and the synthesized-event path copies it correctly. + */ export function $handleCopyForStandardView( event: ClipboardEvent | null | undefined, editor: LexicalEditor, isCut: boolean, ): boolean { const selection = $getSelection(); - if (!$isRangeSelection(selection) || selection.isCollapsed()) return false; + if (!$isRangeSelection(selection) || selection.isCollapsed()) { + const isNullPayloadDispatch = !event || !("clipboardData" in event); + return isNullPayloadDispatch && (!selection || selection.isCollapsed()); + } const data = $getStandardViewClipboardData(editor); if (!data) return false; if (!event || !("clipboardData" in event)) { From 4ef04d8977ed96b00edab0d5d7fbac3f8b0b31e5 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 26 Aug 2026 23:05:45 -0500 Subject: [PATCH 03/44] refactor: close the rebase-review findings on the paste payload and marker kinds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The paste carrier choice now states its trade-off where it is made. Choosing by which carrier an NBSP survived in is the better rule for a foreign clipboard, and unusable here: Standard view's own text/plain deliberately carries no NBSP at all, so an NBSP-presence test is true of every P10 copy and would route the editor's own round trip through html, whose decoded text drops a collapsed note's caller. Recorded in the semantics doc's deferred list too. PastePayload keeps only what a claim may act on. The unresolved carriers were exposed and unread; every claim must replay the same bytes, and a second carrier within reach invites one of them to pick differently. Doc corrections: getPastePayload named three PASTE_COMMAND handlers and described the Standard-view claim as NBSP-gated, both false since the merge; isInternal claimed that claim declines internal payloads outright, false since the attribute path claims them; $handlePasteForStandardView lost its "call inside editor.update()" contract note while gaining a $getEditor() call and a command dispatch that make it matter more. The marker-kind classifiers move to a leaf module. Both tiers ask the question and must answer it identically, and Tier 2 reaching into Tier 1 for it made the two import each other. Their only inputs are a string and a stylesheet lookup, so neither tier is needed to answer it. Also restores the multi-paragraph-html paste test's return-contract assertion, lost when it switched to a command dispatch — the content assertion alone cannot tell this handler's replay from Lexical's own html import. Co-Authored-By: Claude Fable 5 --- .../specs/2026-08-06-clipboard-semantics.md | 6 +-- .../markerEdit/markerEditTier1.utils.ts | 36 +------------ .../src/editor/markerEdit/markerKind.utils.ts | 48 +++++++++++++++++ .../editor/markerEdit/tier2Rebuild.utils.ts | 4 +- .../whitespaceDisplay.plugin.utils.test.tsx | 6 ++- .../whitespaceDisplay.plugin.utils.ts | 54 ++++++++++++------- 6 files changed, 94 insertions(+), 60 deletions(-) create mode 100644 packages/platform/src/editor/markerEdit/markerKind.utils.ts diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 34e146fb7..32b1d79d0 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -87,7 +87,7 @@ Under S3 this is semantically identical to a normal external paste in Standard v Including the empty case: a cut with nothing selected writes nothing and removes nothing, by the same rule and the same two enforcement layers as copy (see S1's "Copying nothing leaves the clipboard untouched"). -The WI-2 filed quirk — a selection-delete ending exactly at a just-settled char-span boundary absorbing one adjacent character — gets a targeted regression pin. **Outcome (2026-08-07):** does NOT reproduce at the Lexical selection level. Pinned as regression armor for both `CUT_COMMAND` and a plain `removeText()`, each given a selection whose focus is an element point mirroring DOM `range.setEndAfter(spanElement)` — the exact boundary shape the live repro's programmatic DOM selection used. Both variants produced byte-exact, correctly-scoped results, narrowing the live repro's root cause to the DOM Range → Lexical selection-resolution layer (`applyDOMRange`/`$internalResolveSelectionPoints`), which is not exercised by a Lexical-level `RangeSelection` and so remains covered only by the E2E selection steps (Task 8). +The WI-2 filed quirk — a selection-delete ending exactly at a just-settled char-span boundary absorbing one adjacent character — gets a targeted regression pin. **Outcome (2026-08-07):** does NOT reproduce at the Lexical selection level. Pinned as regression armor for both `CUT_COMMAND` and a plain `removeText()`, each given a selection whose focus is an element point mirroring DOM `range.setEndAfter(spanElement)` — the exact boundary shape the live repro's programmatic DOM selection used. Both variants produced byte-exact, correctly-scoped results, narrowing the live repro's root cause to the DOM Range → Lexical selection-resolution layer (`applyDOMRange`/`$internalResolveSelectionPoints`), which is not exercised by a Lexical-level `RangeSelection` and so remains covered only by the E2E selection steps. ### S7. Undo — every paste, including rebuilds it triggers, is one undo step @@ -263,8 +263,6 @@ The following items are recorded as deferred and not addressed in this plan: 9. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. ---- - 10. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). 11. **Lexical-flavor copy/paste of figure/table/sidebar/periph/ref UnknownNode kinds (measured, 2026-08-13):** S2's `excludeFromCopy` fix above is scoped to "optbreak" only; every other `UnknownNode` kind keeps the OLD, still-excluding behavior, PENDING per-kind paste verification this task did not do. The residual shape was directly measured (not just inferred from the mechanism), and it is WORSE, in user terms, than the optbreak symptom this task fixed: a same-namespace lexical-flavor copy/paste of `
caption
` renders the figure's FULL literal USFM bytes on screen after paste — `\fig caption|src="cn01617.jpg" size="span" ref="1.18"\fig*` — because the figure's hoisted marker/attribute decorators still `decorate()` their own literal text as loose, unwrapped siblings, giving a completely convincing on-screen appearance that the figure came through intact. The pasted document's USJ, measured against that same on-screen result, silently drops the figure node AND every one of its attributes entirely — only its real caption TextNode child survives, merged into the surrounding paragraph's prose (`content: ["Before caption after."]`, no `figure` entry, no `file`/`size`/`ref` anywhere). A save at this point would silently persist the loss with no error, no warning, and a screen that still shows the figure's bytes — genuine silent data loss with a convincing display. `table`, `sidebar`, `periph`, and `ref` share the identical `excludeFromCopy`/`exportDOM(null)` mechanism (the same content-free `ImmutableTypedTextNode` display-decorator shape `unknownDisplayParts` builds for every kind) and are expected, not merely suspected, to exhibit the same class of residual, though only the figure case above was directly measured. Fixing this for all five kinds the way S2 now does for optbreak is a plausible, low-risk follow-up — narrowing `excludeFromCopy` further, one verified kind at a time — but is out of scope here. @@ -273,6 +271,8 @@ The following items are recorded as deferred and not addressed in this plan: 13. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 5 above. +14. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. + ## Test Mapping (S1–S7) | Semantic | Pinning Test | diff --git a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts index d8d24d81e..6d7e9d447 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts @@ -4,6 +4,7 @@ * Everything Tier 1 cannot express routes to Tier 2 ($requestTier2ForNode). */ +import { isCharKindMarker, isParaKindMarker } from "./markerKind.utils"; import { BARE_OPENER_REGEX, CLOSER_FORM_REGEX, @@ -63,7 +64,6 @@ import { getEditableCallerText, getVisibleOpenMarkerText, ImmutableUnmatchedNode, - isMilestoneHeuristicName, leadingAttributeNames, MarkerLookup, MarkerNode, @@ -145,40 +145,6 @@ export interface MarkerEditContext extends Tier2Context { rebuildAttempted: Set; } -// Milestone-name heuristic shared with the fragment tokenizer (`isMilestoneHeuristicName`): -// only stylesheet-family milestone names (`\qt#-s/-e`, `\ts-s/-e`) plus annotation comment -// markers — see its doc comment for why bare `ts`/`t-s`/`t-e` and the z-prefix wildcard are -// deliberately excluded. Keeping one predicate here and in the tokenizer means Tier-1 kind -// guards and Tier-2 re-tokenization can never disagree about what is positionally a milestone. - -/** Same-positional-kind rule for paragraph openers. Stylesheet-first: - * a marker the effective sheet KNOWS classifies by its styleType; heuristics - * cover only markers absent from the sheet. Unknown markers stay as typed - * (Tier-1 renames to unknown markers stay in place). Exported for - * `tier2Rebuild.utils.ts`'s own-marker-prefix dedup, which needs the SAME - * stylesheet-first/unknown-as-paragraph classification `$buildParaFragment` - * already uses for the paragraph's own marker — a second, narrower - * `type === MarkerType.Paragraph` check there disagreed with it for any - * unknown/custom.sty marker. */ -export function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { - const clean = marker.replace(/^\+/, ""); - if (clean === "v" || clean === "c") return false; - const kind = getMarkerFn(clean)?.type; - if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Paragraph; - if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; - return true; -} - -/** Same-positional-kind rule for char openers (see isParaKindMarker). */ -function isCharKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { - const clean = marker.replace(/^\+/, ""); - if (clean === "v" || clean === "c") return false; - const kind = getMarkerFn(clean)?.type; - if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Character; - if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; - return true; -} - /** * Whether an OPENING glyph's current — possibly mid-edit — bytes have stopped meaning a BLOCK * marker, which is the only thing an unknown-marker paragraph's split ever rested on. diff --git a/packages/platform/src/editor/markerEdit/markerKind.utils.ts b/packages/platform/src/editor/markerEdit/markerKind.utils.ts new file mode 100644 index 000000000..03225f541 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/markerKind.utils.ts @@ -0,0 +1,48 @@ +/** + * The marker-edit engine's positional-KIND rules: given a marker name, is it a paragraph-kind + * marker, or a character-kind one? + * + * A leaf module by design. Both tiers ask the question — Tier 1 to decide whether a rename keeps + * a glyph in the same position, Tier 2's own-marker-prefix dedup to recognize a pasted line's own + * paragraph marker — and they must answer it identically or a rename and a rebuild disagree about + * what the same bytes mean. Its only inputs are a plain string and a stylesheet lookup, so it + * needs nothing from either tier, and keeping it out of both is what stops them importing each + * other for it. + * + * Milestone-name heuristic shared with the fragment tokenizer (`isMilestoneHeuristicName`): only + * stylesheet-family milestone names (`\qt#-s/-e`, `\ts-s/-e`) plus annotation comment markers — + * see its doc comment for why bare `ts`/`t-s`/`t-e` and the z-prefix wildcard are deliberately + * excluded. Keeping one predicate here and in the tokenizer means Tier-1 kind guards and Tier-2 + * re-tokenization can never disagree about what is positionally a milestone. + */ + +import { isMilestoneHeuristicName, MarkerLookup, MarkerType, NoteNode } from "shared"; + +/** + * Same-positional-kind rule for paragraph openers. Stylesheet-first: a marker the effective sheet + * KNOWS classifies by its styleType; heuristics cover only markers absent from the sheet. Unknown + * markers stay as typed (Tier-1 renames to unknown markers stay in place). + * + * `tier2Rebuild.utils.ts`'s own-marker-prefix dedup needs this exact rule too — the SAME + * stylesheet-first/unknown-as-paragraph classification `$buildParaFragment` already uses for the + * paragraph's own marker. A second, narrower `type === MarkerType.Paragraph` check there + * disagreed with it for any unknown/custom.sty marker. + */ +export function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { + const clean = marker.replace(/^\+/, ""); + if (clean === "v" || clean === "c") return false; + const kind = getMarkerFn(clean)?.type; + if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Paragraph; + if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; + return true; +} + +/** Same-positional-kind rule for char openers (see {@link isParaKindMarker}). */ +export function isCharKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { + const clean = marker.replace(/^\+/, ""); + if (clean === "v" || clean === "c") return false; + const kind = getMarkerFn(clean)?.type; + if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Character; + if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; + return true; +} diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index d0fc4e332..b9c27f590 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -11,7 +11,7 @@ */ import usjEditorAdaptor from "../adaptors/usj-editor.adaptor"; -import { isParaKindMarker } from "./markerEditTier1.utils"; +import { isParaKindMarker } from "./markerKind.utils"; import { UNTERMINATED_MARKER_TAIL } from "./markerName.pattern"; import { $serializeExpandedNoteContent, @@ -1051,7 +1051,7 @@ const LEADING_MARKER_LITERAL = /^\\\+?([\w-]+)(?:\*|[ \u00A0])/; * `$buildParaFragment`'s `isPasteRebuild` parameter; typing the same shape keeps the engine's * existing split-with-empty behavior, which this function never runs for. * - * The embedded literal's marker-kind check uses `isParaKindMarker` (markerEditTier1.utils.ts) — + * The embedded literal's marker-kind check uses `isParaKindMarker` (markerKind.utils.ts) — * stylesheet-first, UNKNOWN-AS-PARAGRAPH, the same classification `$buildParaFragment`'s own guard * below applies to the paragraph's own marker — rather than a bare `type === MarkerType.Paragraph` * comparison, which rejected every unknown/custom.sty marker and left the stray-empty-paragraph diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index c7c6ab763..a0654c1c6 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -891,16 +891,20 @@ describe("paste normalization ($handlePasteForStandardView)", () => { // gets its fresh paragraphs correctly prefixed when it goes through MarkerEditPlugin's own // registration, which arms `context.splitExpected` before inserting — see "multi-line paste // interplay" below. + let handled: boolean | undefined; await act(async () => editor.update(() => { text.select(7, 7); // between "before " and "after" - editor.dispatchCommand( + handled = editor.dispatchCommand( PASTE_COMMAND, pasteEvent({ "text/html": "

one two

three
four

" }).event, ); }), ); + // Claimed, so Lexical's own html import never gets the payload. The content assertion below + // cannot tell the two paths apart on its own — both would produce plausible text. + expect(handled).toBe(true); editor.getEditorState().read(() => { // Block boundaries and
become newlines, so "two"/"three" don't fuse into one word — // and each newline is then replayed as a real paragraph split (see the line-replay diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index c3e6150e9..dbecc4dc4 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -104,16 +104,15 @@ export function htmlPasteText(html: string): string { /** The text of a paste, read the one way every `PASTE_COMMAND` claim in this editor reads it. */ export interface PastePayload { - /** `text/plain`, line endings normalized. */ - plainText: string; - /** `text/html` as the clipboard carries it — markup, not text; an NBSP may be a ` ` here. */ - html: string; - /** `text/html` decoded to text ({@link htmlPasteText}), line endings normalized. */ - htmlText: string; /** * What a claim replays: `text/plain` when it carries anything, else the decoded `text/html`. * Some sources (word processors, intermediaries) ship html alone, and those pastes otherwise * reach the generic handling this editor's claims exist to pre-empt. + * + * The unresolved carriers (`text/plain`, the raw `text/html`, and its decoded text) are + * deliberately not exposed alongside it: every claim must replay the SAME bytes, and a second + * carrier within reach is an invitation for one of them to pick differently. A claim that ever + * genuinely needs one can read the clipboard for it, at its own site, with its own reason. */ text: string; /** @@ -121,7 +120,9 @@ export interface PastePayload { * (`application/x-lexical-editor`), whose real nodes a line-by-line replay would flatten. * Deliberately NOT acted on here: the claims disagree about it on purpose — the in-note `\fp` * claim covers internal pastes (an internal multi-paragraph copy is exactly the split it - * prevents), while the char-stack and NBSP claims decline them — so each one applies its own + * prevents), the char-stack claim declines them outright, and the Standard-view claim declines + * them EXCEPT when the selection touches an attribute display run, where plain-text insertion + * is the only safe shape whatever else the clipboard carries — so each one applies its own * rule, in view, at its own site. */ isInternal: boolean; @@ -132,13 +133,14 @@ export interface PastePayload { * clipboard at all (a KeyboardEvent-shaped dispatch, or an event whose data store is * inaccessible), which every claim reads as "not mine". * - * Three handlers race on `PASTE_COMMAND` — the in-note `\fp` claim at CRITICAL, the NBSP display - * normalization and the char-stack replay at HIGH — and they must agree byte-for-byte on what was - * pasted, or the same clipboard is one thing to one of them and another to the next. So the - * extraction lives here once: the jsdom-safe duck-check for the clipboard (jsdom implements no - * `ClipboardEvent`, so `instanceof` against the undefined global throws), the plain-then-decoded- - * html preference, and line-ending normalization BEFORE any caller tests for a line break — so - * `\r\n` and bare-`\r` clipboards break correctly and no `\r` ever reaches content on any path. + * Four handlers race on `PASTE_COMMAND` — the in-note `\fp` claim at CRITICAL, the Standard-view + * external-paste claim and the char-stack line replay at HIGH, the paragraph-split arm at LOW — + * and they must agree byte-for-byte on what was pasted, or the same clipboard is one thing to one + * of them and another to the next. So the extraction lives here once: the jsdom-safe duck-check + * for the clipboard (jsdom implements no `ClipboardEvent`, so `instanceof` against the undefined + * global throws), the carrier choice below, and line-ending normalization BEFORE any caller tests + * for a line break — so `\r\n` and bare-`\r` clipboards break correctly and no `\r` ever reaches + * content on any path. */ export function getPastePayload( event: PasteCommandType | null | undefined, @@ -152,10 +154,19 @@ export function getPastePayload( const plainText = normalizeLineEndings(clipboardData.getData("text/plain")); const html = clipboardData.getData("text/html"); const htmlText = html ? normalizeLineEndings(htmlPasteText(html)) : ""; + // The carrier is chosen by PRESENCE — plain text whenever the clipboard carries any — and not + // by which carrier an NBSP survived in. The stronger rule is genuinely better for a foreign + // clipboard: a source whose `text/plain` collapsed ` ` to a plain space still has the real + // NBSP in its `text/html`, and preferring html there would keep a data-NBSP this loses. But it + // is not usable here, because it inverts on this editor's OWN copy: Standard view's `text/plain` + // deliberately carries no NBSP at all (display ones invert to spaces, a data NBSP displays and + // copies as `~`), while its `text/html` still ships NBSPs — so "the plain text has no NBSP" is + // TRUE of every P10 copy, and an + // NBSP-presence test would route P10's own round trip through html, whose decoded text drops a + // collapsed note's caller entirely (it rides as a `data-caller` attribute, never as text). A + // lost note caller on the editor's own copy is a worse, far likelier loss than a foreign + // clipboard's data-NBSP. Recorded in the semantics doc's accepted asymmetries. return { - plainText, - html, - htmlText, text: plainText || htmlText, isInternal: !!clipboardData.getData("application/x-lexical-editor"), }; @@ -308,8 +319,9 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { * two spaces, matching what two individual Enter-less keystrokes over an attribute run would * produce — attribute values are single-line, so there is no multi-line attribute byte shape to * collapse INTO). `text` arrives with bare `\n` line endings ({@link getPastePayload} normalizes - * them for every paste claim) — no `\r` reaches this function. The result is inserted via the exact `selection.insertText` a keystroke - * uses — no chapter/book-id strip, no positional NBSP mapping, no marker tokenization, all of which + * them for every paste claim) — no `\r` reaches this function. The result is inserted via the exact + * `selection.insertText` a keystroke uses — no chapter/book-id strip, no positional NBSP mapping, + * no marker tokenization, all of which * exist for BODY content's marker-recognition and whitespace-display invariants and apply to none * of it: a pasted `\c 5` or `\p` here is literal value text, not a structural marker, and * `$displayWhitespaceTransform` (this file) already skips textType "attribute" nodes outright, so a @@ -397,6 +409,10 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * and still wins for a multi-line payload whose selection touches EXPANDED note content — an * attribute run that happens to sit inside an expanded note's content is reached by this handler * (and this suspension) only when that in-note claim itself declines. + * + * Mutating: call inside `editor.update()`. It inserts text, removes the selected range, and + * dispatches `INSERT_PARAGRAPH_COMMAND` against the editor it reads from `$getEditor()` — none of + * which has an editor state to act on outside an update. */ export function $handlePasteForStandardView( event: ClipboardEvent | null | undefined, From 45d97fde97ca1e33db3ab95975a39160fed6149b Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 26 Aug 2026 23:12:30 -0500 Subject: [PATCH 04/44] docs: reconcile the clipboard semantics with the per-flavor NBSP inversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The copy flavors no longer split into "text/plain inverts, everything else ships the display form". text/html inverts too, collapse-awarely: an interior single display-NBSP becomes the plain space it stands for, while a run of 2+ and a fragment edge keep NBSP because a rich-text consumer collapses runs and drops edge whitespace. Only the internal Lexical flavor is still untouched, so an internal paste round-trips the display form exactly. That also sharpens why the paste carrier is chosen by presence rather than by where an NBSP survived: Standard view's own text/plain still carries no NBSP at all, while its text/html now carries them precisely where a plain space would be destroyed — so an NBSP-presence test would route every P10 round trip through html and lose the note caller. Co-Authored-By: Claude Fable 5 --- docs/superpowers/specs/2026-08-06-clipboard-semantics.md | 5 +++-- .../src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 32b1d79d0..6a0f53f5e 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -16,7 +16,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Collapsed notes** contribute their full (visually hidden) `\f …\f*` bytes. "What you copy" > "what you see", for fidelity. This is documented, not changed. -- **Display-NBSP handling:** Display-NBSP → space; data-NBSP → `~`; `text/html` and `application/x-lexical-editor` stay unnormalized (existing, pinned). +- **Display-NBSP handling, per flavor:** `text/plain` inverts EVERY display-NBSP to a plain space and leaves a data-NBSP as the `~` the display already shows. `text/html` inverts too, but collapse-awarely (`invertDisplayNbspInHtml`, `whitespaceDisplay.plugin.utils.ts`): an interior single display-NBSP becomes the plain space it stands for, while a run of 2+ and a single NBSP at the fragment's leading/trailing edge stay NBSP, because a rich-text consumer collapses space runs and drops edge whitespace — one selection must not paste as normal spaces into a text editor and as non-breaking spaces into a word processor. Runs and edges are judged on the fragment's concatenated text, not per markup span. `application/x-lexical-editor` stays unnormalized, so a paste back into a Standard-view editor round-trips the display form exactly. A genuine data-NBSP takes no part in either inversion: it is a literal `~` on screen, so both flavors ship `~` and decode to the same document text. - **Byte fidelity around note-internal markers:** Copied text must not contain spaces the source USFM lacks. Live repro (2026-08-07): copying `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*Den` produced `… \xo* \xt 2Cor 4:6\xt* \x*Den` — phantom spaces after each closing marker (display-separator NBSPs leaking through the blanket NBSP→space mapping). The copy walker must be source-faithful, mirroring the serialization inverse, not a blanket `replaceAll`. @@ -271,7 +271,7 @@ The following items are recorded as deferred and not addressed in this plan: 13. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 5 above. -14. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. +14. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. ## Test Mapping (S1–S7) @@ -287,6 +287,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; six known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"` fixture (tight spacing, no flanking spaces) is swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | +| **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | | **E2E** Real-browser clipboard round trip (Standard view, written, not yet run in this task) | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index dbecc4dc4..14e88b0e0 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -160,7 +160,8 @@ export function getPastePayload( // NBSP in its `text/html`, and preferring html there would keep a data-NBSP this loses. But it // is not usable here, because it inverts on this editor's OWN copy: Standard view's `text/plain` // deliberately carries no NBSP at all (display ones invert to spaces, a data NBSP displays and - // copies as `~`), while its `text/html` still ships NBSPs — so "the plain text has no NBSP" is + // copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not + // survive a rich consumer ({@link invertDisplayNbspInHtml}) — so "the plain text has no NBSP" is // TRUE of every P10 copy, and an // NBSP-presence test would route P10's own round trip through html, whose decoded text drops a // collapsed note's caller entirely (it rides as a `data-caller` attribute, never as text). A From a98150fe7a74a464697885326889f697a0a73039 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 26 Aug 2026 23:54:25 -0500 Subject: [PATCH 05/44] fix: guard the empty copy on the command, not in front of the dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking the selection before dispatching read the COMMITTED editor state, which Lexical is a microtask behind — so a selection made earlier in the same tick was invisible and the copy was silently withheld. That is the ordinary shape of the public EditorRef.copy() path: select something programmatically, then copy it. Reading the live state was not a drop-in fix either, since the context menu invokes its options from inside editor.update(). The guard is now a COPY_COMMAND/CUT_COMMAND listener registered above @lexical/rich-text's fallback and below every feature handler. A command listener runs inside the update, where the selection is authoritative for pending and committed state alike, so the staleness class is gone rather than narrowed — and one registration covers the shortcuts, the context menu and the editor ref, since all of them dispatch the same command. The three Standard-view pins asserted nothing that could fail. The dispatch's return value is true either way (rich-text always claims), the clipboard-module spy cannot see a call made through the externalized rich-text import, and the execCommand assertion was masked by @lexical/clipboard's module-level latency timer left set by the preceding test. They now watch for the dispatch reaching past the claim — immune to module state and test order — and drain that timer, and all three fail in a whole-file run when the claim is reverted. Also pins the node-selection carve-out (a copy that has content must still happen), the context-menu and editor-ref legs, records the empty-selection no-op on the public API's TSDoc, and marks the browser's drag-over-a-read-only- block selection shape as the assumption jsdom cannot verify. Co-Authored-By: Claude Fable 5 --- .../specs/2026-08-06-clipboard-semantics.md | 2 +- .../src/plugins/usj/ClipboardPlugin.test.tsx | 127 +++++++++++------- .../src/plugins/usj/ClipboardPlugin.tsx | 36 ++--- .../plugins/usj/ContextMenuPlugin.test.tsx | 109 +++++++++++++++ .../src/plugins/usj/ContextMenuPlugin.tsx | 5 +- .../src/plugins/usj/clipboard.utils.ts | 79 ++++++----- .../editor/CommandMenuPlugin.gate.test.tsx | 90 ++++++++++--- packages/platform/src/editor/Editor.tsx | 6 +- packages/platform/src/editor/editor.model.ts | 10 +- .../whitespaceDisplay.plugin.utils.test.tsx | 107 ++++++++++++--- .../whitespaceDisplay.plugin.utils.ts | 48 ++++--- 11 files changed, 450 insertions(+), 169 deletions(-) create mode 100644 libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 6a0f53f5e..a4bb5119f 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -22,7 +22,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Multi-block selections:** One `\n` between blocks; a selection starting mid-paragraph omits that paragraph's own `\p ` glyph (matches P9's rendered-text copy). Pinned, not changed. -- **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection *before* calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it *likelier*: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers: `ClipboardPlugin`/`ContextMenuPlugin`/`EditorRef` (via `copySelection`/`cutSelection`, `clipboard.utils.ts`, shared-react) do not synthesize a copy for a selection that cannot produce content, and do not `preventDefault` the key either — the browser's own copy of an empty selection is itself a no-op, and letting it run is what "untouched" means; and `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into `@lexical/rich-text`'s fallback. A real clipboard event is still declined at a collapsed caret (the browser already writes nothing). A NodeSelection is still declined too — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it — the shape a browser resolves a drag or click over a `contentEditable=false` block into, since it will not place a selection endpoint inside one. Such a selection reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view, where no Standard-view handler is registered at all), and `ClipboardPlugin.test.tsx` (shared-react). +- **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection *before* calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it *likelier*: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers, both of which decide INSIDE the update where the selection is authoritative — a check in front of the dispatch would read the last COMMITTED selection, which Lexical is a microtask behind, and would silently no-op a `copy()` made in the same tick as the selection it is copying. (1) `registerEmptyCopyGuard` (`clipboard.utils.ts`, shared-react), a `COPY_COMMAND`/`CUT_COMMAND` listener `ClipboardPlugin` registers at `COMMAND_PRIORITY_LOW` — above `@lexical/rich-text`'s EDITOR fallback, which is what synthesizes the event, and below every feature handler. One registration covers every leg (the keyboard shortcuts, the context menu's Cut/Copy, `EditorRef.copy()`/`.cut()`), since all of them dispatch the same command against the same editor. (2) `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into that fallback. A real clipboard event is still declined by layer 2 at a collapsed caret (the browser already writes nothing, and declining leaves it un-`preventDefault`ed). A NodeSelection is claimed by neither layer — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it. Such a selection — measured at the Lexical level as two ELEMENT points on the parent paragraph either side of the block — reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. **Unverified assumption:** that a real drag or click over a `contentEditable=false` block is what a browser resolves INTO that selection shape (on the reasoning that it will not place a selection endpoint inside one). jsdom cannot observe DOM-range → Lexical selection resolution — the same boundary S6 already records for the WI-2 quirk — so this needs a real-browser check before it is treated as fact. If a browser instead leaves the caret collapsed beside the block, the leak is still closed (nothing is written) but the user gets nothing rather than the figure, which would be a selection-affordance gap to file separately. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, including the node-selection carve-out, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view where no Standard-view handler is registered at all, and through `EditorRef.copy()`/`.cut()`), and shared-react's `ClipboardPlugin.test.tsx` (the shortcuts, and the same-tick select-then-copy case) and `ContextMenuPlugin.test.tsx` (the menu leg). - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). diff --git a/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx b/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx index 6151b3f7a..cc061543b 100644 --- a/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx +++ b/libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx @@ -1,25 +1,24 @@ import { ClipboardPlugin } from "./ClipboardPlugin"; +import { copySelection } from "./clipboard.utils"; import { baseTestEnvironment } from "./react-test.utils"; import { act } from "@testing-library/react"; import { + $createNodeSelection, $createTextNode, $getRoot, - COMMAND_PRIORITY_CRITICAL, - COPY_COMMAND, - CUT_COMMAND, + $setSelection, LexicalEditor, TextNode, } from "lexical"; -import { $createParaNode } from "shared"; +import { $createImmutableTypedTextNode, $createParaNode } from "shared"; /** - * `document.execCommand` is the mechanism a clipboard write ultimately runs through when no real - * clipboard event exists: `@lexical/clipboard` points the DOM selection at a hidden placeholder it - * appends to the editor and calls `execCommand("copy")` to provoke one. jsdom implements no - * `execCommand` at all, so these tests install a spy in its place — a call to it means a clipboard - * write reached the browser, and no call means the clipboard was left exactly as the user left it. - * That is the observable these tests pin: what does or does not get written, never the placeholder - * itself, which lives in `@lexical/clipboard` and is not ours to depend on. + * `document.execCommand("copy")` is how a clipboard write reaches the browser when there is no real + * clipboard event to fill in: `@lexical/clipboard` points the DOM selection at a hidden placeholder + * element it appends to the editor and runs it to provoke one. jsdom implements no `execCommand` at + * all, so these tests install a spy in its place — **called means a write reached the browser, not + * called means the clipboard is untouched**. That is the observable throughout; the placeholder's + * own content belongs to `@lexical/clipboard` and is never asserted on here. */ let execCommand: ReturnType; @@ -39,7 +38,13 @@ beforeEach(() => { }); }); -afterEach(() => { +afterEach(async () => { + // `@lexical/clipboard` keeps a MODULE-level timer handle while it waits for the clipboard event + // its `execCommand` call should provoke, and refuses to start another copy until that handle + // clears. A test that reaches the real copy path therefore silences the `execCommand` assertion + // in the NEXT test unless the window is drained here — which would make a failure show up only in + // whichever test happens to run first. The window is `EVENT_LATENCY`, 50ms. + await new Promise((resolve) => setTimeout(resolve, 60)); Reflect.deleteProperty(document, "execCommand"); vi.unstubAllGlobals(); }); @@ -79,76 +84,94 @@ async function pressShortcut( return event; } -/** - * Watches a clipboard command without claiming it, so a regression runs the whole real chain — - * `@lexical/rich-text`'s fallback and the synthesized-copy machinery behind it — and trips the - * `execCommand` assertion instead of being masked by the watcher itself. - */ -function watchCommand(editor: LexicalEditor, command: typeof COPY_COMMAND) { - const seen = vi.fn(() => false); - const unregister = editor.registerCommand(command, seen, COMMAND_PRIORITY_CRITICAL); - return { seen, unregister }; -} - describe("ClipboardPlugin — copy/cut with nothing selected", () => { it("leaves the clipboard untouched on Ctrl+C at a collapsed caret", async () => { const { editor, text } = await clipboardEnvironment(); await act(async () => editor.update(() => text.select(3, 3))); - const { seen, unregister } = watchCommand(editor, COPY_COMMAND); - const event = await pressShortcut(editor, "c"); - unregister(); + await pressShortcut(editor, "c"); - // No copy is synthesized at all, so nothing can be written over what the clipboard holds. expect(execCommand).not.toHaveBeenCalled(); - expect(seen).not.toHaveBeenCalled(); - // The browser's own copy is left to run: for an empty selection it writes nothing, which is - // the behavior this plugin's `preventDefault` would otherwise have suppressed. - expect(event.defaultPrevented).toBe(false); }); it("leaves the clipboard untouched on Ctrl+X at a collapsed caret, and removes nothing", async () => { const { editor, text } = await clipboardEnvironment(); await act(async () => editor.update(() => text.select(3, 3))); - const { seen, unregister } = watchCommand(editor, CUT_COMMAND); - const event = await pressShortcut(editor, "x"); - unregister(); + await pressShortcut(editor, "x"); expect(execCommand).not.toHaveBeenCalled(); - expect(seen).not.toHaveBeenCalled(); - expect(event.defaultPrevented).toBe(false); editor.getEditorState().read(() => expect(text.getTextContent()).toBe("In the beginning")); }); }); describe("ClipboardPlugin — copy/cut with a selection", () => { - it("copies on Ctrl+C and claims the key", async () => { + it("copies on Ctrl+C", async () => { const { editor, text } = await clipboardEnvironment(); await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); - // Claimed here so the copy stops at this assertion instead of running the real clipboard - // machinery, whose synthesized-event timer would outlive the test. - const seen = vi.fn(() => true); - const unregister = editor.registerCommand(COPY_COMMAND, seen, COMMAND_PRIORITY_CRITICAL); - const event = await pressShortcut(editor, "c"); - unregister(); + await pressShortcut(editor, "c"); - expect(seen).toHaveBeenCalledTimes(1); - expect(event.defaultPrevented).toBe(true); + expect(execCommand).toHaveBeenCalledWith("copy"); }); - it("cuts on Ctrl+X and claims the key", async () => { + it("cuts on Ctrl+X", async () => { const { editor, text } = await clipboardEnvironment(); await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); - const seen = vi.fn(() => true); - const unregister = editor.registerCommand(CUT_COMMAND, seen, COMMAND_PRIORITY_CRITICAL); - const event = await pressShortcut(editor, "x"); - unregister(); + await pressShortcut(editor, "x"); + + expect(execCommand).toHaveBeenCalledWith("copy"); + }); + + it("copies a node selection — the guard is about having nothing to copy, not about ranges", async () => { + // A node selection covers real content and is not collapsed, so it copies like any other. The + // guard tests "is there anything here", NOT "is this a range": narrowing it to range selections + // would silently swallow this copy. + const { editor } = await clipboardEnvironment(); + await act(async () => + editor.update(() => { + const decorator = $createImmutableTypedTextNode("marker", "\\p"); + $getRoot().getFirstChild()?.insertBefore?.($createParaNode("p").append(decorator)); + const nodeSelection = $createNodeSelection(); + nodeSelection.add(decorator.getKey()); + $setSelection(nodeSelection); + }), + ); + + await pressShortcut(editor, "c"); + + expect(execCommand).toHaveBeenCalledWith("copy"); + }); +}); + +describe("ClipboardPlugin — the guard reads the live selection, not the committed one", () => { + // Lexical commits on a microtask, so the last COMMITTED selection lags a selection made earlier + // in the same synchronous tick. A guard that read the committed state would see "nothing + // selected" here and silently copy nothing — and this is the ordinary shape of the public + // `EditorRef.copy()` path: select something programmatically, then copy it. + it("copies a selection made earlier in the same synchronous tick", async () => { + const { editor, text } = await clipboardEnvironment(); + + await act(async () => { + editor.update(() => text.select(0, text.getTextContentSize())); + copySelection(editor); + }); + + expect(execCommand).toHaveBeenCalledWith("copy"); + }); + + it("copies a selection made inside the same editor.update() as the copy call", async () => { + const { editor, text } = await clipboardEnvironment(); + + await act(async () => + editor.update(() => { + text.select(0, text.getTextContentSize()); + copySelection(editor); + }), + ); - expect(seen).toHaveBeenCalledTimes(1); - expect(event.defaultPrevented).toBe(true); + expect(execCommand).toHaveBeenCalledWith("copy"); }); }); diff --git a/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx b/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx index f77f95f4c..cbcca1856 100644 --- a/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/ClipboardPlugin.tsx @@ -3,9 +3,10 @@ import { cutSelection, pasteSelection, pasteSelectionAsPlainText, + registerEmptyCopyGuard, } from "./clipboard.utils"; import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext"; -import { IS_APPLE } from "@lexical/utils"; +import { IS_APPLE, mergeRegister } from "@lexical/utils"; import { useEffect } from "react"; export function ClipboardPlugin(): null { @@ -16,14 +17,12 @@ export function ClipboardPlugin(): null { const { key, shiftKey, metaKey, ctrlKey, altKey } = event; if (!(IS_APPLE ? metaKey : ctrlKey) || altKey) return; - // Copy/cut suppress the browser's own handling only when this plugin actually replaces it. - // With nothing selected there is nothing to replace, and leaving the browser to it is what - // keeps the clipboard untouched — see `copySelection` for what happens when a copy with no - // content behind it is synthesized anyway. if (!shiftKey && key.toLowerCase() === "c") { - if (copySelection(editor)) event.preventDefault(); + event.preventDefault(); + copySelection(editor); } else if (!shiftKey && key.toLowerCase() === "x") { - if (cutSelection(editor)) event.preventDefault(); + event.preventDefault(); + cutSelection(editor); } else if (key.toLowerCase() === "v") { event.preventDefault(); if (shiftKey) pasteSelectionAsPlainText(editor); @@ -31,15 +30,20 @@ export function ClipboardPlugin(): null { } }; - return editor.registerRootListener( - (rootElement: HTMLElement | null, prevRootElement: HTMLElement | null) => { - if (prevRootElement !== null) { - prevRootElement.removeEventListener("keydown", onKeyDown); - } - if (rootElement !== null) { - rootElement.addEventListener("keydown", onKeyDown); - } - }, + return mergeRegister( + // Every copy/cut this plugin's shortcuts synthesize — and every one the context menu or an + // editor ref synthesizes against the same editor — passes through this guard. + registerEmptyCopyGuard(editor), + editor.registerRootListener( + (rootElement: HTMLElement | null, prevRootElement: HTMLElement | null) => { + if (prevRootElement !== null) { + prevRootElement.removeEventListener("keydown", onKeyDown); + } + if (rootElement !== null) { + rootElement.addEventListener("keydown", onKeyDown); + } + }, + ), ); }, [editor]); diff --git a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx new file mode 100644 index 000000000..49ec6c863 --- /dev/null +++ b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx @@ -0,0 +1,109 @@ +import { ClipboardPlugin } from "./ClipboardPlugin"; +import { ContextMenuPlugin } from "./ContextMenuPlugin"; +import { baseTestEnvironment } from "./react-test.utils"; +import { act } from "@testing-library/react"; +import { $createTextNode, $getRoot, LexicalEditor, TextNode } from "lexical"; +import { $createParaNode } from "shared"; + +/** + * The context menu's Cut/Copy dispatch the same commands the keyboard shortcuts do, so they are + * covered by the same empty-copy guard (`registerEmptyCopyGuard`, mounted by `ClipboardPlugin` — + * both plugins ship together in every editor that mounts either). This pins that the leg really + * does go through it, rather than dispatching around it. + * + * Note where `onSelect` runs: inside `editor.update()` (see the plugin's Enter handler). That is + * why the guard has to live on the COMMAND and read the live selection — a check in front of the + * dispatch reading the last committed state would be both stale and, via `editor.read()`, unsafe + * to call from there. + * + * `document.execCommand("copy")` is the observable, as everywhere else in this suite: called means + * a clipboard write reached the browser, not called means the clipboard is untouched. + */ +let execCommand: ReturnType; + +beforeEach(() => { + execCommand = vi.fn(() => true); + Object.defineProperty(document, "execCommand", { + configurable: true, + writable: true, + value: execCommand, + }); +}); + +afterEach(async () => { + // Drain `@lexical/clipboard`'s module-level `EVENT_LATENCY` (50ms) handle, which otherwise makes + // a test that reached the real copy path silence the next test's assertion. + await new Promise((resolve) => setTimeout(resolve, 60)); + Reflect.deleteProperty(document, "execCommand"); +}); + +async function contextMenuEnvironment(): Promise<{ editor: LexicalEditor; text: TextNode }> { + let text: TextNode | undefined; + const { editor } = await baseTestEnvironment( + () => { + text = $createTextNode("In the beginning"); + $getRoot().append($createParaNode("p").append(text)); + }, + <> + + + , + ); + if (!text) throw new Error("expected the initial text node to exist"); + return { editor, text }; +} + +/** + * Opens the context menu over the editor's content and activates the option at `index` the way a + * keyboard user would (the plugin's own arrow/Enter handling), rather than by reaching for the + * rendered menu's markup. Built-in order: Cut, Copy, Paste, Paste as Plain Text. + */ +async function chooseContextMenuOption(editor: LexicalEditor, index: number): Promise { + const rootElement = editor.getRootElement(); + const target = rootElement?.firstElementChild; + if (!target) throw new Error("expected the editor to have rendered content to right-click"); + await act(async () => { + target.dispatchEvent(new MouseEvent("contextmenu", { bubbles: true, cancelable: true })); + }); + for (let step = 0; step <= index; step++) { + await act(async () => { + document.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowDown", bubbles: true })); + }); + } + await act(async () => { + document.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); + }); +} + +const COPY_OPTION = 1; +const CUT_OPTION = 0; + +describe("ContextMenuPlugin — Cut/Copy go through the empty-copy guard", () => { + it("Copy with a collapsed caret leaves the clipboard untouched", async () => { + const { editor, text } = await contextMenuEnvironment(); + await act(async () => editor.update(() => text.select(3, 3))); + + await chooseContextMenuOption(editor, COPY_OPTION); + + expect(execCommand).not.toHaveBeenCalled(); + }); + + it("Cut with a collapsed caret leaves the clipboard untouched and removes nothing", async () => { + const { editor, text } = await contextMenuEnvironment(); + await act(async () => editor.update(() => text.select(3, 3))); + + await chooseContextMenuOption(editor, CUT_OPTION); + + expect(execCommand).not.toHaveBeenCalled(); + editor.getEditorState().read(() => expect(text.getTextContent()).toBe("In the beginning")); + }); + + it("Copy with a selection copies — the menu leg is really wired to the command", async () => { + const { editor, text } = await contextMenuEnvironment(); + await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); + + await chooseContextMenuOption(editor, COPY_OPTION); + + expect(execCommand).toHaveBeenCalledWith("copy"); + }); +}); diff --git a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx index 7428bdb02..d7e77ea04 100644 --- a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx @@ -138,8 +138,9 @@ export function ContextMenuPlugin({ const options = useMemo(() => { const builtIn = [ - // Cut/Copy are no-ops with nothing selected rather than dispatches the browser would have to - // synthesize a clipboard event for — see `copySelection`. They are not disabled instead, + // Cut/Copy with nothing selected leave the clipboard alone rather than writing a placeholder + // over it — `ClipboardPlugin`'s guard claims the command (see `registerEmptyCopyGuard`), so + // this plugin needs no selection check of its own. They are not disabled in that case, // because this option list is built once per editor rather than per menu opening, so its // `isDisabled` flags cannot track the live selection. new ContextMenuOption(`Cut`, { diff --git a/libs/shared-react/src/plugins/usj/clipboard.utils.ts b/libs/shared-react/src/plugins/usj/clipboard.utils.ts index a723b873c..45544b7d7 100644 --- a/libs/shared-react/src/plugins/usj/clipboard.utils.ts +++ b/libs/shared-react/src/plugins/usj/clipboard.utils.ts @@ -1,50 +1,69 @@ -import { $getSelection, COPY_COMMAND, CUT_COMMAND, LexicalEditor, PASTE_COMMAND } from "lexical"; +import { mergeRegister } from "@lexical/utils"; +import { + $getSelection, + COMMAND_PRIORITY_LOW, + COPY_COMMAND, + CUT_COMMAND, + LexicalEditor, + PASTE_COMMAND, +} from "lexical"; function cleanupText(text: string): string { return text.replaceAll("\t", " "); } /** - * Whether the editor's selection holds content a copy could put on the clipboard: a selection - * exists and covers something. A collapsed caret — or no selection at all — does not. + * Whether the selection holds content a copy could put on the clipboard: one exists and covers + * something. A collapsed caret — or no selection at all — does not. * - * This is the precondition for SYNTHESIZING a copy (see {@link copySelection}), not a general - * "can the user copy" predicate: a real, browser-generated copy needs no such check because the - * browser already knows an empty selection copies nothing. + * Reads the ACTIVE editor state, so it must be called inside an update or a command listener. That + * is the whole point: the committed state is a microtask behind, so a selection made earlier in the + * same tick is invisible to `editor.getEditorState().read()` — a `copy()` right after a + * programmatic select would read "nothing selected" and silently do nothing. */ -export function canCopySelection(editor: LexicalEditor): boolean { - return editor.getEditorState().read(() => { - const selection = $getSelection(); - return !!selection && !selection.isCollapsed(); - }); +export function $hasCopyableSelection(): boolean { + const selection = $getSelection(); + return !!selection && !selection.isCollapsed(); } /** - * Copies the selection by dispatching `COPY_COMMAND` with no event of its own, and reports whether - * it did. Callers that suppressed the browser's own copy to get here should suppress it only when - * this returns `true`. + * Stops a copy or cut that has nothing to copy, wherever it was dispatched from. + * + * A copy with no clipboard event of its own — the keyboard shortcuts below, the context menu's + * Cut/Copy, an editor-ref `copy()` call — has to have one synthesized before anything can be + * written: `@lexical/clipboard` appends a hidden placeholder element to the editor, points the DOM + * selection at it, and runs `document.execCommand("copy")` to provoke a real clipboard event it can + * fill in. With nothing selected, that filling step declines — and declines BEFORE suppressing the + * browser's own copy — so the browser copies what it was pointed at, the placeholder, and the user + * loses whatever the clipboard already held to a character that was never in the document. + * + * So the command is CLAIMED here and does nothing, which is what copying an empty selection means. + * The guard lives on the command rather than in front of each dispatch for two reasons: a command + * listener runs inside the update, where the selection is authoritative for both pending and + * committed state, and every dispatcher is covered by the one registration. * - * Nothing is dispatched for a selection that cannot produce clipboard content. Without an event, - * the copy has to be synthesized: `@lexical/clipboard` selects a hidden placeholder element it - * appends to the editor and runs `document.execCommand("copy")` to provoke a real clipboard event - * it can fill in. That filling step declines an empty selection — and declines it BEFORE - * suppressing the browser's own copy — so the browser copies what it was pointed at, the - * placeholder, and the user loses whatever the clipboard already held to a character that was - * never in the document. Copying nothing must leave the clipboard alone, which is exactly what - * not dispatching achieves. + * Registered at `COMMAND_PRIORITY_LOW` — above `@lexical/rich-text`'s own copy/cut fallback at + * EDITOR, which is the thing that synthesizes the event, and below every feature handler, so a view + * with its own clipboard payload (Standard view's USFM copy) still claims first and this never runs. */ -export const copySelection = (editor: LexicalEditor): boolean => { - if (!canCopySelection(editor)) return false; +export function registerEmptyCopyGuard(editor: LexicalEditor): () => void { + const $claimWhenNothingToCopy = () => !$hasCopyableSelection(); + return mergeRegister( + editor.registerCommand(COPY_COMMAND, $claimWhenNothingToCopy, COMMAND_PRIORITY_LOW), + editor.registerCommand(CUT_COMMAND, $claimWhenNothingToCopy, COMMAND_PRIORITY_LOW), + ); +} + +/** Copies the selection. Dispatched unconditionally; an empty selection copies nothing because + * {@link registerEmptyCopyGuard} claims the command, not because the dispatch is withheld. */ +export const copySelection = (editor: LexicalEditor) => { editor.dispatchCommand(COPY_COMMAND, null); - return true; }; -/** Cuts the selection, under the same "nothing selected, nothing dispatched" rule as - * {@link copySelection} — a cut copies first, so it reaches the identical synthesized-copy path. */ -export const cutSelection = (editor: LexicalEditor): boolean => { - if (!canCopySelection(editor)) return false; +/** Cuts the selection, under the same guard as {@link copySelection} — a cut copies first, so it + * reaches the identical synthesized-copy path. */ +export const cutSelection = (editor: LexicalEditor) => { editor.dispatchCommand(CUT_COMMAND, null); - return true; }; export const pasteSelection = (editor: LexicalEditor) => { diff --git a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx index 5e3d44c48..afbcaa619 100644 --- a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx +++ b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx @@ -3,7 +3,7 @@ import { EditorRef } from "./editor.model"; import { execCommandSpy } from "./markerEdit/markerEdit.test-helpers"; import { act, render } from "@testing-library/react"; import { Usj } from "@eten-tech-foundation/scripture-utilities"; -import { createRef } from "react"; +import { createRef, RefObject } from "react"; import { $createPoint, $createRangeSelection, @@ -47,7 +47,9 @@ const sampleUsj: Usj = { /** Renders the real Editor in `viewMode` and returns its underlying Lexical editor. The editor * is captured through a child `EditorRefPlugin` (which `` renders inside its composer) * rather than a `.__lexicalEditor` DOM reach-in. */ -async function renderEditor(viewMode: string): Promise { +async function renderEditorWithRef( + viewMode: string, +): Promise<{ editor: LexicalEditor; ref: RefObject }> { const ref = createRef(); const lexicalRef = createRef(); await act(async () => { @@ -58,7 +60,11 @@ async function renderEditor(viewMode: string): Promise { ); }); if (!lexicalRef.current) throw new Error("lexical editor was not captured"); - return lexicalRef.current; + return { editor: lexicalRef.current, ref }; +} + +async function renderEditor(viewMode: string): Promise { + return (await renderEditorWithRef(viewMode)).editor; } /** @@ -283,24 +289,31 @@ describe("MarkerEditPlugin's Standard-view clipboard handlers do not leak into F * The empty-copy rule is not Standard view's. It lives in `ClipboardPlugin` (shared-react), which * every view mounts, and this view has no Standard-view copy handler registered at all — so a * regression here would leak the same way the reported one did, in a view where nothing else is - * watching. `Ctrl+C` is pressed on the editor's root element, where that plugin listens, rather - * than through a command dispatch, so what runs is the whole real path from key to clipboard. + * watching. These exercise the whole real path from the public surface to the clipboard: a keydown + * on the root element where the plugin listens, and `EditorRef.copy()`/`.cut()`, the API a host + * app drives the editor through. + * + * `execCommandSpy` supplies the `document.execCommand` jsdom lacks and reports whether anything was + * written. `ClipboardEvent` is stubbed for the same reason `formattedViewCopyEvent` stubs it: + * without it a regression would die on Lexical's own bare reference to the missing global before + * reaching the clipboard, hiding what actually got written. */ describe("copying an empty selection leaves the clipboard alone in a hidden-marker view", () => { - // `execCommandSpy` supplies the `document.execCommand` jsdom lacks and reports whether anything - // was written. `ClipboardEvent` is stubbed for the same reason `formattedViewCopyEvent` stubs - // it: without it a regression would die on Lexical's own bare reference to the missing global - // before reaching the clipboard, hiding what actually got written. const execCommand = execCommandSpy(); beforeEach(() => { vi.stubGlobal("ClipboardEvent", StubClipboardEvent); }); - afterEach(() => vi.unstubAllGlobals()); + afterEach(async () => { + // Drain `@lexical/clipboard`'s module-level `EVENT_LATENCY` (50ms) handle so a test that + // reached the real copy path cannot silence the next test's assertion. + await new Promise((resolve) => setTimeout(resolve, 60)); + vi.unstubAllGlobals(); + }); - it("Ctrl+C at a collapsed caret in Formatted view writes nothing and leaves the key unclaimed", async () => { - const editor = await renderEditor(FORMATTED_VIEW_MODE); + /** Collapses the selection at the end of the loaded document. */ + async function collapseCaretAtEnd(editor: LexicalEditor): Promise { await act(async () => editor.update(() => { const last = $getRoot().getLastDescendant(); @@ -313,21 +326,56 @@ describe("copying an empty selection leaves the clipboard alone in a hidden-mark $setSelection(selection); }), ); + } + + it("Ctrl+C at a collapsed caret in Formatted view writes nothing", async () => { + const { editor } = await renderEditorWithRef(FORMATTED_VIEW_MODE); + await collapseCaretAtEnd(editor); const rootElement = editor.getRootElement(); if (!rootElement) throw new Error("editor has no root element to press a key on"); - const event = new KeyboardEvent("keydown", { - key: "c", - ctrlKey: true, - bubbles: true, - cancelable: true, - }); await act(async () => { - rootElement.dispatchEvent(event); + rootElement.dispatchEvent( + new KeyboardEvent("keydown", { key: "c", ctrlKey: true, bubbles: true, cancelable: true }), + ); }); expect(execCommand()).not.toHaveBeenCalled(); - // Unclaimed, so the browser's own copy of an empty selection runs — and writes nothing. - expect(event.defaultPrevented).toBe(false); + }); + + it("EditorRef.copy() at a collapsed caret writes nothing", async () => { + const { editor, ref } = await renderEditorWithRef(FORMATTED_VIEW_MODE); + await collapseCaretAtEnd(editor); + + await act(async () => ref.current?.copy()); + + expect(execCommand()).not.toHaveBeenCalled(); + }); + + it("EditorRef.cut() at a collapsed caret writes nothing and removes nothing", async () => { + const { editor, ref } = await renderEditorWithRef(FORMATTED_VIEW_MODE); + await collapseCaretAtEnd(editor); + let before = ""; + editor.getEditorState().read(() => { + before = $getRoot().getTextContent(); + }); + + await act(async () => ref.current?.cut()); + + expect(execCommand()).not.toHaveBeenCalled(); + editor.getEditorState().read(() => expect($getRoot().getTextContent()).toBe(before)); + }); + + it("EditorRef.copy() right after a programmatic selection still copies", async () => { + // The public API's ordinary shape — select, then copy — and the case a guard reading the last + // COMMITTED selection would silently turn into a no-op, since Lexical commits on a microtask. + const { editor, ref } = await renderEditorWithRef(FORMATTED_VIEW_MODE); + + await act(async () => { + selectWholeDocument(editor); + ref.current?.copy(); + }); + + expect(execCommand()).toHaveBeenCalledWith("copy"); }); }); diff --git a/packages/platform/src/editor/Editor.tsx b/packages/platform/src/editor/Editor.tsx index 74cfd0415..ad301c336 100644 --- a/packages/platform/src/editor/Editor.tsx +++ b/packages/platform/src/editor/Editor.tsx @@ -491,8 +491,10 @@ const Editor = forwardRef(function Editor( redo() { editorRef.current?.dispatchCommand(REDO_COMMAND, undefined); }, - // Both leave the clipboard untouched when nothing is selected, rather than asking the browser - // to synthesize a clipboard event for an empty copy — see `copySelection` (shared-react). + // Both leave the clipboard untouched when nothing is selected, rather than writing a + // placeholder over it — `ClipboardPlugin`'s guard claims the command (shared-react's + // `registerEmptyCopyGuard`). Going through `copySelection`/`cutSelection` rather than + // dispatching here keeps that one seam named. cut() { assertEditable("cut"); if (editorRef.current) cutSelection(editorRef.current); diff --git a/packages/platform/src/editor/editor.model.ts b/packages/platform/src/editor/editor.model.ts index a5dd6b41d..ca075e710 100644 --- a/packages/platform/src/editor/editor.model.ts +++ b/packages/platform/src/editor/editor.model.ts @@ -77,12 +77,18 @@ export interface EditorRef { /** Redo the last undone action. */ redo(): void; /** - * Cut the selected text. + * Cut the selected text. With nothing selected it does nothing and the clipboard keeps whatever + * it already held — see {@link EditorRef.copy}. * @throws Will throw an error if the editor is in readonly mode or uses the block verse layout * (`ViewOptions.verseLayout: "block"`), which is read-only by construction. */ cut(): void; - /** Copy the selected text. */ + /** + * Copy the selected text. With nothing selected — no selection, or a collapsed caret — it does + * nothing and the clipboard keeps whatever it already held, rather than receiving a placeholder + * character the editor never contained. A selection made programmatically immediately before this + * call still copies: the guard reads the live selection, not the last committed one. + */ copy(): void; /** * Paste text at the current cursor position. diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index a0654c1c6..56d55b811 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -22,8 +22,9 @@ import { LexicalClipboardData } from "@lexical/clipboard"; // eslint-disable-next-line @nx/enforce-module-boundaries import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; import { Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; -import { $dfs } from "@lexical/utils"; +import { $dfs, mergeRegister } from "@lexical/utils"; import { + $createNodeSelection, $createPoint, $createRangeSelection, $createTextNode, @@ -31,6 +32,7 @@ import { $isTextNode, $setSelection, $setState, + COMMAND_PRIORITY_NORMAL, COPY_COMMAND, CUT_COMMAND, LexicalEditor, @@ -39,6 +41,7 @@ import { } from "lexical"; import { $createCharNode, + $createImmutableTypedTextNode, $createMarkerNode, $createParaNode, $isCharNode, @@ -244,8 +247,13 @@ function $selectAcrossFigure(): void { /** * Selects the figure and nothing else, as the two element points on its parent paragraph that sit - * either side of it — the shape a browser resolves a drag or click over a `contentEditable=false` - * block into, since it will not put a selection endpoint inside one. + * either side of it. + * + * That is ASSUMED to be what a browser resolves a real drag or click over a + * `contentEditable=false` block into, on the reasoning that it will not put a selection + * endpoint inside one — but jsdom cannot observe DOM-range → Lexical selection resolution, so + * the assumption is untested here and the pin below states only what the copy walker does with + * such a selection once it exists. * * Mutating: call inside `editor.update()`. */ @@ -662,13 +670,43 @@ describe("clipboard normalization — null-event leg (ClipboardPlugin/ContextMen * character on the clipboard; a plain collapsed caret anywhere in the view does the same, and so * does cut. * - * `document.execCommand` — which jsdom does not implement, so these tests supply it — is the - * observable: called means a write reached the browser, not called means the clipboard is - * untouched. The placeholder's own content belongs to `@lexical/clipboard` and is not pinned here. + * Two observables, because either alone can lie. `belowTheClaim` watches at a priority between this + * handler's HIGH and rich-text's EDITOR: reached means the command was NOT claimed here, which is + * the leak's first step and is immune to any module state. `document.execCommand` — which jsdom + * does not implement, so `execCommandSpy` supplies it — is the clipboard outcome itself: called + * means a write reached the browser. Note what is NOT asserted: the dispatch's own return value + * says nothing (rich-text's handlers return `true` unconditionally, so a leaked copy still reports + * "handled"), and `copyToClipboardSpy` says nothing either (`@lexical/rich-text` is externalized + * and imports the REAL `@lexical/clipboard`, which this file's `vi.mock` cannot reach). The + * placeholder's own content belongs to `@lexical/clipboard` and is not pinned here. */ describe("copying an empty selection leaves the clipboard alone", () => { const execCommand = execCommandSpy(); + /** + * Watches for the dispatch reaching PAST the Standard-view claim, without consuming it — so a + * regression runs the whole real chain and trips this AND the clipboard assertion, rather than + * being masked by the watcher itself. NORMAL sits below the claim's HIGH and above rich-text's + * EDITOR. + */ + function watchBelowTheClaim(editor: LexicalEditor) { + const reached = vi.fn(() => false); + const unregister = mergeRegister( + editor.registerCommand(COPY_COMMAND, reached, COMMAND_PRIORITY_NORMAL), + editor.registerCommand(CUT_COMMAND, reached, COMMAND_PRIORITY_NORMAL), + ); + return { reached, unregister }; + } + + afterEach(async () => { + // `@lexical/clipboard` keeps a MODULE-level timer handle while it waits for the clipboard event + // its `execCommand` call should provoke, and refuses to start another copy until that handle + // clears. A test that reaches the real copy path therefore silences the `execCommand` assertion + // in the NEXT test unless the window is drained here — which would make a regression fail only + // the first of these pins. The window is `EVENT_LATENCY`, 50ms. + await new Promise((resolve) => setTimeout(resolve, 60)); + }); + it("COPY_COMMAND(null) at a collapsed caret is claimed and writes nothing", async () => { let text: TextNode; const { editor } = await testEnvironment(() => { @@ -676,15 +714,14 @@ describe("copying an empty selection leaves the clipboard alone", () => { $appendMarkerAndText(text); }); await act(async () => editor.update(() => text.select(1, 1))); - copyToClipboardSpy.mockClear(); - let handled: boolean | undefined; + const { reached, unregister } = watchBelowTheClaim(editor); await act(async () => { - handled = editor.dispatchCommand(COPY_COMMAND, null); + editor.dispatchCommand(COPY_COMMAND, null); }); + unregister(); + // Claimed here, so nothing downstream gets the chance to synthesize the copy this declined. + expect(reached).not.toHaveBeenCalled(); expect(execCommand()).not.toHaveBeenCalled(); - expect(copyToClipboardSpy).not.toHaveBeenCalled(); - // Claimed, so nothing downstream gets the chance to synthesize the copy this declined to make. - expect(handled).toBe(true); editor.getEditorState().read(() => expect(text.getTextContent()).toBe(`a${NBSP}b`)); }); @@ -695,14 +732,13 @@ describe("copying an empty selection leaves the clipboard alone", () => { $appendMarkerAndText(text); }); await act(async () => editor.update(() => text.select(1, 1))); - copyToClipboardSpy.mockClear(); - let handled: boolean | undefined; + const { reached, unregister } = watchBelowTheClaim(editor); await act(async () => { - handled = editor.dispatchCommand(CUT_COMMAND, null); + editor.dispatchCommand(CUT_COMMAND, null); }); + unregister(); + expect(reached).not.toHaveBeenCalled(); expect(execCommand()).not.toHaveBeenCalled(); - expect(copyToClipboardSpy).not.toHaveBeenCalled(); - expect(handled).toBe(true); editor.getEditorState().read(() => expect(text.getTextContent()).toBe(`a${NBSP}b`)); }); @@ -721,14 +757,41 @@ describe("copying an empty selection leaves the clipboard alone", () => { before.select(before.getTextContentSize(), before.getTextContentSize()); }), ); - copyToClipboardSpy.mockClear(); - let handled: boolean | undefined; + const { reached, unregister } = watchBelowTheClaim(editor); await act(async () => { - handled = editor.dispatchCommand(COPY_COMMAND, null); + editor.dispatchCommand(COPY_COMMAND, null); }); + unregister(); + expect(reached).not.toHaveBeenCalled(); expect(execCommand()).not.toHaveBeenCalled(); - expect(copyToClipboardSpy).not.toHaveBeenCalled(); - expect(handled).toBe(true); + }); + + it("does NOT claim a node selection — it has content, and Lexical's own copy handles it", async () => { + // The deliberate carve-out in the claim's condition, pinned because it is the one way this + // guard could go wrong in the other direction: narrowing the test to "is this a RANGE + // selection" would swallow a copy that has real content behind it. The opposite outcome to + // every pin above — the dispatch passes through and the copy is made. + let decorator: ReturnType; + const { editor } = await testEnvironment(() => { + const text = $createTextNode(`a${NBSP}b`); + $appendMarkerAndText(text); + decorator = $createImmutableTypedTextNode("marker", "\\p"); + $getRoot().append($createParaNode("p").append(decorator)); + }); + await act(async () => + editor.update(() => { + const nodeSelection = $createNodeSelection(); + nodeSelection.add(decorator.getKey()); + $setSelection(nodeSelection); + }), + ); + const { reached, unregister } = watchBelowTheClaim(editor); + await act(async () => { + editor.dispatchCommand(COPY_COMMAND, null); + }); + unregister(); + expect(reached).toHaveBeenCalled(); + expect(execCommand()).toHaveBeenCalledWith("copy"); }); }); diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 14e88b0e0..25820a63b 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -133,14 +133,20 @@ export interface PastePayload { * clipboard at all (a KeyboardEvent-shaped dispatch, or an event whose data store is * inaccessible), which every claim reads as "not mine". * - * Four handlers race on `PASTE_COMMAND` — the in-note `\fp` claim at CRITICAL, the Standard-view - * external-paste claim and the char-stack line replay at HIGH, the paragraph-split arm at LOW — - * and they must agree byte-for-byte on what was pasted, or the same clipboard is one thing to one - * of them and another to the next. So the extraction lives here once: the jsdom-safe duck-check - * for the clipboard (jsdom implements no `ClipboardEvent`, so `instanceof` against the undefined - * global throws), the carrier choice below, and line-ending normalization BEFORE any caller tests - * for a line break — so `\r\n` and bare-`\r` clipboards break correctly and no `\r` ever reaches - * content on any path. + * THREE handlers replay the pasted bytes and so must agree on them byte-for-byte, or the same + * clipboard is one thing to one of them and another to the next: the in-note `\fp` claim at + * CRITICAL, and the Standard-view external-paste claim and the char-stack line replay at HIGH + * (`MarkerEditPlugin.tsx`). All three read the clipboard only through here: the jsdom-safe + * duck-check (jsdom implements no `ClipboardEvent`, so `instanceof` against the undefined global + * throws), the carrier choice below, and line-ending normalization BEFORE any caller tests for a + * line break — so `\r\n` and bare-`\r` clipboards break correctly and no `\r` ever reaches content + * on any path. + * + * Other handlers race on the same command without going through this, correctly: the LOW + * paragraph-split arm (`MarkerEditPlugin.tsx`) only sets a flag and never looks at the payload, + * and shared-react's `StructureKeyboardPlugin` (HIGH) and `CommandMenuPlugin` (NORMAL) read the + * clipboard directly for their own decisions — sanitizing and gating, not replaying — so they have + * nothing to agree with the three about. */ export function getPastePayload( event: PasteCommandType | null | undefined, @@ -162,11 +168,10 @@ export function getPastePayload( // deliberately carries no NBSP at all (display ones invert to spaces, a data NBSP displays and // copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not // survive a rich consumer ({@link invertDisplayNbspInHtml}) — so "the plain text has no NBSP" is - // TRUE of every P10 copy, and an - // NBSP-presence test would route P10's own round trip through html, whose decoded text drops a - // collapsed note's caller entirely (it rides as a `data-caller` attribute, never as text). A - // lost note caller on the editor's own copy is a worse, far likelier loss than a foreign - // clipboard's data-NBSP. Recorded in the semantics doc's accepted asymmetries. + // TRUE of every P10 copy, and an NBSP-presence test would route P10's own round trip through + // html, whose decoded text drops a collapsed note's caller entirely (it rides as a `data-caller` + // attribute, never as text). A lost note caller on the editor's own copy is a worse, far likelier + // loss than a foreign clipboard's data-NBSP. Recorded in the semantics doc's deferred list. return { text: plainText || htmlText, isInternal: !!clipboardData.getData("application/x-lexical-editor"), @@ -322,9 +327,9 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { * collapse INTO). `text` arrives with bare `\n` line endings ({@link getPastePayload} normalizes * them for every paste claim) — no `\r` reaches this function. The result is inserted via the exact * `selection.insertText` a keystroke uses — no chapter/book-id strip, no positional NBSP mapping, - * no marker tokenization, all of which - * exist for BODY content's marker-recognition and whitespace-display invariants and apply to none - * of it: a pasted `\c 5` or `\p` here is literal value text, not a structural marker, and + * no marker tokenization, all of which exist for BODY content's marker-recognition and + * whitespace-display invariants and apply to none of it: a pasted `\c 5` or `\p` here is literal + * value text, not a structural marker, and * `$displayWhitespaceTransform` (this file) already skips textType "attribute" nodes outright, so a * literal NBSP a user TYPES into an attribute run is never touched — a pasted one must not be * treated any differently. For a non-collapsed (mixed or fully-inside) selection, @@ -384,9 +389,9 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * unconditionally, per the doc comment above, and the "private Lexical flavor is dead on Ctrl+V" * fact recorded in the semantics doc's S3 still holds for the reconstructed-`DataTransfer` paste * path this repro most likely used). What both mechanisms — the old NBSP gate and this file's own - * same-namespace-flavor decline — share is the SAME failure shape once either one declines: Lexical's - * default rich-paste node insertion has no notion that an attribute run's text must stay inside its - * one tagged TextNode, and (reproduced directly on THIS branch, see + * same-namespace-flavor decline — share is the SAME failure shape once either one declines: + * Lexical's default rich-paste node insertion has no notion that an attribute run's text must stay + * inside its one tagged TextNode, and (reproduced directly on THIS branch, see * `attributeContextPasteFidelity.test.tsx`'s "root cause" describe) merges the run, the closing * glyph, and even the FOLLOWING paragraph's own sibling text into one plain node: the attribute * display and the closing marker both vanish, and the pasted bytes end up loose in body content. @@ -648,8 +653,9 @@ export function $getStandardViewClipboardData( * that was never in the document. Standard view is where a caret most often has nothing copyable * around it — a click on a read-only construct (a figure, a table) leaves the caret beside it * rather than inside — so a null-payload dispatch with no content behind it is CLAIMED here and - * does nothing, which is what copying an empty selection means. Callers that can decline earlier - * do (`copySelection`, shared-react); this covers the ones that reach the command anyway. + * does nothing, which is what copying an empty selection means. shared-react's + * `registerEmptyCopyGuard` states the same rule for every view, one priority lower; claiming here + * too keeps the rule from depending on which plugins a host happens to mount. * * A selection that is merely not a RANGE (a node selection) is still declined: it has real content, * and the synthesized-event path copies it correctly. From f60ac5804bd05663a4d818e5892596698e327fb8 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 12:25:18 -0500 Subject: [PATCH 06/44] test(platform): complete the table-optbreak context literal for the widened MarkerEditContext Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx index 0f3bf80aa..7822ccf46 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx @@ -1075,7 +1075,9 @@ describe("$textNodeTier2Transform on `//` inside a table", () => { getMarker: bundledGetMarker, pendingKeys: new Set(), splitExpected: { current: false }, + pasteRebuildArmed: { current: false }, rebuildAttempted: new Set(), + structureProtectionMode: "off", }; } From 1056f541b6ec75c8fc7303403ca1fd7f18743226 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 12:48:18 -0500 Subject: [PATCH 07/44] fix: keep a pasted figure's caption inside the figure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pasting `\fig At once they left their nets.|src="avnt016.jpg" size="span" ref="1.18"\fig*` into Standard view left a figure box holding only `\fig |src="…"\fig*`, the caption stranded after the closing glyph as paragraph prose, and an exported USJ whose `figure` object had no content at all — the caption merged into the neighbouring text run. `$textNodeInUnknownTransform` (TextSpacingPlugin) ejects any TextNode that is new in the current update out of an UnknownNode parent, so a user typing into a read-only opaque block never plants prose inside it. A Tier-2 rebuild materializes the whole figure — wrapper and caption together — in one update, so the caption looked exactly like typed text to that rule. Eject only when the wrapper predates the update, which is the shape a typed intrusion actually has. The class is invisible to a MarkerEditPlugin-only harness (the ejection lives in a sibling plugin), so the new pins mount the full Editor. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 1 + .../plugins/usj/TextSpacingPlugin.test.tsx | 28 ++++ .../src/plugins/usj/TextSpacingPlugin.tsx | 15 +- .../markerEdit/figurePasteFidelity.test.tsx | 148 ++++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index a4bb5119f..0ef5524d9 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -289,6 +289,7 @@ The following items are recorded as deferred and not addressed in this plan: | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | +| **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | | **E2E** Real-browser clipboard round trip (Standard view, written, not yet run in this task) | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | --- diff --git a/libs/shared-react/src/plugins/usj/TextSpacingPlugin.test.tsx b/libs/shared-react/src/plugins/usj/TextSpacingPlugin.test.tsx index c3dbebdcd..cd1f36c5d 100644 --- a/libs/shared-react/src/plugins/usj/TextSpacingPlugin.test.tsx +++ b/libs/shared-react/src/plugins/usj/TextSpacingPlugin.test.tsx @@ -461,6 +461,34 @@ describe("TextSpacingPlugin", () => { }); }); + it("should keep the content of an UnknownNode created WITH its content in one update — only a later intrusion is ejected", async () => { + // A construct that arrives whole (a Tier-2 rebuild materializing a pasted `\fig …\fig*`, a + // document load, a collab insert, an undo restore) creates the wrapper and its content + // children in the SAME update, so that content is indistinguishable from typed text by + // newness alone. Ejecting it strands a figure's caption outside its own box, both on screen + // and in the exported USJ. + const { editor } = await testEnvironment(() => { + $getRoot().append($createParaNode()); + }); + + await act(async () => { + editor.update(() => { + const para = $getRoot().getFirstChild(); + if (!$isParaNode(para)) throw new Error("Expected a ParaNode"); + para.append($createUnknownNode("figure", "fig").append($createTextNode("a caption"))); + }); + }); + + editor.getEditorState().read(() => { + const para = $getRoot().getFirstChild(); + if (!$isParaNode(para)) throw new Error("Expected a ParaNode"); + expect(para.getChildren()).toHaveLength(1); + const unknown = para.getFirstChild(); + if (!$isUnknownNode(unknown)) throw new Error("Expected an UnknownNode"); + expect(unknown.getTextContent()).toBe("a caption"); + }); + }); + it("should insert a space before a verse if preceded by a CharNode", async () => { const { editor } = await testEnvironment(() => { $getRoot().append( diff --git a/libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx b/libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx index 9a49f8df6..1a22d5e54 100644 --- a/libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx @@ -176,7 +176,8 @@ function $textNodeTrailingSpaceTransform(node: TextNode): void { } /** - * Moves a TextNode after its parent if the parent is an UnknownNode. + * Moves a TextNode out of an UnknownNode when it was planted there by an edit, so a read-only + * opaque block never gains prose of its own. * @param node - The TextNode to check. * @param editor - The LexicalEditor instance. */ @@ -184,9 +185,15 @@ function $textNodeInUnknownTransform(node: TextNode, editor: LexicalEditor): voi const unknownNode = node.getParent(); if (!$isUnknownNode(unknownNode) || !node.isAttached()) return; - // If a text node is created inside an UnknownNode (e.g., by typing), move it after the - // UnknownNode. - if (wasNodeCreated(editor, node.getKey())) unknownNode.insertAfter(node); + // Only text planted inside a PRE-EXISTING opaque block is an intrusion (e.g. typing into a + // figure). A wrapper that appeared in this SAME update brought its own content with it — a + // Tier-2 rebuild materializing a pasted `\fig caption|src="…"\fig*`, a document load, a collab + // insert, an undo restore — and every one of those creates the wrapper and its content children + // together. Ejecting there strands the construct's own text outside it: a figure's caption + // lands after the `\fig*` glyph as ordinary paragraph prose, and the exported USJ keeps a + // `figure` object with no content at all. + if (wasNodeCreated(editor, node.getKey()) && !wasNodeCreated(editor, unknownNode.getKey())) + unknownNode.insertAfter(node); } /** Transform for a verse node (handles non-TextNode predecessors) */ diff --git a/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx new file mode 100644 index 000000000..0eedfb691 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx @@ -0,0 +1,148 @@ +/** + * A pasted `\fig …\fig*` must land as ONE figure: its caption inside the construct, not stranded + * as paragraph prose after the closing glyph. + * + * Live symptom, with the full editor plugin stack mounted: pasting + * `\fig At once they left their nets.|src="avnt016.jpg" size="span" ref="1.18"\fig*` produced a + * figure box holding only `\fig |src="…"\fig*` — the caption ejected past the closer into the + * surrounding prose — and an exported USJ whose `figure` object had no `content` at all, the + * caption merged into the neighbouring text run instead. + * + * Mechanism: `$textNodeInUnknownTransform` (`TextSpacingPlugin`, shared-react) ejects any TextNode + * that is NEW in the current update out of an `UnknownNode` parent, so a user typing into a + * read-only opaque block never plants prose inside it. A Tier-2 rebuild materializes the WHOLE + * figure — wrapper and caption together — in one update, so the caption looked exactly like typed + * text to that rule. The transform now ejects only when the wrapper predates the update, which is + * the shape a typed intrusion actually has. + * + * These pins mount the real `Editor` (via `mountStandardViewEditor`) rather than the marker-edit + * plugin alone: the ejection lives in a sibling plugin, so a harness carrying only + * `MarkerEditPlugin` reports a clean round trip for the same paste and cannot see this class at + * all. + */ + +import { pasteEvent } from "./markerEdit.test-helpers"; +import { mountStandardViewEditor } from "../settledGetUsj.test-helpers"; +import { MarkerObject, Usj } from "@eten-tech-foundation/scripture-utilities"; +import { act } from "@testing-library/react"; +import { $getRoot, $isTextNode, LexicalEditor, PASTE_COMMAND, TextNode } from "lexical"; +import { $isParaNode, NBSP } from "shared"; + +// jsdom implements neither `ClipboardEvent` nor `DragEvent`; Lexical's own paste fallback +// duck-types against both (`objectKlassEquals`). Same stub as the sibling clipboard suites. +const globalStubs: { DragEvent?: unknown; ClipboardEvent?: unknown } = globalThis; +if (typeof globalStubs.DragEvent === "undefined") + globalStubs.DragEvent = class DragEvent extends Event {}; +if (typeof globalStubs.ClipboardEvent === "undefined") + globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; + +/** The exact bytes from the live report. */ +const FIGURE_USFM = `\\fig At once they left their nets.|src="avnt016.jpg" size="span" ref="1.18"\\fig*`; + +const CAPTION = "At once they left their nets."; + +/** The figure object those bytes mean, straight from the USFM: caption as the figure's own + * content, `src` stored under USX/USJ's `file`. */ +const figureObject: MarkerObject = { + type: "figure", + marker: "fig", + file: "avnt016.jpg", + size: "span", + ref: "1.18", + content: [CAPTION], +} as unknown as MarkerObject; + +/** A two-paragraph document whose first paragraph holds `content`; the second is somewhere for the + * caret to depart to. */ +function figureUsj(content: MarkerObject["content"]): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { type: "book", marker: "id", code: "GEN", content: ["GEN"] }, + { type: "chapter", marker: "c", number: "1" }, + { type: "para", marker: "p", content }, + { type: "para", marker: "p", content: ["depart here"] }, + ], + } as unknown as Usj; +} + +async function settle(): Promise { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** The paragraph text the editor actually displays, with display NBSPs read back as the plain + * spaces they stand in for — the byte ORDER is what these pins are about, not which whitespace + * codepoint carries a separator. */ +function displayText(editor: LexicalEditor): string { + return editor + .getEditorState() + .read(() => $getRoot().getTextContent()) + .replaceAll(NBSP, " "); +} + +/** Pastes `FIGURE_USFM` as a plain-text clipboard payload at `offset` within the first + * paragraph's `"Before after"` run, then departs the caret and settles. */ +async function pasteFigureAt(editor: LexicalEditor, offset: number): Promise { + await act(async () => + editor.update(() => { + const para = $getRoot().getChildren().filter($isParaNode)[0]; + const run = para + ?.getChildren() + .find( + (node): node is TextNode => $isTextNode(node) && node.getTextContent().includes("Before"), + ); + if (!run) throw new Error("expected the paragraph's `Before after` text run"); + run.select(offset, offset); + editor.dispatchCommand(PASTE_COMMAND, pasteEvent({ "text/plain": FIGURE_USFM }).event); + }), + ); + await settle(); + await act(async () => + editor.update(() => { + const departure = $getRoot().getChildren().filter($isParaNode)[1]?.getLastChild(); + if (!departure || !$isTextNode(departure)) + throw new Error("expected the `depart here` paragraph's text node"); + departure.select(0, 0); + }), + ); + await settle(); +} + +const hostContent: MarkerObject["content"] = [ + { type: "verse", marker: "v", number: "18" } as unknown as MarkerObject, + "Before after", +]; + +describe("pasting a figure into Standard view", () => { + it("keeps the caption inside the figure — the exported USJ carries it as the figure's own content, mid-paragraph", async () => { + const { ref, lexical } = await mountStandardViewEditor(figureUsj(hostContent)); + await pasteFigureAt(lexical, 7); // between "Before " and "after" + expect(ref.current?.getUsj()).toEqual( + figureUsj([hostContent[0], "Before ", figureObject, "after"]), + ); + }); + + it("keeps the caption inside the figure when the paste lands at the paragraph's end", async () => { + const { ref, lexical } = await mountStandardViewEditor(figureUsj(hostContent)); + await pasteFigureAt(lexical, "Before after".length); + expect(ref.current?.getUsj()).toEqual( + figureUsj([hostContent[0], "Before after", figureObject]), + ); + }); + + it("displays the pasted figure's bytes in the same order a LOAD of the identical document does — caption between the opening glyph and the attribute run, not after the closer", async () => { + // The oracle is a real load of the document the paste is supposed to produce, so this pin + // states "paste looks like load" rather than re-encoding the expected byte string by hand. + const { lexical: loaded } = await mountStandardViewEditor( + figureUsj([hostContent[0], "Before ", figureObject, "after"]), + ); + const { lexical: pasted } = await mountStandardViewEditor(figureUsj(hostContent)); + await pasteFigureAt(pasted, 7); + expect(displayText(pasted)).toContain(`\\fig ${CAPTION}|src="avnt016.jpg"`); + expect(displayText(pasted)).toBe(displayText(loaded)); + }); +}); From abdb640f60dffa48bb150f65f1fd763e6a630fd4 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 12:59:05 -0500 Subject: [PATCH 08/44] fix: keep every child-bearing opaque construct in the lexical-flavor copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An internal (`application/x-lexical-editor`) copy→paste of a figure, sidebar, periph or ref pasted the construct's full literal USFM bytes onto the screen while the exported USJ silently dropped the node and every attribute on it — a convincing display over lost data that a save then persisted with no error. `UnknownNode.excludeFromCopy` kept only a child-bearing optbreak out of the exclusion; every other kind stayed excluded, and `$appendNodesToJSON` does not drop an excluded node — it hoists its children into the parent, stranding the content-free display decorators that render each kind's marker and attribute bytes. Key both `excludeFromCopy` and the `isSelected` boundary override on the node's own child count rather than on its kind: the kinds share one display-decorator shape, so they shared one failure. Verified per kind across all three real clipboard payload shapes (plain-only, plain+html, and the full payload carrying the lexical flavor). Tables are not part of this — `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now and never took the exclusion — and are pinned in the same sweep to say so. The plain-carrier losses each kind genuinely has (a block construct split across lines; a `ref` wrapper USFM has no bytes for) are asserted as losses rather than left unpinned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 5 +- .../src/nodes/features/UnknownNode.test.ts | 23 +- libs/shared/src/nodes/features/UnknownNode.ts | 84 +++--- .../optbreakClipboardFidelity.test.tsx | 14 +- .../unknownClipboardFidelity.test.tsx | 244 ++++++++++++++++++ 5 files changed, 301 insertions(+), 69 deletions(-) create mode 100644 packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 0ef5524d9..9816b70d2 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -28,11 +28,11 @@ Already ~true because markers are real text nodes; this plan closes the exceptio ### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) -**Fixed gap (2026-08-13), lexical-JSON flavor only:** the "exact node tree" promise did not hold for a selection containing an `UnknownNode` (figure, table, sidebar, periph, ref, optbreak) — fixed here for optbreak; the identical mechanism affects the other five kinds too, left as a measured, un-fixed residual (see "Deferred / Out of Scope" below). `@lexical/clipboard`'s JSON generator (`$appendNodesToJSON`, the function behind the `application/x-lexical-editor` flavor) computes node exclusion from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, hardcoded for EVERY copy-out destination `$appendNodesToJSON` itself handles, not only actual HTML generation (`'clone'` is never passed by any Lexical-shipped code path in the installed version). This change touches ONLY that flavor: `$appendNodesToHTML` (`@lexical/html`, the function behind `text/html`) computes the same `excludeFromCopy('html')` value, but `UnknownNode.exportDOM()` always returns `{element: null}` for every kind, and `$appendNodesToHTML` returns early on a null element BEFORE ever consulting the exclusion value it just computed — so `text/html` output is byte-identical before and after this fix, for every `UnknownNode` kind. +**Fixed gap (2026-08-13, extended to every kind 2026-09-10), lexical-JSON flavor only:** the "exact node tree" promise did not hold for a selection containing an `UnknownNode` (figure, sidebar, periph, ref, optbreak). `@lexical/clipboard`'s JSON generator (`$appendNodesToJSON`, the function behind the `application/x-lexical-editor` flavor) computes node exclusion from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, hardcoded for EVERY copy-out destination `$appendNodesToJSON` itself handles, not only actual HTML generation (`'clone'` is never passed by any Lexical-shipped code path in the installed version). This change touches ONLY that flavor: `$appendNodesToHTML` (`@lexical/html`, the function behind `text/html`) computes the same `excludeFromCopy('html')` value, but `UnknownNode.exportDOM()` always returns `{element: null}` for every kind, and `$appendNodesToHTML` returns early on a null element BEFORE ever consulting the exclusion value it just computed — so `text/html` output is byte-identical before and after this fix, for every `UnknownNode` kind. `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked a CHILD-BEARING optbreak "selected" under Lexical's default `isSelected`, producing a CHILDLESS `{type:"optbreak"}` placeholder in the copy even though the live node has a child — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. Every OTHER `UnknownNode` kind (figure, table, sidebar, periph, ref) keeps the OLD, still-excluding behavior — not because their content differs meaningfully from optbreak's (it doesn't: every kind's marker/attribute bytes are the same content-free `ImmutableTypedTextNode` display decorators, built by the identical `unknownDisplayParts` machinery), but pending per-kind paste verification this task did not do. Pinned end-to-end (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, plus a direct unit pin on the narrowed predicate in `UnknownNode.test.ts`. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked a CHILD-BEARING optbreak "selected" under Lexical's default `isSelected`, producing a CHILDLESS `{type:"optbreak"}` placeholder in the copy even though the live node has a child — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx`, plus direct unit pins on the predicate in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -287,6 +287,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; six known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"` fixture (tight spacing, no flanking spaces) is swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | +| **S2 (per-kind opaque constructs)** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind, each asserting the lexical-flavor round trip and, where a carrier genuinely cannot express the construct, asserting the LOSS by name (the plain carrier's block split for sidebar/periph/table; USFM having no bytes at all for a `ref` wrapper). Mounts the full `Editor`, not `MarkerEditPlugin` alone | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | | **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | diff --git a/libs/shared/src/nodes/features/UnknownNode.test.ts b/libs/shared/src/nodes/features/UnknownNode.test.ts index c732c5caf..25e5c93dd 100644 --- a/libs/shared/src/nodes/features/UnknownNode.test.ts +++ b/libs/shared/src/nodes/features/UnknownNode.test.ts @@ -343,15 +343,16 @@ describe("UnknownNode", () => { }); }); - // The narrowed predicate behind the same-namespace `application/x-lexical-editor` optbreak copy - // fix: @lexical/clipboard's `$appendNodesToJSON` computes node exclusion from + // The predicate behind the same-namespace `application/x-lexical-editor` copy path: + // @lexical/clipboard's `$appendNodesToJSON` computes node exclusion from // `excludeFromCopy('html')` for EVERY copy-out format it handles, not only // actual `text/html` generation (see `excludeFromCopy`'s own doc comment). A direct unit pin on // the predicate here — independent of the integration-level clipboard pins in - // `optbreakClipboardFidelity.test.tsx` — catches a regression to this method locally, without - // needing the full editor/selection/clipboard-event machinery those pins exercise. + // `optbreakClipboardFidelity.test.tsx` and `unknownClipboardFidelity.test.tsx` — catches a + // regression to this method locally, without needing the full editor/selection/clipboard-event + // machinery those pins exercise. describe("excludeFromCopy()", () => { - it("does NOT exclude a CHILD-BEARING optbreak — the fix that lets a copied optbreak survive the lexical-flavor paste path", () => { + it("does NOT exclude a CHILD-BEARING optbreak — what lets a copied optbreak survive the lexical-flavor paste path", () => { const { editor } = createBasicTestEnvironment([UnknownNode, ImmutableTypedTextNode]); editor.update(() => { const optbreak = $createUnknownNode("optbreak"); @@ -360,7 +361,7 @@ describe("UnknownNode", () => { }); }); - it('excludes a CHILDLESS optbreak — a genuinely empty live husk falls back to the old, excluding behavior, matching text/plain\'s own "nothing here" for that shape', () => { + it('excludes a CHILDLESS optbreak — a genuinely empty live husk stays excluded, matching text/plain\'s own "nothing here" for that shape', () => { const { editor } = createBasicTestEnvironment([UnknownNode]); editor.update(() => { const optbreak = $createUnknownNode("optbreak"); @@ -369,11 +370,19 @@ describe("UnknownNode", () => { }); }); - it('still excludes a figure, child-bearing or not — the fix is scoped to "optbreak" only; every other UnknownNode kind is an un-fixed, documented residual (clipboard semantics doc, "Deferred / Out of Scope")', () => { + it("does NOT exclude a child-bearing figure either — the rule is the node's own child count, not its kind, because every kind's display bytes are the same content-free decorators that get stranded by a hoist", () => { const { editor } = createBasicTestEnvironment([UnknownNode, ImmutableTypedTextNode]); editor.update(() => { const figure = $createUnknownNode("figure", "fig"); figure.append($createImmutableTypedTextNode("marker", "\\fig ")); + expect(figure.excludeFromCopy("html")).toBe(false); + }); + }); + + it("excludes a CHILDLESS figure — nothing of it would survive the copy anyway, and a childless placeholder in the lexical flavor has no counterpart in text/plain", () => { + const { editor } = createBasicTestEnvironment([UnknownNode]); + editor.update(() => { + const figure = $createUnknownNode("figure", "fig"); expect(figure.excludeFromCopy("html")).toBe(true); }); }); diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index 8c7117c9c..7aa8b320f 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -207,69 +207,51 @@ export class UnknownNode extends ElementNode { return false; } - // Narrowed from an unconditional `return destination !== "clone"` to leave a CHILD-BEARING - // "optbreak" out of the exclusion. This changes ONLY the `application/x-lexical-editor` (lexical- - // JSON) flavor — `text/html` is byte-identical before and after, for every `UnknownNode` kind, not - // only optbreak: `$appendNodesToHTML` (`@lexical/html`) computes this same `excludeFromCopy('html')` - // value too, but `UnknownNode.exportDOM()` always returns `{element: null}`, and - // `$appendNodesToHTML` returns early on a null `element` BEFORE ever consulting the exclusion - // value it just computed — so this predicate's return value has no observable effect on `text/html` - // output at all, confirmed by reading `LexicalHtml.dev.js`. `'clone'` is never passed by any - // Lexical-shipped code path in the installed version (confirmed by reading - // `LexicalClipboard.dev.js`), so the old, unconditional form excluded every `UnknownNode` from the - // lexical-JSON flavor unconditionally. Excluding a node does not drop it silently: - // `$appendNodesToJSON` hoists the excluded node's own children into its parent's list in its place. - // For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a - // content-free DecoratorNode with no meaning once separated from its owning `UnknownNode`), that - // stranded a loose decorator on the same-namespace `application/x-lexical-editor` paste fast - // path — `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, so the - // paste silently lost the discretionary line break (live report: copying an optbreak worked, - // pasting the same clipboard back did not restore it). + // A CHILD-BEARING `UnknownNode` of any kind stays IN the copy. This affects ONLY the + // `application/x-lexical-editor` (lexical-JSON) flavor: `$appendNodesToHTML` (`@lexical/html`) + // computes this same `excludeFromCopy('html')` value, but `exportDOM()` above always returns + // `{element: null}` and `$appendNodesToHTML` returns early on a null `element` BEFORE ever + // consulting the exclusion value it just computed, so `text/html` output is byte-identical for + // every kind whatever this predicate answers (confirmed by reading `LexicalHtml.dev.js`). + // `'clone'` is never passed by any Lexical-shipped code path in the installed version (confirmed + // by reading `LexicalClipboard.dev.js`), so an unconditional `destination !== "clone"` would + // exclude every `UnknownNode` from the lexical-JSON flavor outright. // - // Every OTHER `UnknownNode` kind (figure, table, sidebar, periph, ref) keeps the OLD, still- - // excluding behavior — NOT because their content differs meaningfully from optbreak's (it doesn't: - // every kind's marker/attribute bytes are the SAME content-free `ImmutableTypedTextNode` display - // decorators optbreak's `//` is, built by the identical `unknownDisplayParts` machinery), but - // PENDING per-kind paste verification this task did not do. The measured shape of the un-fixed - // residual for those kinds (a lexical-flavor copy of `\fig …\fig*` renders its full literal USFM - // bytes on screen after paste — the hoisted decorators still `decorate()` their own text as loose - // siblings — while the pasted document's USJ silently drops the figure node AND its attributes - // entirely, keeping only its real caption text merged into surrounding prose) is recorded in the - // clipboard semantics doc's "Deferred / Out of Scope" list rather than fixed here. + // Excluding a node does not drop it silently: `$appendNodesToJSON` HOISTS the excluded node's own + // children into its parent's list in its place. Every kind's marker and attribute bytes are + // content-free `ImmutableTypedTextNode` display decorators (`unknownDisplayParts` builds them + // identically for all of them), so hoisting strands decorators that still `decorate()` their own + // literal text as loose siblings with no owning wrapper: the pasted document RENDERS the + // construct's full USFM bytes while its USJ has lost the node and every attribute on it — a + // convincing display over missing data, which a save then persists with no error. // - // The `getChildrenSize() > 0` guard covers a genuinely childless live optbreak (an emptied husk - // mid-settle — `markerEditTier1.utils.ts`'s own husk-removal check uses the identical shape, - // "optbreak" plus zero children): such a node falls back to excluded, matching `text/plain`'s - // "nothing here" for the same case. It does NOT, on its own, cover a DIFFERENT carrier-agreement - // gap: a selection whose ending boundary resolves to an ELEMENT-type point ON this node at - // offset 0 (touching the wrapper without covering its own `//` child) still marks a CHILD-BEARING - // optbreak "selected" under Lexical's default `isSelected` (key-membership in - // `selection.getNodes()`, unaffected by this node's own child count), while the child itself is - // not — `$appendNodesToJSON` then serializes a CHILDLESS `{type:"optbreak"}` placeholder even - // though the live node has a child, disagreeing with `text/plain` (`$selectionToUsfmText`, which - // walks that same `getNodes()` list and correctly emits nothing for this boundary). The - // `isSelected` override below closes that second gap directly, at its actual source, since - // `excludeFromCopy` has no visibility into which of a node's children will end up selected. + // The `getChildrenSize() > 0` guard keeps a genuinely childless husk excluded, matching + // `text/plain`'s "nothing here" for the same node (`markerEditTier1.utils.ts`'s own + // husk-removal check recognizes an emptied optbreak by that same zero-child shape). It does NOT, + // on its own, cover a DIFFERENT carrier-agreement gap: a selection whose ending boundary resolves + // to an ELEMENT-type point ON this node at offset 0 (touching the wrapper without covering any of + // its content) still marks a CHILD-BEARING node "selected" under Lexical's default `isSelected` + // (key-membership in `selection.getNodes()`, unaffected by this node's own child count), so + // `$appendNodesToJSON` would serialize a CHILDLESS placeholder for a node that has children, + // disagreeing with `text/plain` (`$selectionToUsfmText`, which walks that same `getNodes()` list + // and correctly emits nothing for this boundary). The `isSelected` override below closes that + // second gap at its actual source, since `excludeFromCopy` has no visibility into which of a + // node's children a given selection will include. override excludeFromCopy(destination: "clone" | "html"): boolean { - return this.getTag() === "optbreak" && this.getChildrenSize() > 0 - ? false - : destination !== "clone"; + return this.getChildrenSize() > 0 ? false : destination !== "clone"; } - // An optbreak is only meaningfully "selected" (and so only copy-included, per the - // `excludeFromCopy` guard above) when at least one of its own display children is — mirrors + // An `UnknownNode` is only meaningfully "selected" (and so only copy-included, per the + // `excludeFromCopy` guard above) when at least one of its own children is — mirrors // `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at the same selection // boundary, the same way Lexical's own base `isSelected` (`LexicalNode.prototype.isSelected`) // already special-cases an inline DECORATOR node sitting as a parent's last child at an // exactly-there boundary point, for the identical reason. Without this, a selection ending // exactly at this node's own start (an ElementNode touch-boundary that covers none of its // content) still counts the WRAPPER as selected via the default `ElementNode.isSelected` - // (key-membership in `selection.getNodes()`), while the child is not — producing a childless - // `{type:"optbreak"}` entry in the `application/x-lexical-editor` copy with no corresponding - // `//` in `text/plain`. Scoped to "optbreak" only, matching `excludeFromCopy`'s own scoping; every - // other kind keeps the inherited default. + // (key-membership in `selection.getNodes()`), while no child is — producing a childless entry in + // the `application/x-lexical-editor` copy with nothing corresponding to it in `text/plain`. override isSelected(selection?: BaseSelection | null): boolean { - if (this.getTag() !== "optbreak") return super.isSelected(selection); const targetSelection = selection ?? $getSelection(); if (!targetSelection) return false; const selectedNodes = targetSelection.getNodes(); diff --git a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx index f99258b01..6595d185e 100644 --- a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx @@ -53,15 +53,11 @@ * `{type:"optbreak"}` placeholder in the lexical-JSON copy even though the live node has a * child — disagreeing with `text/plain`, which correctly emits nothing for that same boundary. * `excludeFromCopy` cannot see this (it has no visibility into which children a given selection - * will include), so a scoped `isSelected` override closes it directly, mirroring - * `$selectionToUsfmText`'s own `getNodes()`-based walk. Every OTHER `UnknownNode` kind (figure, - * table, sidebar, periph, ref) keeps the OLD, still-excluding behavior — not because their - * content differs meaningfully (it doesn't: every kind's marker/attribute bytes are the same - * content-free `ImmutableTypedTextNode` display decorators optbreak's `//` is), but pending - * per-kind paste verification this task did not do; the measured, more severe residual for those - * kinds (a lexical-flavor figure paste renders its full literal USFM bytes on screen while the - * USJ silently drops the figure node and its attributes) is recorded in the clipboard semantics - * doc's "Deferred / Out of Scope" list rather than fixed here. + * will include), so an `isSelected` override closes it directly, mirroring + * `$selectionToUsfmText`'s own `getNodes()`-based walk. Both rules key on the node's own child + * count rather than on its kind, so every other `UnknownNode` construct (figure, sidebar, periph, + * ref) is covered by the same predicate — their per-kind copy→paste round trips are pinned in + * `unknownClipboardFidelity.test.tsx`. */ import { MarkerEditPlugin } from "./MarkerEditPlugin"; diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx new file mode 100644 index 000000000..9c498e815 --- /dev/null +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -0,0 +1,244 @@ +/** + * Copy→paste fidelity for every opaque construct the editor carries as an `UnknownNode` — figure, + * sidebar, periph, ref — across the three clipboard payload shapes a real Ctrl+C/Ctrl+V produces, + * plus the table kinds that used to be `UnknownNode`s and are now their own `ImmutableTable*` + * nodes. + * + * The shape under test is the one `optbreakClipboardFidelity.test.tsx` established: a payload + * carrying `application/x-lexical-editor` takes Lexical's own same-namespace fast path, which + * rebuilds nodes from the JSON generator's output rather than re-tokenizing the plain text. That + * generator computes exclusion from `UnknownNode.excludeFromCopy`, and an excluded node is not + * dropped — its children are HOISTED into the parent in its place. For a construct whose children + * are the content-free display decorators `unknownDisplayParts` builds (`\fig `, `|src="…"`, + * `\fig*`), that hoisting produced a convincing lie: the pasted document RENDERED the construct's + * full literal USFM bytes as loose siblings while its exported USJ silently dropped the node and + * every one of its attributes. A save at that point persisted the loss with no error and a screen + * that still showed the bytes. + * + * These pins mount the full `Editor` rather than `MarkerEditPlugin` alone: the plugins around the + * engine take part in what a paste actually produces (see `figurePasteFidelity.test.tsx`), so a + * narrower harness can report a paste clean that a user's does not. + * + * Where a kind cannot round-trip, the LOSS is asserted rather than the fidelity, so the sweep never + * hides one — each such pin says in its own name what is lost and why. + */ + +import { copyEvent, pasteEvent } from "./markerEdit.test-helpers"; +import { mountStandardViewEditor } from "../settledGetUsj.test-helpers"; +import { corpusFixtures } from "../adaptors/corpus/corpus-data"; +import { MarkerObject, Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; +import { act } from "@testing-library/react"; +import { $getRoot, COPY_COMMAND, PASTE_COMMAND, RootNode } from "lexical"; +import { $isChapterNode, $isImmutableChapterNode } from "shared"; + +// jsdom implements neither `ClipboardEvent` nor `DragEvent`; Lexical's own rich-paste fallback — +// the path a lexical-flavor payload takes once the Standard-view handler declines — duck-types +// against both (`objectKlassEquals`). Same stub as the sibling clipboard suites. +const globalStubs: { DragEvent?: unknown; ClipboardEvent?: unknown } = globalThis; +if (typeof globalStubs.DragEvent === "undefined") + globalStubs.DragEvent = class DragEvent extends Event {}; +if (typeof globalStubs.ClipboardEvent === "undefined") + globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; + +const PLAIN = "text/plain"; +const HTML = "text/html"; +const LEXICAL = "application/x-lexical-editor"; + +interface Payload { + [mimeType: string]: string; +} + +function corpusUsj(name: string): Usj { + const fixture = corpusFixtures.find((entry) => entry.name === name); + if (!fixture) throw new Error(`no corpus fixture named ${name}`); + return usxStringToUsj(fixture.usx); +} + +/** The index of the first top-level node after the document's header — everything past the LAST + * chapter marker, or past the book node for header-only front matter (`periph`) that has no + * chapter at all. This is what Standard view actually copies out of: a user selects inside an open + * chapter, never the header itself. */ +function $contentStartIndex(root: RootNode): number { + const children = root.getChildren(); + let headerEnd = 0; + children.forEach((child, index) => { + if ($isChapterNode(child) || $isImmutableChapterNode(child)) headerEnd = index + 1; + }); + // No chapter: the book node alone is the header. + return headerEnd === 0 ? 1 : headerEnd; +} + +/** The fixture's header, byte-identical, plus one EMPTY `\p` paragraph as the paste host — the + * same shape `clipboardCorpusRoundTrip.test.tsx` seeds its target editor with, so the paste only + * has to reproduce the content (it can neither carry nor rebuild `\c`/`\id`; paste normalization + * strips them). */ +function headerSkeletonUsj(usj: Usj): Usj { + let headerEnd = 0; + usj.content.forEach((item, index) => { + if (typeof item !== "string" && (item.type === "chapter" || item.type === "book")) + headerEnd = index + 1; + }); + const emptyHost: MarkerObject = { type: "para", marker: "p", content: [] } as MarkerObject; + return { ...usj, content: [...usj.content.slice(0, headerEnd), emptyHost] }; +} + +async function settle(): Promise { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** Copies the fixture's content (header excluded) out of a real Standard-view editor and hands + * back all three carriers the copy wrote, so each paste shape below replays the editor's OWN bytes + * rather than a hand-written approximation of them. */ +async function copyContentPayload(usj: Usj): Promise { + const { lexical } = await mountStandardViewEditor(usj); + await act(async () => + lexical.update(() => { + const root = $getRoot(); + root.select($contentStartIndex(root), root.getChildrenSize()); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => lexical.dispatchCommand(COPY_COMMAND, event)); + return { [PLAIN]: getData(PLAIN), [HTML]: getData(HTML), [LEXICAL]: getData(LEXICAL) }; +} + +/** Pastes `payload` into a fresh header-only editor and returns the USJ a host would save. */ +async function pasteIntoFreshHost(usj: Usj, payload: Payload): Promise { + const { ref, lexical } = await mountStandardViewEditor(headerSkeletonUsj(usj)); + await act(async () => + lexical.update(() => { + $getRoot().getLastChild()?.selectEnd(); + lexical.dispatchCommand(PASTE_COMMAND, pasteEvent(payload).event); + }), + ); + await settle(); + // A pasted literal that pends (a marker completed under the caret) settles only once the caret + // DEPARTS the paragraph it landed in; the lexical-flavor path inserts an already-structural tree + // and has nothing pending, so this step is a harmless no-op there. + await act(async () => + lexical.update(() => { + $getRoot().getLastChild()?.selectEnd(); + }), + ); + await settle(); + return ref.current?.getUsj(); +} + +/** The three real-world clipboard shapes: a plain-text source, the async Ctrl+V read (which drops + * the private flavor), and the synchronous/native copy that keeps it. */ +async function pastedUsjFor(usj: Usj, shape: "plain" | "plain+html" | "full"): Promise { + const payload = await copyContentPayload(usj); + const shaped: Payload = + shape === "plain" + ? { [PLAIN]: payload[PLAIN] } + : shape === "plain+html" + ? { [PLAIN]: payload[PLAIN], [HTML]: payload[HTML] } + : payload; + if (shape === "full" && !payload[LEXICAL]) + throw new Error("copy wrote no application/x-lexical-editor payload"); + const pasted = await pasteIntoFreshHost(usj, shaped); + if (!pasted) throw new Error("editor produced no USJ"); + return pasted; +} + +/** Every object of `type` anywhere in `usj`, at any depth. */ +function objectsOfType(usj: Usj, type: string): MarkerObject[] { + const found: MarkerObject[] = []; + const walk = (items: MarkerObject["content"]) => + items?.forEach((item) => { + if (typeof item === "string") return; + if (item.type === type) found.push(item); + walk(item.content); + }); + walk(usj.content); + return found; +} + +const SHAPES = ["plain", "plain+html", "full"] as const; + +describe("figure (UnknownNode) copy→paste across all three payload shapes", () => { + const usj = corpusUsj("figure (USFM 3 attributes)"); + SHAPES.forEach((shape) => { + it(`${shape} payload round-trips the figure, its caption, and every attribute`, async () => { + expect(await pastedUsjFor(usj, shape)).toEqual(usj); + }); + }); +}); + +describe("sidebar (UnknownNode) copy→paste across all three payload shapes", () => { + const usj = corpusUsj("sidebar (esb)"); + it("full payload (lexical flavor) round-trips the sidebar whole — its category attribute and nested paragraph included", async () => { + expect(await pastedUsjFor(usj, "full")).toEqual(usj); + }); + + (["plain", "plain+html"] as const).forEach((shape) => { + // The plain carrier cannot express a sidebar in one line: the copy walker puts the nested `\p` + // on its own line, paste replays that newline as a paragraph split, and Tier 2 re-tokenizes + // one paragraph at a time, so the `\esb`/`\esbe` pairing the tokenizer already implements + // never sees both halves at once. The loss is asserted, not the fidelity. + it(`${shape} payload loses the sidebar's pairing — the copy-introduced paragraph split outlives the per-paragraph rebuild`, async () => { + const pasted = await pastedUsjFor(usj, shape); + expect(pasted).not.toEqual(usj); + expect(objectsOfType(pasted, "sidebar")[0]?.content ?? []).toEqual([]); + }); + }); +}); + +describe("periph (UnknownNode) copy→paste across all three payload shapes", () => { + const usj = corpusUsj("periph"); + it("full payload (lexical flavor) round-trips the periph construct whole — its id/alt attributes and nested paragraph included", async () => { + // Compared construct-to-construct rather than document-to-document: `periph` is book-level + // front matter with no chapter, so this sweep's "header plus one empty `\\p` host" target + // leaves the pasted block nested inside that host paragraph. Where the block LANDS is generic + // Lexical insertion, not construct fidelity; what this pin is about is that nothing inside the + // construct was lost on the way. + const pasted = await pastedUsjFor(usj, "full"); + expect(objectsOfType(pasted, "periph")).toEqual(objectsOfType(usj, "periph")); + }); + + (["plain", "plain+html"] as const).forEach((shape) => { + // Same block-split mechanism as the sidebar above: `\periph` has no closing bytes at all, so + // once the nested paragraph lands on its own line there is nothing to reunite the two. + it(`${shape} payload loses the periph's nested paragraph — the copy-introduced split outlives the per-paragraph rebuild`, async () => { + const pasted = await pastedUsjFor(usj, shape); + expect(objectsOfType(pasted, "periph")).not.toEqual(objectsOfType(usj, "periph")); + expect(objectsOfType(pasted, "periph")[0]?.content ?? []).toEqual([]); + }); + }); +}); + +describe("ref (UnknownNode) copy→paste across all three payload shapes", () => { + const usj = corpusUsj("cross-reference ref target"); + it("full payload (lexical flavor) round-trips the ref wrapper and its loc attribute", async () => { + expect(await pastedUsjFor(usj, "full")).toEqual(usj); + }); + + (["plain", "plain+html"] as const).forEach((shape) => { + // Inherent to the construct, not to the clipboard: USJ invented the `` container and USFM + // never carried it, so a plain-text carrier has no bytes anywhere that mark the wrapper's + // extent. A raw USFM export of the same document has the identical gap. + it(`${shape} payload loses the ref wrapper — USFM has no bytes for it, so only its child text survives`, async () => { + const pasted = await pastedUsjFor(usj, shape); + expect(objectsOfType(pasted, "ref")).toEqual([]); + expect(JSON.stringify(pasted)).toContain("Genesis 1:1"); + }); + }); +}); + +describe("table (ImmutableTable* nodes, not UnknownNode) copy→paste across all three payload shapes", () => { + const usj = corpusUsj("table with header and cells"); + it("full payload (lexical flavor) round-trips the table whole — rows, cells, and their markers", async () => { + expect(await pastedUsjFor(usj, "full")).toEqual(usj); + }); + + (["plain", "plain+html"] as const).forEach((shape) => { + // A table's rows and cells are real block-level nodes, so the plain carrier spreads one table + // over eight lines and the paste replays each as its own paragraph. + it(`${shape} payload loses the table's assembly — every row and cell copies onto its own line`, async () => { + expect(await pastedUsjFor(usj, shape)).not.toEqual(usj); + }); + }); +}); From 408d4ad2cc7c76166229550b2e9911f60dfe2e5b Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 13:13:03 -0500 Subject: [PATCH 09/44] fix: copy an opaque construct on one line so a paste can reassemble it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sidebar's nested `\p` and a table's rows and cells are real block-level nodes, so the copy walker put a `\n` at each of them and spread ONE construct over several lines. Paste replays every `\n` as a paragraph split and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct: a pasted sidebar came back unclosed with its paragraph hoisted out, a pasted table as empty row wrappers with the cells stranded as sibling paragraphs. Suppress the line break for a block boundary whose opaque ancestor is above it, so the construct stays byte-contiguous and a single-paragraph rebuild can reassemble it. The construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document — just not the line-per-marker layout a writer emits. Tables needed one more thing on the paste side: usfm.sty classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the tokenizer as a char token and the table-cell assembly, which only ran for paragraph tokens, never saw it. ParatextData derives a cell from the marker NAME, so recognize one on either token kind, guarded on an open row. The corpus sweep's sidebar and table fixtures are un-skipped and now sweep clean. `\periph` is not fixed: the tokenizer has no periph assembly and periph renders attributes as a pipe run on a marker with no closing bytes. That is recorded and pinned as a loss rather than left to look fixed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 92 +++++++------------ .../src/converters/usfm/usfmFragmentToUsj.ts | 52 ++++++++--- .../clipboardCorpusRoundTrip.test.tsx | 58 ++---------- .../unknownClipboardFidelity.test.tsx | 45 ++++----- .../whitespaceDisplay.plugin.utils.ts | 31 ++++++- 5 files changed, 123 insertions(+), 155 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 9816b70d2..109090e86 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -26,6 +26,8 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). +- **An opaque construct copies out on ONE line:** the walk's ordinary rule puts a `\n` at every non-inline element boundary it crosses, which would spread a single construct over several lines — a sidebar's nested `\p`, and every row and cell of a table, are real block-level nodes. Pasting that text replays each `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct in one pass. `$startsBlockLine` (`whitespaceDisplay.plugin.utils.ts`) therefore suppresses the line break for a block boundary with an opaque ancestor ABOVE it — the construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document; it is simply not the line-per-marker layout a USFM writer emits, which is the deliberate trade for a paste that survives. Pinned by the corpus sweep's now-unskipped `"sidebar (esb)"` and `"table with header and cells"` fixtures and by `unknownClipboardFidelity.test.tsx`. Tables needed one more thing on the paste side: `usfm.sty` classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the fragment tokenizer as a char token and its table-cell assembly — which only ran for paragraph tokens — never saw it, turning every cell into a loose `char` span inside an empty row. ParatextData derives a cell from the marker NAME, so the assembly now recognizes a cell marker on either token kind, guarded on an open `\tr` row. + ### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) **Fixed gap (2026-08-13, extended to every kind 2026-09-10), lexical-JSON flavor only:** the "exact node tree" promise did not hold for a selection containing an `UnknownNode` (figure, sidebar, periph, ref, optbreak). `@lexical/clipboard`'s JSON generator (`$appendNodesToJSON`, the function behind the `application/x-lexical-editor` flavor) computes node exclusion from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, hardcoded for EVERY copy-out destination `$appendNodesToJSON` itself handles, not only actual HTML generation (`'clone'` is never passed by any Lexical-shipped code path in the installed version). This change touches ONLY that flavor: `$appendNodesToHTML` (`@lexical/html`, the function behind `text/html`) computes the same `excludeFromCopy('html')` value, but `UnknownNode.exportDOM()` always returns `{element: null}` for every kind, and `$appendNodesToHTML` returns early on a null element BEFORE ever consulting the exclusion value it just computed — so `text/html` output is byte-identical before and after this fix, for every `UnknownNode` kind. @@ -137,13 +139,13 @@ These differences are documented and intentional; they are not implemented as wo Found by a corpus-style sweep (`clipboardCorpusRoundTrip.test.tsx`): for every fixture in the shared USJ round-trip corpus (`corpus-data.ts`), select the chapter's content, copy, paste into a fresh editor holding the same chapter header, and compare the resulting USJ to the source. Two of -the six failures below are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes -capable of representing the construct, so no paste-side fix is possible without a different -carrier. Two are real, out-of-scope structural gaps in how a paste's rebuild is scoped (the same -gap, reached by two different constructs). One is an out-of-scope interaction between a deferred -settle and the paste-scoped prefix dedup. One is ACCEPTED normalization that matches Paratext 9's -own behavior, not a bug at all. All six fixtures stay in the sweep as `it.skip`, not deleted, so a -future engine change un-skips them automatically instead of the gap going unnoticed. +the remaining failures are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes capable +of representing the construct, so no paste-side fix is possible without a different carrier. One is +a construct the fragment tokenizer has no assembly for. One is an out-of-scope interaction between +a deferred settle and the paste-scoped prefix dedup. One is ACCEPTED normalization that matches +Paratext 9's own behavior, not a bug at all. Every one of them stays in the sweep as `it.skip`, not +deleted, so a future engine change un-skips it automatically instead of the gap going unnoticed — +which is how the sidebar and table entries that used to sit here left the list. 1. **Cross-reference `` target wrapper (inherent):** USJ's `ref` element is a wrapper USFM itself never carried (`unknownUsfm.utils.ts`'s own doc comment: "USJ invented this container, @@ -154,32 +156,7 @@ future engine change un-skips them automatically instead of the gap going unnoti wrapper gone). A raw USFM export of this same fixture has the identical gap — not specific to clipboard mechanics. -2. **Sidebar `\esb`/`\esbe` (structural — a paste-rebuild SCOPING gap, not a missing tokenizer - rule; out of scope here):** the tokenizer itself is not the problem — `usfmFragmentToUsj.ts` - already implements the `\esb`/`\esbe` pairing (its `SIDEBAR_MARKER`/`SIDEBAR_END_MARKER` - assembly case tracks an open sidebar across tokens and closes it on `\esbe`). The real mechanism - is upstream and structural: a sidebar's nested `\p` child is a real `ParaNode`, and the copy - walker (`$selectionToUsfmText`) inserts a `\n` before any non-inline `ElementNode` boundary it - crosses — a `ParaNode` is non-inline, so a `\n` lands between `\esb \cat History\cat*` and the - nested paragraph's own content, even though both came from ONE sidebar. On paste, - the paste replays every `\n` as an `INSERT_PARAGRAPH_COMMAND` dispatch, so that single `\n` - turns into TWO sibling `ParaNode`s where the source had one sidebar wrapping one paragraph. Tier - 2 then re-tokenizes strictly per paragraph — `$requestTier2ForNode` (`tier2Rebuild.utils.ts`), - the only production call site, always invokes `$rebuildParas([current], context)` with a - single-element array — so the tokenizer's "current open sidebar" state can never span the two - separate `$rebuildParas` calls the two now-sibling paragraphs each trigger; the pairing that DOES - exist in `usfmFragmentToUsj.ts` never gets the chance to run across both lines at once. Result: - an UNCLOSED sidebar (`closed:"false"`, no content), the inner paragraph hoisted out to become a - top-level sibling, and a stray EMPTY paragraph with marker `"esbe"`. Tables hit the identical mode - — their rows and cells are real block-level `ElementNode`s - (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the copy walker's - non-inline-boundary `\n`-insertion rule fires for every one of them; see item 4 below. A real - fix means grouping a paste's newly-inserted SIBLING - paragraphs that originated from one selection back into a single rebuild fragment before Tier 2 - tokenizes — a Tier-2 rebuild-granularity change, genuinely out of this work item's scope. The - byte-level corruption stays pinned by the skip rather than fixed. - -3. **`closed="false"` char span followed by more paragraph content (inherent):** a `closed="false"` +2. **`closed="false"` char span followed by more paragraph content (inherent):** a `closed="false"` span has, by definition, no closing marker byte anywhere in its own USFM. When such a span is not the last thing in its paragraph (`Tell the Lord plainly.`), the copied text (`\nd Lord plainly.`) carries no byte marking where the span's content ends and the @@ -190,21 +167,19 @@ future engine change un-skips them automatically instead of the gap going unnoti confirming the ambiguity is specifically about trailing content after an implicit close, not `closed="false"` itself. -4. **Table rows and cells split into sibling paragraphs (structural — the SAME paste-rebuild - SCOPING gap as item 2, out of scope here):** a table's rows and cells are now real block-level - `ElementNode`s (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), not the - inline `UnknownNode`s they were when this sweep was first written, whose unconditionally-`true` - `isInline()` kept a table byte-contiguous within one paste-insertion unit. The copy walker - inserts a `\n` at every non-inline element boundary it crosses, so one table copies out as eight - lines (`\tr `, `\th1 Day`, `\th2 Tribe`, `\tr `, `\tc1 First`, `\tc2 Judah`, plus the flanking - prose). Paste replays each line as its own paragraph and Tier 2 re-tokenizes strictly per - paragraph, so the `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never sees - the whole table in one pass: the result is empty `table:row` wrappers with the cells stranded as - top-level sibling paragraphs. Same fix as item 2 — group a paste's newly-split sibling paragraphs - back into one rebuild fragment — and equally a Tier-2 rebuild-granularity change out of scope - here. - -5. **A pasted milestone that ejects content leaves the host's redundant paragraph glyph behind +3. **`\periph` degrades to an ordinary paragraph (tokenizer gap):** the copy is byte-faithful and + now lands on ONE line (`\periph Title Page|id="title"\mt1 The Title`), so this is no longer the + block-split the sidebar and table entries used to describe. Two `\periph`-specific gaps remain. + The fragment tokenizer has no periph assembly at all — nothing turns `\periph …` back into a + `periph` object, so the paste comes back as an ordinary paragraph whose marker happens to be + `periph`. And `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` + run on a marker that has NO closing bytes, so even with an assembly there is nothing in the text + marking where the attribute run ends and the content resumes. Closing this means giving the + tokenizer a periph assembly and deciding how a closer-less division marker carries attributes at + all — a construct-modelling question, not a clipboard one. Pinned as a loss (not left unpinned) + in `unknownClipboardFidelity.test.tsx`'s periph describe. + +4. **A pasted milestone that ejects content leaves the host's redundant paragraph glyph behind (structural — a paste-PROVENANCE lifetime gap, out of scope here):** the copied text is byte-correct (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host paragraph with no marker prefix of its own round-trips clean — so neither the copy walker nor the @@ -223,7 +198,7 @@ future engine change un-skips them automatically instead of the gap going unnoti tokenizer one — which also has to answer what happens if the user TYPES between the paste and the departure. Recorded rather than guessed at. -6. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a +5. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a bug):** isolated with a minimal non-corpus repro: pasting the literal text `"\p X"` (marker, its own required separator, and a SECOND, real content-leading space) into a fresh empty `"\p"` host produces `"\p X"` — one space, not two. The mechanism is `consumeSeparator()` @@ -257,21 +232,18 @@ The following items are recorded as deferred and not addressed in this plan: 6. **Popover expanded-note `isStandardView` gating issue:** An issue with note-expansion behavior in popovers has an open tracking document at `docs/superpowers/specs/2026-07-07-standard-view-followups.md`. This is not addressed in the current plan. -7. **Paste-rebuild scoping loses a sidebar's `\esb`/`\esbe` pairing across a copy-introduced paragraph split (corpus-found 2026-08-07):** a sidebar's nested paragraph is a real `ParaNode`, so copying a sidebar inserts a `\n` before it and pasting that `\n` splits one sidebar into two sibling paragraphs; Tier 2 rebuilds strictly per paragraph (`$requestTier2ForNode` always calls `$rebuildParas` with a single-element array), so the `\esb`/`\esbe` pairing `usfmFragmentToUsj.ts` already implements never sees both halves in the same tokenize pass. See "Known Lossy Constructs — Copy→Paste Round Trip" above (item 2) for the full mechanism. A real fix means grouping a paste's newly-split sibling paragraphs back into one rebuild fragment — a Tier-2 rebuild-granularity change, out of scope for this test-only task. - -8. **2SA corpus (`libs/test-data/src/data/2sa.usj.ts`) not included in the copy→paste corpus sweep:** this fixture IS single-chapter (one top-level chapter object), so the earlier "multi-chapter/book-level" exclusion reason recorded in an earlier draft of this work was wrong. It was tried against the sweep's harness directly and does not round-trip clean: it hits the sidebar gap above three times, plus several additional, unrelated fidelity gaps (an empty `\b` blank-line paragraph, a verse's derived `sid` attribute, a `\ref` target) that this sweep's one-construct-per-fixture design has no clean way to itemize as a single byte diff. `tier2Rebuild.corpus.test.tsx` already exercises this fixture, but for a narrower, different property (an unedited `$rebuildParas` call refusing as a fixed point) that does not cover the paste path this sweep does — so the exclusion is a real coverage gap, not a redundant re-test. Recorded here rather than force-fit into the sweep. +7. **2SA corpus (`libs/test-data/src/data/2sa.usj.ts`) not included in the copy→paste corpus sweep:** this fixture IS single-chapter (one top-level chapter object), so the earlier "multi-chapter/book-level" exclusion reason recorded in an earlier draft of this work was wrong. It was tried against the sweep's harness directly and did not round-trip clean: it hit the sidebar gap three times — since closed — plus several additional, unrelated fidelity gaps (an empty `\b` blank-line paragraph, a verse's derived `sid` attribute, a `\ref` target) that this sweep's one-construct-per-fixture design has no clean way to itemize as a single byte diff. It has not been re-measured since the sidebar gap closed, so how much of the residual survives is unknown; re-running it against the harness is the first step whenever this is picked up. `tier2Rebuild.corpus.test.tsx` already exercises this fixture, but for a narrower, different property (an unedited `$rebuildParas` call refusing as a fixed point) that does not cover the paste path this sweep does — so the exclusion is a real coverage gap, not a redundant re-test. Recorded here rather than force-fit into the sweep. -9. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. +8. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. -10. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). +9. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). -11. **Lexical-flavor copy/paste of figure/table/sidebar/periph/ref UnknownNode kinds (measured, 2026-08-13):** S2's `excludeFromCopy` fix above is scoped to "optbreak" only; every other `UnknownNode` kind keeps the OLD, still-excluding behavior, PENDING per-kind paste verification this task did not do. The residual shape was directly measured (not just inferred from the mechanism), and it is WORSE, in user terms, than the optbreak symptom this task fixed: a same-namespace lexical-flavor copy/paste of `
caption
` renders the figure's FULL literal USFM bytes on screen after paste — `\fig caption|src="cn01617.jpg" size="span" ref="1.18"\fig*` — because the figure's hoisted marker/attribute decorators still `decorate()` their own literal text as loose, unwrapped siblings, giving a completely convincing on-screen appearance that the figure came through intact. The pasted document's USJ, measured against that same on-screen result, silently drops the figure node AND every one of its attributes entirely — only its real caption TextNode child survives, merged into the surrounding paragraph's prose (`content: ["Before caption after."]`, no `figure` entry, no `file`/`size`/`ref` anywhere). A save at this point would silently persist the loss with no error, no warning, and a screen that still shows the figure's bytes — genuine silent data loss with a convincing display. `table`, `sidebar`, `periph`, and `ref` share the identical `excludeFromCopy`/`exportDOM(null)` mechanism (the same content-free `ImmutableTypedTextNode` display-decorator shape `unknownDisplayParts` builds for every kind) and are expected, not merely suspected, to exhibit the same class of residual, though only the figure case above was directly measured. Fixing this for all five kinds the way S2 now does for optbreak is a plausible, low-risk follow-up — narrowing `excludeFromCopy` further, one verified kind at a time — but is out of scope here. +10. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 4 above. -12. **The same paste-rebuild scoping gap now reaches TABLES:** item 7's sidebar mechanism is no longer sidebar-specific. A table's rows and cells are real block-level `ElementNode`s (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the copy walker's non-inline-boundary `\n` rule fires for every one of them and a single table pastes back as eight sibling paragraphs Tier 2 rebuilds one at a time. The `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never sees the whole table in one pass. Same fix, same scope call as item 7 — grouping a paste's newly-split sibling paragraphs back into one rebuild fragment. See "Known Lossy Constructs" item 4 above for the byte-level shape. +11. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. -13. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 5 above. +12. **`\periph` has no tokenizer assembly, so a copied periph pastes back as an ordinary paragraph:** the sidebar/table paste-rebuild scoping gaps that used to sit in this list are closed — an opaque construct now copies out on ONE line (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) so a single-paragraph rebuild can reassemble it. `\periph` does not benefit, for two reasons of its own. `usfmFragmentToUsj.ts` has no periph assembly at all (it has `\esb`/`\esbe` for sidebars and `\tr`/`\th`/`\tc` for tables), so nothing turns `\periph …` back into a `periph` object; and `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` run on a marker with NO closing bytes, so nothing in the text marks where the attribute run ends and the content resumes. The paste therefore comes back as a paragraph whose marker is `periph`, carrying its own attribute bytes as literal content. Closing this means giving the tokenizer a periph assembly AND deciding how a closer-less division marker carries attributes at all — a construct-modelling question rather than a clipboard one, so it is recorded rather than guessed at. Pinned as a loss in `unknownClipboardFidelity.test.tsx`. See "Known Lossy Constructs" item 3 above. -14. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. ## Test Mapping (S1–S7) @@ -285,9 +257,9 @@ The following items are recorded as deferred and not addressed in this plan: | **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | -| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; six known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"` fixture (tight spacing, no flanking spaces) is swept clean, not skipped | +| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; four known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **S2 (per-kind opaque constructs)** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind, each asserting the lexical-flavor round trip and, where a carrier genuinely cannot express the construct, asserting the LOSS by name (the plain carrier's block split for sidebar/periph/table; USFM having no bytes at all for a `ref` wrapper). Mounts the full `Editor`, not `MarkerEditPlugin` alone | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | | **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts index ac3cb05de..4fcbc0549 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts @@ -882,6 +882,28 @@ export function usfmFragmentToUsjContent( table = undefined; tableRow = undefined; }; + /** + * Open a cell in the current row and make it the content container: text, char spans, notes and + * verses then flow into it through the ordinary `para`-based container logic. Shared by both + * token kinds a cell marker can arrive as — a stylesheet that knows `\th1`/`\tc1` classifies + * them as Character (usfm.sty does), one that does not delivers them as unknown paragraph + * markers, and ParatextData derives a cell from the NAME either way. + */ + const pushTableCell = (row: MarkerObject, marker: string, cellMatch: RegExpExecArray) => { + closeCharStack(); + const [, alignInfix, spanStart, spanEnd] = cellMatch; + // The cell keeps only the starting column in its marker (`thc3-4` → `thc3`); the span width + // becomes `colspan`, a string of columns spanned (`thc3-4` → "2"). + const cell: TableCellObject = { + type: "table:cell", + marker: spanEnd ? marker.slice(0, marker.indexOf("-")) : marker, + align: TABLE_CELL_ALIGN_BY_INFIX[alignInfix], + content: [], + }; + if (spanEnd) cell.colspan = String(Number(spanEnd) + 1 - Number(spanStart)); + getContent(row).push(cell); + para = cell; + }; const closeSidebar = (terminated: boolean) => { if (!sidebar) return; // Only `\esbe` terminates a sidebar explicitly; an implicit close (fragment end or a @@ -1240,21 +1262,7 @@ export function usfmFragmentToUsjContent( // isRecognizedTableCell) is an unknown marker that ENDS the table (and the next // `\tr` starts a fresh one). if (cellMatch && isRecognizedTableCell(cellMatch)) { - closeCharStack(); - const [, alignInfix, spanStart, spanEnd] = cellMatch; - // The cell keeps only the starting column in its marker (`thc3-4` → `thc3`); - // the span width becomes `colspan`, a string of columns spanned (`thc3-4` → "2"). - const cell: TableCellObject = { - type: "table:cell", - marker: spanEnd ? token.marker.slice(0, token.marker.indexOf("-")) : token.marker, - align: TABLE_CELL_ALIGN_BY_INFIX[alignInfix], - content: [], - }; - if (spanEnd) cell.colspan = String(Number(spanEnd) + 1 - Number(spanStart)); - getContent(tableRow).push(cell); - // The cell becomes the current content container: text, char spans, notes, and - // verses flow into it through the ordinary `para`-based container logic. - para = cell; + pushTableCell(tableRow, token.marker, cellMatch); break; } } @@ -1331,6 +1339,20 @@ export function usfmFragmentToUsjContent( break; } case "charOpen": { + // A cell marker inside an OPEN row is a cell, whichever token kind the stylesheet's + // classification made it: usfm.sty declares `\th1`/`\tc1`/… as Character styles, so with a + // real project sheet they arrive here rather than as paragraph tokens, and without this a + // table copied out of the editor pastes back as loose char spans inside empty rows. + // Guarded on an open row — only `\tr` opens one — so a char span that merely shares a + // cell marker's name outside a table keeps its ordinary resolution, and a NESTED span + // (`\+tc1`) is never a cell. + if (!note && !isNoteContext && tableRow && !token.isNested) { + const cellMatch = TABLE_CELL_MARKER_REGEX.exec(token.marker); + if (cellMatch && isRecognizedTableCell(cellMatch)) { + pushTableCell(tableRow, token.marker, cellMatch); + break; + } + } // A new non-nested char marker auto-closes open char styles (PT9) — but never // across an open note's boundary: the frames enclosing the note stay open. if (!token.isNested) { diff --git a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx index 283cb4fe1..16cbe187e 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx @@ -23,12 +23,13 @@ * * The OTHER USJ corpus in this repo, `libs/test-data/src/data/2sa.usj.ts` (141 paragraphs; IS * single-chapter — one top-level chapter object, despite the size), is deliberately NOT included - * here. It was tried against this exact harness and does not round-trip clean: it hits the - * `"sidebar (esb)"` gap below three times (2SA's own sidebar coverage), PLUS several additional, - * unrelated fidelity gaps this sweep's one-construct-per-fixture design has no clean way to - * itemize as a single byte diff (an empty `\b` blank-line paragraph folds into literal text inside - * a `p` paragraph; a verse's derived `sid` attribute is dropped; a `\ref` cross-reference target - * degrades differently than the corpus's own dedicated `ref` fixture above). `tier2Rebuild.corpus + * here. It was tried against this exact harness and did not round-trip clean: it hit the sidebar + * gap three times (2SA's own sidebar coverage) — since closed — PLUS several additional, unrelated + * fidelity gaps this sweep's one-construct-per-fixture design has no clean way to itemize as a + * single byte diff (an empty `\b` blank-line paragraph folds into literal text inside a `p` + * paragraph; a verse's derived `sid` attribute is dropped; a `\ref` cross-reference target degrades + * differently than the corpus's own dedicated `ref` fixture above). It has not been re-measured + * since the sidebar gap closed, so how much of that residual survives is unknown. `tier2Rebuild.corpus * .test.tsx` already exercises this exact fixture, but for a narrower, DIFFERENT property (an * unedited `$rebuildParas` call refusing as a fixed point) — that coverage does not include the * paste path (NBSP normalization, `\c`/`\id` strip, own-marker-wins dedup) this sweep does, so @@ -117,30 +118,6 @@ function chapterHeaderSkeletonUsj(usj: Usj): Usj { * `["See Genesis 1:1 for details."]`, the `ref` wrapper gone. Inherent to the construct — a raw * USFM export of this same fixture has the identical gap, independent of clipboard mechanics. * - * - **"sidebar (esb)"** — NOT a tokenizer gap: `usfmFragmentToUsj.ts` already implements the - * `\esb`/`\esbe` pairing (its `SIDEBAR_MARKER`/`SIDEBAR_END_MARKER` assembly case tracks an open - * sidebar across tokens and closes it on `\esbe`). The real mechanism is structural, upstream of - * tokenizing: a sidebar's nested `\p` child is a real `ParaNode`, and `$selectionToUsfmText` (the - * copy walker) inserts a `\n` before any non-inline `ElementNode` boundary it crosses — a - * `ParaNode` is non-inline, so a `\n` lands between `\esb \cat History\cat*` and the nested - * paragraph's own content, even though both came from ONE sidebar. On paste, the line replay - * dispatches an `INSERT_PARAGRAPH_COMMAND` for every `\n`, so that single `\n` turns into TWO - * sibling `ParaNode`s where the source had one sidebar wrapping one paragraph. Tier 2 then - * re-tokenizes strictly per paragraph — `$requestTier2ForNode` (tier2Rebuild.utils.ts), the only - * production call site, always invokes `$rebuildParas([current], context)` with a single-element - * array — so the tokenizer's "current open sidebar" state can never span the two separate - * `$rebuildParas` calls the two now-sibling paragraphs each trigger; each is tokenized alone, and - * the pairing that DOES exist in `usfmFragmentToUsj.ts` never gets the chance to run across both - * lines at once. Result: an UNCLOSED sidebar (`closed:"false"`, no content), the inner paragraph - * hoisted OUT to become a top-level sibling, and a stray EMPTY paragraph with marker `"esbe"`. - * Tables hit the identical mode — their rows and cells are real block-level `ElementNode`s - * (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`), so the walker's - * non-inline-boundary rule fires for every one of them; see the "table with header and cells" - * entry below. A real fix means grouping a paste's newly-inserted SIBLING paragraphs that originated - * from one selection back into a single rebuild fragment before Tier 2 tokenizes — a Tier-2 - * architecture change (rebuild granularity, not a per-paragraph tweak), genuinely out of this - * work item's scope. The byte-level corruption stays pinned here rather than fixed. - * * - **"closed=false body char span (implicit close, no closer)"** — a `closed="false"` char span * has, by definition, no closing marker byte anywhere in its own USFM. When such a span is not * the last thing in its paragraph (`Tell the Lord plainly.`), the @@ -151,17 +128,6 @@ function chapterHeaderSkeletonUsj(usj: Usj): Usj { * sibling `"unclosed note (closed=false)"` fixture, whose unclosed span IS the last thing in its * paragraph, has no such trailing content to lose and round-trips clean. * - * - **"table with header and cells"** — the SAME rebuild-scoping gap as the sidebar entry above, - * reaching tables now that a table's rows and cells are real block-level `ElementNode`s - * (`ImmutableTableNode`/`ImmutableTableRowNode`/`ImmutableTableCellNode`) rather than the inline - * `UnknownNode`s they used to be. The copy walker inserts a `\n` at every non-inline element - * boundary it crosses, so one table copies out as eight lines (`\tr `, `\th1 Day`, `\th2 Tribe`, - * …); paste replays each line as its own paragraph, and Tier 2 re-tokenizes strictly per - * paragraph, so the `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never - * sees the whole table in one pass. Result: empty `table:row` wrappers and the cells stranded as - * sibling paragraphs. The fix is the same one the sidebar entry names — grouping a paste's - * newly-split sibling paragraphs back into one rebuild fragment — and is equally out of scope. - * * - **"milestones (ts)"** — a COMPOUND of two deliberate behaviors, neither of them a byte-level * copy or normalization defect: the copied text is byte-correct * (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host with NO marker @@ -196,11 +162,6 @@ const KNOWN_LOSSY: { name: string; reason: string }[] = [ name: "cross-reference ref target", reason: "USJ wrapper carries no USFM bytes; content survives as prose", }, - { - name: "sidebar (esb)", - reason: - "copy's \\n before the sidebar's nested \\p splits one paste into two ParaNodes; Tier 2 rebuilds per-paragraph so the sidebar pairing usfmFragmentToUsj.ts already has never sees both", - }, { name: "closed=false body char span (implicit close, no closer)", reason: "no closer byte to mark where an unclosed span's content ends before trailing prose", @@ -210,11 +171,6 @@ const KNOWN_LOSSY: { name: string; reason: string }[] = [ reason: "consumeSeparator() eats the whole whitespace run after a marker, matching P9's NormalizeUsfm parity — accepted, not a bug", }, - { - name: "table with header and cells", - reason: - "a table's rows and cells are real block-level ElementNodes, so copy puts each on its own line and paste splits one table into eight sibling paragraphs Tier 2 rebuilds one at a time — the same rebuild-scoping gap as the sidebar above", - }, { name: "milestones (ts)", reason: diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx index 9c498e815..cef1176d5 100644 --- a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -170,19 +170,9 @@ describe("figure (UnknownNode) copy→paste across all three payload shapes", () describe("sidebar (UnknownNode) copy→paste across all three payload shapes", () => { const usj = corpusUsj("sidebar (esb)"); - it("full payload (lexical flavor) round-trips the sidebar whole — its category attribute and nested paragraph included", async () => { - expect(await pastedUsjFor(usj, "full")).toEqual(usj); - }); - - (["plain", "plain+html"] as const).forEach((shape) => { - // The plain carrier cannot express a sidebar in one line: the copy walker puts the nested `\p` - // on its own line, paste replays that newline as a paragraph split, and Tier 2 re-tokenizes - // one paragraph at a time, so the `\esb`/`\esbe` pairing the tokenizer already implements - // never sees both halves at once. The loss is asserted, not the fidelity. - it(`${shape} payload loses the sidebar's pairing — the copy-introduced paragraph split outlives the per-paragraph rebuild`, async () => { - const pasted = await pastedUsjFor(usj, shape); - expect(pasted).not.toEqual(usj); - expect(objectsOfType(pasted, "sidebar")[0]?.content ?? []).toEqual([]); + SHAPES.forEach((shape) => { + it(`${shape} payload round-trips the sidebar whole — its category attribute and nested paragraph included`, async () => { + expect(await pastedUsjFor(usj, shape)).toEqual(usj); }); }); }); @@ -200,12 +190,19 @@ describe("periph (UnknownNode) copy→paste across all three payload shapes", () }); (["plain", "plain+html"] as const).forEach((shape) => { - // Same block-split mechanism as the sidebar above: `\periph` has no closing bytes at all, so - // once the nested paragraph lands on its own line there is nothing to reunite the two. - it(`${shape} payload loses the periph's nested paragraph — the copy-introduced split outlives the per-paragraph rebuild`, async () => { + // Not a block split: an opaque construct copies out on ONE line (`$startsBlockLine`, + // `whitespaceDisplay.plugin.utils.ts`), which is what lets the sidebar above round-trip. Two + // `\periph`-specific gaps stop periph taking the same route: the fragment tokenizer has no + // periph assembly (nothing turns `\periph …` back into a `periph` object), and periph's display + // renders its non-`alt` attributes as a `|name="value"` run on a marker that has no closing + // bytes, so nothing marks where the attribute run ends and the content resumes. The paste + // therefore comes back as an ORDINARY paragraph whose marker is `periph`, carrying the + // attribute bytes as literal text. + it(`${shape} payload loses the periph construct — it re-tokenizes as an ordinary paragraph carrying its own attribute bytes as text`, async () => { const pasted = await pastedUsjFor(usj, shape); - expect(objectsOfType(pasted, "periph")).not.toEqual(objectsOfType(usj, "periph")); - expect(objectsOfType(pasted, "periph")[0]?.content ?? []).toEqual([]); + expect(objectsOfType(pasted, "periph")).toEqual([]); + const degraded = objectsOfType(pasted, "para").find((item) => item.marker === "periph"); + expect(degraded?.content).toEqual(['Title Page|id="title"']); }); }); }); @@ -230,15 +227,9 @@ describe("ref (UnknownNode) copy→paste across all three payload shapes", () => describe("table (ImmutableTable* nodes, not UnknownNode) copy→paste across all three payload shapes", () => { const usj = corpusUsj("table with header and cells"); - it("full payload (lexical flavor) round-trips the table whole — rows, cells, and their markers", async () => { - expect(await pastedUsjFor(usj, "full")).toEqual(usj); - }); - - (["plain", "plain+html"] as const).forEach((shape) => { - // A table's rows and cells are real block-level nodes, so the plain carrier spreads one table - // over eight lines and the paste replays each as its own paragraph. - it(`${shape} payload loses the table's assembly — every row and cell copies onto its own line`, async () => { - expect(await pastedUsjFor(usj, shape)).not.toEqual(usj); + SHAPES.forEach((shape) => { + it(`${shape} payload round-trips the table whole — rows, cells, their markers and derived alignment`, async () => { + expect(await pastedUsjFor(usj, shape)).toEqual(usj); }); }); }); diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 25820a63b..7962ed523 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -47,7 +47,7 @@ import { NBSP, textTypeState, } from "shared"; -import { $isImmutableNoteCallerNode } from "shared-react"; +import { $isImmutableNoteCallerNode, $opaqueBlockAncestor } from "shared-react"; /** Spaces in runs display as NBSP so they are visible while typing. */ export function $displayWhitespaceTransform(node: TextNode): void { @@ -502,6 +502,31 @@ function $noteCallerText(callerNode: LexicalNode): string { return caller || GENERATOR_NOTE_CALLER; } +/** + * Whether this block-level element opens a new LINE of copied USFM, or is engine-internal to an + * opaque construct and must stay on the construct's own line. + * + * A sidebar's nested `\p` and a table's rows and cells are real block-level nodes, so the walk's + * "one `\n` per non-inline boundary" rule would spread ONE construct over several lines. A paste of + * that text replays every `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph + * at a time (`$requestTier2ForNode` always rebuilds a single-element array), so the `\esb`/`\esbe` + * and `\tr`/`\th`/`\tc` assembly `usfmFragmentToUsj.ts` already implements never sees the whole + * construct in one pass: the sidebar comes back unclosed with its paragraph hoisted out, and the + * table comes back as empty row wrappers with the cells stranded as sibling paragraphs. + * + * Keeping the construct byte-contiguous is what lets a single-paragraph rebuild reassemble it. USFM + * markers self-delimit, so the one-line form is valid USFM of the same document — just not the + * line-per-marker layout a USFM writer would emit, which is a deliberate trade for a paste that + * survives. + * + * The construct's own outermost node still starts its own line: only a block boundary with an + * opaque ancestor ABOVE it is suppressed, so a table stays separated from the prose around it. + */ +function $startsBlockLine(node: LexicalNode): boolean { + const parent = node.getParent(); + return !parent || $opaqueBlockAncestor(parent) === undefined; +} + /** * Source-faithful USFM text of `selection` — the `text/plain` leg of Standard-view copy/cut. Walks * `selection.getNodes()` the same way `RangeSelection.getTextContent()` does (single `\n` between @@ -520,6 +545,8 @@ function $noteCallerText(callerNode: LexicalNode): string { * instead of becoming phantom spaces; every other NBSP (marker-trailing spaces, a char span's * structural leading separator, a verse's own marker-to-number gap) represents a real source * space and still maps to one. Data-NBSP (displayed as `~`) is untouched either way. + * 3. A block boundary INSIDE an opaque construct contributes no line break, so the construct + * copies out on ONE line (see {@link $startsBlockLine}). */ export function $selectionToUsfmText(selection: RangeSelection): string { const nodes = selection.getNodes(); @@ -533,7 +560,7 @@ export function $selectionToUsfmText(selection: RangeSelection): string { let prevWasElement = true; for (const node of nodes) { if ($isElementNode(node) && !node.isInline()) { - if (!prevWasElement) text += "\n"; + if (!prevWasElement && $startsBlockLine(node)) text += "\n"; prevWasElement = !node.isEmpty(); continue; } From 969126430016c664313eda819c5004e44f03dbd0 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 13:59:03 -0500 Subject: [PATCH 10/44] fix: carry a paste's provenance across a deferred Tier-2 settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A paste's own update does not always get to perform the rebuild its bytes ask for. A paragraph whose re-tokenization would EJECT content out of a milestone deliberately waits for caret departure, and by then `pasteRebuildArmed` — a per-commit flag — has reset, so the departure settle rebuilt the pasted line as though the user had TYPED it: the host's `\p ` glyph and the pasted line's own `\p ` literal both survived, leaving a stray empty paragraph ahead of the correct one. `Tier2Context.pastePendedKeys` gives the arm a lifetime tied to the pended work instead of to the commit clock. A pend recorded while the paste's own update is armed records its key; the settle that routes that key to a rebuild consumes the provenance and passes it down. Keying it that way is what keeps it out of typed input: keys are only added inside a paste's own update, consumed by the first rebuild that reaches them, and pruned by the per-commit update listener as soon as they stop pending — and text typed beside the pasted bytes pends its own key, which carries none. Closes the last "Known Lossy" clipboard entry. The corpus `milestones (ts)` fixture it was filed against does not in fact reach the deferred path (its milestone ejects nothing), so it is un-skipped and swept clean; the shape that does reach it is pinned directly, together with its counter-pin that the same bytes with no paste behind them still split with an empty predecessor. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 37 ++------ .../editor/markerEdit/MarkerEditPlugin.tsx | 8 ++ .../clipboardCorpusRoundTrip.test.tsx | 19 ---- .../markerEdit/markerEditTier1.utils.test.tsx | 5 ++ .../markerEdit/markerEditTier1.utils.ts | 25 +++++- .../markerEditTier2Trigger.utils.test.tsx | 4 + .../markerEditTier2Trigger.utils.ts | 18 +++- .../markerEdit/markerPasteFidelity.test.tsx | 87 +++++++++++++++++++ .../milestoneAttributeSettle.test.tsx | 2 + .../editor/markerEdit/tier2Rebuild.utils.ts | 56 +++++++++--- 10 files changed, 200 insertions(+), 61 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 109090e86..64e15961e 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -55,6 +55,8 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra - **A multi-line payload is replayed line by line through `INSERT_PARAGRAPH_COMMAND`**, not through a bare `selection.insertParagraph()` per newline. Going through the command is what makes a paste landing inside a character-style stack close and reopen that stack at every line break, exactly as Enter does — a raw split tears the span, dropping its closing marker and leaving every line after the first outside the reopened style. The selection is removed up front rather than relying on the first `insertText` to replace it, so a payload whose first line is empty (a leading newline) cannot split a still-selected range. `MarkerEditContext.splitExpected` is armed before the first line goes in — the command's own handler arms it for each dispatch, but the first line is inserted before any dispatch runs — and `Tier2Context.pasteRebuildArmed` is armed unconditionally, single-line pastes included, since a single line can trigger the own-marker-prefix dedup just as easily. Foreign HTML formatting is dropped by design — Standard view is markers-as-text, exactly like P9's Standard view where the reformat pipeline re-tokenizes everything. This generalizes the existing NBSP-gated handler and kills both live-observed corruptions plus the latent doubled-glyph path (re-imported Standard-view HTML producing a `CharNode` via `data-marker` importDOM *and* literal `\nd` text, since MarkerNode has no importDOM). +- **A paste's provenance outlives its own commit, keyed to the pended work rather than to the clock:** the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, `tier2Rebuild.utils.ts`) drops a host paragraph's now-redundant `\p ` glyph when the pasted line carries its own, and it is deliberately a PASTE-shape recognition — typing the same bytes keeps the engine's P9-parity split-with-empty behavior. `Tier2Context.pasteRebuildArmed` answers "is this the paste's own update", which is not enough on its own: a paragraph whose re-tokenization would EJECT content out of a milestone deliberately does not settle in the commit that produced it (`markerEditTier2Trigger.utils.ts` — ejection rearranges the line under a caret the user is still on), so its rebuild reaches a later, caret-departure settle with the flag long since reset, and the host was left with a stray empty paragraph ahead of the correct one. `Tier2Context.pastePendedKeys` closes that: a pend recorded while the paste's own update is armed also records its KEY, and the settle that finally routes that key to a rebuild consumes the provenance and passes it down. Scoping it by key is what keeps it out of typed input — a key is only ever added inside a paste's own update, it is consumed by the first rebuild it reaches, the plugin's per-commit update listener prunes any key that has stopped pending, and text typed beside the pasted bytes pends its OWN key, which carries no provenance. Answering "what if the user types between the paste and the departure": the typed bytes get an ordinary typed rebuild, and the paste's deferred rebuild still gets the paste's. + - **Positional NBSP normalization (replaces the blanket NBSP→`~`), three-part rule:** (1) a leading NBSP — string-start or right after a newline — is a structural separator with nothing in front of it to match against and becomes a space; (2) an NBSP immediately FOLLOWING a marker token (`\marker`, nested `\+marker`, either's closer, or a milestone's anonymous `\*` self-closer) is the required opener/closer separator and becomes a space; (3) an NBSP immediately PRECEDING a marker token is a structural spacer with no source counterpart and is DROPPED entirely (neither spaced nor kept as data) — `createNote` (`usj-editor.adaptor.ts`) appends a spacer after EVERY note child, not just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller; a browser-hop/html-derived copy of a collapsed note therefore carries structural NBSPs on BOTH sides of its interior markers, not only after the opener. Every remaining NBSP is user data → `~`. This is P9's `PostprocessUsfm` model. Live repro that mandates it (2026-08-07): pasting P10's own copied footnote back produced `\f~ \fr~1:1 ~ \ft~Caller test.~ \f*` (every display-NBSP became `~`, breaking marker recognition); a browser-hop paste of `\nd …\nd*` produced `\nd~light … \nd*` with an unmatched pair. **Rule precedence when both position rules match the same NBSP:** a closed-children note shape (`\xo*` immediately followed by NBSP immediately followed by `\xt`) sits after a closing marker AND before the next opening marker at once; the after-marker rule wins and the NBSP becomes a space rather than being dropped, because `$normalizePastedNbsp`'s `.replace` chain applies `AFTER_MARKER_NBSP` before `BEFORE_MARKER_NBSP`, consuming the NBSP into a space before the before-marker pass ever sees it. The tie-break is correct, not incidental: an after-closer NBSP in ordinary body text (`\nd*and`) is a real content space that dropping would silently lose, and the case is reachable only via a foreign NBSP-preserving carrier — P10's own `text/plain` copy (S1) never emits an NBSP in this position to begin with. - **Optbreak (`//`) is a non-backslash marker token the positional NBSP rule does not recognize (characterized, not a live gap):** `AFTER_MARKER_NBSP`/`BEFORE_MARKER_NBSP` (`whitespaceDisplay.plugin.utils.ts`) only match `\`-shaped tokens, so an NBSP adjacent to `//` is not treated as display whitespace and falls to the blanket `~` rule like any other interior NBSP. This is UNREACHABLE via P10's own copy, and DOUBLY so: (1) S1 above establishes `text/plain` never carries an NBSP next to `//` to begin with, and the plain payload is always preferred when present; (2) even if `text/plain` were somehow absent, P10's own `text/html` never carries `//` at ALL for an optbreak (see the "Known Accepted Asymmetries" entry below), so there is no `//` for a foreign-sourced NBSP to even be adjacent to in P10's own html either. Reachable only via a genuinely foreign clipboard source supplying its OWN `text/html` (never P10's) with no `text/plain` at all: a synthetic `

before // after

`-shaped payload settles to `before~// after` — the optbreak token itself still recognizes correctly (the tokenizer's `//` split is a plain string `.split("//")`, unaffected by an adjacent `~`), and the NBSP itself is genuine data FROM that foreign source (not a byte P10 invented) — the `~` it becomes is the correct display form for real data-NBSP under the "every remaining NBSP is user data → `~`" rule above, just landing one position earlier than a marker-adjacent rule would recognize. Characterized (not extended/fixed — no live repro reaches it) in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). @@ -141,11 +143,10 @@ shared USJ round-trip corpus (`corpus-data.ts`), select the chapter's content, c fresh editor holding the same chapter header, and compare the resulting USJ to the source. Two of the remaining failures are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes capable of representing the construct, so no paste-side fix is possible without a different carrier. One is -a construct the fragment tokenizer has no assembly for. One is an out-of-scope interaction between -a deferred settle and the paste-scoped prefix dedup. One is ACCEPTED normalization that matches +a construct the fragment tokenizer has no assembly for. One is ACCEPTED normalization that matches Paratext 9's own behavior, not a bug at all. Every one of them stays in the sweep as `it.skip`, not deleted, so a future engine change un-skips it automatically instead of the gap going unnoticed — -which is how the sidebar and table entries that used to sit here left the list. +which is how the sidebar, table and milestone entries that used to sit here left the list. 1. **Cross-reference `` target wrapper (inherent):** USJ's `ref` element is a wrapper USFM itself never carried (`unknownUsfm.utils.ts`'s own doc comment: "USJ invented this container, @@ -179,26 +180,7 @@ which is how the sidebar and table entries that used to sit here left the list. all — a construct-modelling question, not a clipboard one. Pinned as a loss (not left unpinned) in `unknownClipboardFidelity.test.tsx`'s periph describe. -4. **A pasted milestone that ejects content leaves the host's redundant paragraph glyph behind - (structural — a paste-PROVENANCE lifetime gap, out of scope here):** the copied text is - byte-correct (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host - paragraph with no marker prefix of its own round-trips clean — so neither the copy walker nor the - positional NBSP rule is involved. Two deliberate behaviors compound instead. A milestone whose - rebuild would EJECT content does NOT settle inside the commit that produced it - (`markerEditTier2Trigger.utils.ts`: ejection moves bytes out of the milestone and past a closer, - so applying it the instant the `\*` lands would rearrange the line under a caret the user is - still on); it pends until caret departure, and the departure settle then performs the identical - rebuild correctly. But `Tier2Context.pasteRebuildArmed` — the flag that lets - `$withoutRedundantOwnPrefix` drop the host's now-redundant `\p ` glyph when the pasted line - carries its own — is armed only for the PASTE's own update. By the time the deferred settle runs, - the rebuild no longer knows the fragment came from a paste, so it takes the TYPED path - (split-with-empty, the P9-parity behavior PT-4201 deliberately kept for typed input) and the - document ends with an empty `\p` paragraph ahead of the correct one. Closing it means giving the - dedup's arm a lifetime that spans the deferred settle — a paste-provenance change, not a - tokenizer one — which also has to answer what happens if the user TYPES between the paste and the - departure. Recorded rather than guessed at. - -5. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a +4. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a bug):** isolated with a minimal non-corpus repro: pasting the literal text `"\p X"` (marker, its own required separator, and a SECOND, real content-leading space) into a fresh empty `"\p"` host produces `"\p X"` — one space, not two. The mechanism is `consumeSeparator()` @@ -238,11 +220,9 @@ The following items are recorded as deferred and not addressed in this plan: 9. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). -10. **A pasted marker-bearing line whose rebuild PENDS loses the own-marker-prefix dedup:** `Tier2Context.pasteRebuildArmed` is armed only for the paste's own update, so a paragraph whose rebuild is deliberately deferred to the caret-departure settle (currently: a milestone that would eject content — `markerEditTier2Trigger.utils.ts`) reaches that settle with no record that its bytes came from a paste, and takes the TYPED split-with-empty path instead. The user sees a stray empty paragraph ahead of the correct one. The construct is narrow (the pasted first line must carry its own paragraph-kind marker literal AND its rebuild must pend), but the fix is not: giving the arm a lifetime that spans the deferred settle has to answer what happens if the user TYPES between the paste and the departure, which is exactly the product decision PT-4201 refused to make for typed input. Recorded rather than guessed at. See "Known Lossy Constructs" item 4 above. - -11. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. +10. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. -12. **`\periph` has no tokenizer assembly, so a copied periph pastes back as an ordinary paragraph:** the sidebar/table paste-rebuild scoping gaps that used to sit in this list are closed — an opaque construct now copies out on ONE line (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) so a single-paragraph rebuild can reassemble it. `\periph` does not benefit, for two reasons of its own. `usfmFragmentToUsj.ts` has no periph assembly at all (it has `\esb`/`\esbe` for sidebars and `\tr`/`\th`/`\tc` for tables), so nothing turns `\periph …` back into a `periph` object; and `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` run on a marker with NO closing bytes, so nothing in the text marks where the attribute run ends and the content resumes. The paste therefore comes back as a paragraph whose marker is `periph`, carrying its own attribute bytes as literal content. Closing this means giving the tokenizer a periph assembly AND deciding how a closer-less division marker carries attributes at all — a construct-modelling question rather than a clipboard one, so it is recorded rather than guessed at. Pinned as a loss in `unknownClipboardFidelity.test.tsx`. See "Known Lossy Constructs" item 3 above. +11. **`\periph` has no tokenizer assembly, so a copied periph pastes back as an ordinary paragraph:** the sidebar/table paste-rebuild scoping gaps that used to sit in this list are closed — an opaque construct now copies out on ONE line (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) so a single-paragraph rebuild can reassemble it. `\periph` does not benefit, for two reasons of its own. `usfmFragmentToUsj.ts` has no periph assembly at all (it has `\esb`/`\esbe` for sidebars and `\tr`/`\th`/`\tc` for tables), so nothing turns `\periph …` back into a `periph` object; and `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` run on a marker with NO closing bytes, so nothing in the text marks where the attribute run ends and the content resumes. The paste therefore comes back as a paragraph whose marker is `periph`, carrying its own attribute bytes as literal content. Closing this means giving the tokenizer a periph assembly AND deciding how a closer-less division marker carries attributes at all — a construct-modelling question rather than a clipboard one, so it is recorded rather than guessed at. Pinned as a loss in `unknownClipboardFidelity.test.tsx`. See "Known Lossy Constructs" item 3 above. ## Test Mapping (S1–S7) @@ -257,9 +237,10 @@ The following items are recorded as deferred and not addressed in this plan: | **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | -| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; four known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | +| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | | **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone | +| **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | | **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | diff --git a/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx b/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx index 49f599952..2f617c6ff 100644 --- a/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx +++ b/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx @@ -554,6 +554,7 @@ export function MarkerEditPlugin({ wholeParaDeleteExpected: new Set(), collapsedDeleteCaretParas: new Set(), pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), logger, structureProtectionMode, @@ -1116,6 +1117,12 @@ export function MarkerEditPlugin({ context.wholeParaDeleteExpected?.clear(); context.collapsedDeleteCaretParas?.clear(); context.pasteRebuildArmed.current = false; + // Paste provenance never outlives the pend it decorates: a key that stopped pending — + // settled, detached, or shielded onto a different key — must not arm the own-marker-prefix + // dedup if the same node is ever pended again by ordinary typing. + context.pastePendedKeys.forEach((key) => { + if (!context.pendingKeys.has(key)) context.pastePendedKeys.delete(key); + }); context.rebuildAttempted.clear(); // Typing path: ScriptureReferencePlugin's async scrRef echo re-enters // `$moveCursorToVerseStart` and yanks the caret to the para/verse start via @@ -1164,6 +1171,7 @@ export function MarkerEditPlugin({ // are cleared first — they describe the pre-restore document, and a leftover key // pointing at a now-canonical node would drive a pointless refused rebuild later. context.pendingKeys.clear(); + context.pastePendedKeys.clear(); editorState.read(() => $rependPendShapedNodes(context)); // The restored caret is app-placed (history put it there, not a fresh user gesture), // and a historic restore is NOT a departure: resolving now — or on any follow-on diff --git a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx index 16cbe187e..1b774b504 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx @@ -128,20 +128,6 @@ function chapterHeaderSkeletonUsj(usj: Usj): Usj { * sibling `"unclosed note (closed=false)"` fixture, whose unclosed span IS the last thing in its * paragraph, has no such trailing content to lose and round-trips clean. * - * - **"milestones (ts)"** — a COMPOUND of two deliberate behaviors, neither of them a byte-level - * copy or normalization defect: the copied text is byte-correct - * (`\p \ts-s\*\v 1 Translator section text.\ts-e\*`), and pasting it into a host with NO marker - * prefix of its own round-trips clean. A milestone whose rebuild would EJECT content deliberately - * does NOT settle inside the commit that produced it (`markerEditTier2Trigger.utils.ts`: - * rearranging the line under a caret the user is still on is worse than waiting) — it pends until - * caret departure. But the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, - * `tier2Rebuild.utils.ts`) is armed only for the PASTE's own update, so the later - * departure-triggered settle no longer knows the fragment came from a paste and takes the typed - * path instead: the host's own `\p ` glyph and the pasted line's `\p ` literal both survive as - * two paragraphs, the first of them empty. Closing it means giving the dedup's arm a lifetime - * that spans the deferred settle — a paste-provenance change, not a tokenizer one, and out of - * scope here. - * * - **"paragraph-leading space (display rule)"** — ACCEPTED normalization, matching Paratext 9, * not a bug: isolated with a minimal non-corpus repro, pasting the literal text `"\p X"` * (marker, its own required separator, and a SECOND, real content-leading space) into a fresh @@ -171,11 +157,6 @@ const KNOWN_LOSSY: { name: string; reason: string }[] = [ reason: "consumeSeparator() eats the whole whitespace run after a marker, matching P9's NormalizeUsfm parity — accepted, not a bug", }, - { - name: "milestones (ts)", - reason: - "a milestone that ejects content pends until caret departure, and the paste-scoped own-marker-prefix dedup is armed only for the paste's own commit — so the deferred settle leaves the host's now-redundant \\p glyph behind as an empty paragraph", - }, ]; describe("corpus copy/paste round trip (Standard view)", () => { diff --git a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx index c23dab953..5cf6e60af 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.test.tsx @@ -929,6 +929,7 @@ describe("$resolvePendingMarkers attribute-run re-pend guard", () => { pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -1005,6 +1006,7 @@ describe("$settlePendedDisplayOwner AttributeRunNode husk arm (dual-read)", () = pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -1135,6 +1137,7 @@ describe("$resolvePendingMarkers folds a husk-only settle's mutation", () => { pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -1199,6 +1202,7 @@ describe("$settlePendedDisplayOwner verse migration + fallthrough interaction", pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -1391,6 +1395,7 @@ describe("$resolvePendingMarkers routes a pended run PIECE through its owner's g pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; diff --git a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts index 6d7e9d447..36ec026c6 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerEditTier1.utils.ts @@ -119,6 +119,12 @@ export interface MarkerEditContext extends Tier2Context { * `Tier2Context` for what it does. */ pasteRebuildArmed: { current: boolean }; + /** + * Narrows `Tier2Context.pastePendedKeys` from optional to REQUIRED, for the same reason as + * {@link pasteRebuildArmed}: the engine always constructs and maintains the set. See its doc + * comment on `Tier2Context` for what carries the provenance and why it cannot leak. + */ + pastePendedKeys: Set; /** * Mirrors the host `Editor`'s `structureProtectionMode` option. Read by * `$handlePasteForStandardView` (whitespaceDisplay.plugin.utils.ts), which must decline a @@ -1204,6 +1210,17 @@ function $exceptKeysAround(exceptKey: NodeKey | undefined): Set { return keys; } +/** + * Takes `key`'s paste provenance ({@link MarkerEditContext.pastePendedKeys}) for the rebuild about + * to be routed for it, and clears it: one key, one use, so a later rebuild of the same paragraph — + * typed input, an idle re-settle — is an ordinary typed rebuild again. `undefined` (not `false`) + * for a key with no provenance, so the rebuild falls back to its own default ("is this the paste's + * own update") rather than being told it is not a paste. + */ +function consumePasteProvenance(key: NodeKey, context: MarkerEditContext): true | undefined { + return context.pastePendedKeys.delete(key) || undefined; +} + /** * Completion trigger. PT9 completes mid-edit markers via its 1s debounced * reformat; our deterministic equivalents are Enter, blur, and the caret @@ -1243,6 +1260,7 @@ export function $resolvePendingMarkers( const node: LexicalNode | null = $getNodeByKey(key); if (!node?.isAttached()) { context.pendingKeys.delete(key); + context.pastePendedKeys.delete(key); continue; } if ($isMarkerNode(node)) { @@ -1271,7 +1289,9 @@ export function $resolvePendingMarkers( context.logger?.debug( "[MarkerEdit] unknown-split paragraph rejoined its predecessor on marker degradation", ); - } else mutated = $requestTier2ForNode(node, context) || mutated; + } else + mutated = + $requestTier2ForNode(node, context, consumePasteProvenance(key, context)) || mutated; continue; } // A pended run PIECE settles at its OWNER. `$settlePendedDisplayOwner` recognizes only owners @@ -1334,7 +1354,8 @@ export function $resolvePendingMarkers( context.pendingKeys.add(targetKey); continue; } - mutated = $requestTier2ForNode(target, context) || mutated; + mutated = + $requestTier2ForNode(target, context, consumePasteProvenance(key, context)) || mutated; } return mutated; } diff --git a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx index 7822ccf46..2821eff3f 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.test.tsx @@ -824,6 +824,7 @@ describe("$textNodeTier2Transform on attribute-run text", () => { pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -893,6 +894,7 @@ describe("$textNodeTier2Transform on pipe-text in plain content", () => { pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -986,6 +988,7 @@ describe("$textNodeTier2Transform on `//` optbreak text in plain content", () => pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; @@ -1076,6 +1079,7 @@ describe("$textNodeTier2Transform on `//` inside a table", () => { pendingKeys: new Set(), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; diff --git a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.ts b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.ts index 9668562eb..4fa67199b 100644 --- a/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerEditTier2Trigger.utils.ts @@ -141,6 +141,20 @@ function $inLiteralOnlyBlock(node: LexicalNode): boolean { return false; } +/** + * Pend `node`'s own key for a rebuild this update declines to perform, carrying the paste's + * provenance along with it when the update IS an external paste's own + * ({@link MarkerEditContext.pastePendedKeys}). A pasted line's terminated marker literal would + * otherwise reach the caret-departure settle with nothing recording where its bytes came from, and + * the settle would rebuild it as though the user had TYPED them — splitting the host paragraph in + * two instead of letting the pasted marker replace the host's now-redundant glyph. + */ +function $pendDeferredRebuild(node: TextNode, context: MarkerEditContext): void { + const key = node.getKey(); + context.pendingKeys.add(key); + if (context.pasteRebuildArmed.current) context.pastePendedKeys.add(key); +} + /** * The engine's plain-`TextNode` transform — the TRIGGER that decides what a text edit means for * the settle. Exactly one of: clear the node's pend (bytes at rest/canonical), record a pend for @@ -293,7 +307,7 @@ export function $textNodeTier2Transform(node: TextNode, context: MarkerEditConte // rearranges the line under the caret while the user is still on it. Pending instead leaves // what they typed alone until they depart, and the departure settle performs the same rebuild. if (milestoneEjectionPending(text)) { - context.pendingKeys.add(node.getKey()); + $pendDeferredRebuild(node, context); return; } context.pendingKeys.delete(node.getKey()); @@ -309,7 +323,7 @@ export function $textNodeTier2Transform(node: TextNode, context: MarkerEditConte // materializes no nodes — see `$rebuildParas`), so pending never destabilizes the // damping; for the identical-second-paragraph case it performs the rebuild the guard // would otherwise have swallowed. - context.pendingKeys.add(node.getKey()); + $pendDeferredRebuild(node, context); return; } context.rebuildAttempted.add(text); diff --git a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx index 525fc5dcb..9c8d3dfd0 100644 --- a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx @@ -450,6 +450,93 @@ describe("own-marker-prefix dedup: unknown/custom.sty markers", () => { }); }); +/** A `\p` host holding "A", plus a second paragraph to depart into. Returns the host's content + * text node (the caret target: a paste at "A"'s offset 0 is a paste at the paragraph's content + * start) and the departure paragraph's text node. */ +async function deferredSettleHost(): Promise<{ + editor: LexicalEditor; + content: TextNode; + departure: TextNode; +}> { + initializeDeserialize(undefined); + let content!: TextNode; + let departure!: TextNode; + const { editor } = await historyTestEnvironment(() => { + const host = $createParaNode("p"); + const separator = $createTextNode(NBSP); + $setState(separator, textTypeState, "marker-trailing-space"); + content = $createTextNode("A"); + const departurePara = $createParaNode("p"); + departure = $createTextNode("depart here"); + $getRoot().append( + host.append($createMarkerNode("p"), separator, content), + departurePara.append($createMarkerNode("p"), departure), + ); + }); + return { editor, content, departure }; +} + +/** Moves the caret out of the host paragraph and flushes the settle it triggers. */ +async function departAndSettle(editor: LexicalEditor, departure: TextNode): Promise { + await act(async () => editor.update(() => departure.select(0, 0))); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe("own-marker-prefix dedup survives a DEFERRED settle", () => { + // Bytes whose rebuild would EJECT content out of a milestone deliberately do NOT settle in the + // update that produced them (`markerEditTier2Trigger.utils.ts`: ejection rearranges the line + // under a caret the user is still on) — they pend until caret departure. A PASTE of such bytes + // therefore reaches its rebuild in a later update than the one that inserted it, which is the + // one shape where "is this a paste?" cannot be answered by a per-commit flag. + const EJECTING_LINE = '\\p \\qt1-s\\*|who=""\\*'; + /** What loading `EJECTING_LINE` produces: the ejected fixed point — the milestone closed, the + * attribute list it could not hold left outside it, and the author's own `\*` stranded. */ + const EJECTED_PARA = { + type: "para", + marker: "p", + content: [{ type: "ms", marker: "qt1-s" }, '|who=""', { type: "unmatched", marker: "*" }, "A"], + }; + const DEPARTURE_PARA = { type: "para", marker: "p", content: ["depart here"] }; + + it("paste at an existing paragraph's content start: the pasted line owns the paragraph even though its rebuild waits for caret departure", async () => { + const { editor, content, departure } = await deferredSettleHost(); + + await pasteAndSettle(editor, () => content.select(0, 0), EJECTING_LINE); + // The paste's own update inserted the bytes and stopped: the paragraph still holds them as + // literal text. Asserted so this pin cannot pass through the immediate-rebuild path it exists + // to look past. + expect(usjOf(editor).content).toEqual([ + { type: "para", marker: "p", content: [`${EJECTING_LINE}A`] }, + DEPARTURE_PARA, + ]); + + await departAndSettle(editor, departure); + + expect(usjOf(editor).content).toEqual([EJECTED_PARA, DEPARTURE_PARA]); + }); + + it("the same bytes arriving with no paste behind them still split with an empty predecessor — what carries across the settle is the paste's provenance, not the shape of the bytes", async () => { + const { editor, content, departure } = await deferredSettleHost(); + + await act(async () => + editor.update(() => { + const selection = content.select(0, 0); + selection.insertText(EJECTING_LINE); + }), + ); + await departAndSettle(editor, departure); + + expect(usjOf(editor).content).toEqual([ + { type: "para", marker: "p" }, + EJECTED_PARA, + DEPARTURE_PARA, + ]); + }); +}); + describe("paste-as-plain-text equivalence (S4): no literal mode, plain always wins", () => { // S4 (docs/superpowers/specs/2026-08-06-clipboard-semantics.md): Ctrl+Shift+V / "paste as plain // text" narrows the clipboard payload down to `text/plain` only, but `$handlePasteForStandardView` diff --git a/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx b/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx index 86c7502bb..6006832ba 100644 --- a/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx +++ b/packages/platform/src/editor/markerEdit/milestoneAttributeSettle.test.tsx @@ -51,6 +51,7 @@ import { $isTextNode, $setState, LexicalNode, + NodeKey, TextNode, } from "lexical"; import { @@ -357,6 +358,7 @@ describe("collab-materialized milestone settles into a re-tokenizable run", () = pendingKeys: new Set(milestoneFirst ? [msKey, runKey] : [runKey, msKey]), splitExpected: { current: false }, pasteRebuildArmed: { current: false }, + pastePendedKeys: new Set(), rebuildAttempted: new Set(), structureProtectionMode: "off", }; diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index b9c27f590..c44f48892 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -36,6 +36,7 @@ import { $parseSerializedNode, ElementNode, LexicalNode, + NodeKey, SerializedLexicalNode, TextNode, } from "lexical"; @@ -92,7 +93,8 @@ export interface Tier2Context { /** * Armed by `$handlePasteForStandardView` (whitespaceDisplay.plugin.utils.ts), alongside * `MarkerEditContext.splitExpected`, for the duration of one external paste's update — reset by - * the same per-commit update listener that resets `splitExpected`. `$rebuildParas` reads it to + * the same per-commit update listener that resets `splitExpected`. A rebuild that update DEFERS + * outlives the flag; {@link pastePendedKeys} is what carries the same provenance to it. `$rebuildParas` reads it to * decide whether `$buildParaFragment`'s own-marker-prefix dedup (`$withoutRedundantOwnPrefix`) * may run: that dedup is a PASTE-shape recognition ("a whole-paragraph copy's own glyph rides * along with the pasted text"), not a general typed-retag rule — applying it unconditionally @@ -104,6 +106,27 @@ export interface Tier2Context { * pre-existing, dedup-free behavior. */ pasteRebuildArmed?: { current: boolean }; + /** + * The pended keys whose deferred rebuild still belongs to a paste. A paste's own update does not + * always get to perform the rebuild its bytes ask for: a paragraph whose re-tokenization would + * EJECT content out of a milestone deliberately waits for caret departure + * (`markerEditTier2Trigger.utils.ts`), and by then {@link pasteRebuildArmed} — a per-commit flag + * — has long since reset. Recording the pended key here carries the paste's provenance to + * whichever settle finally runs that rebuild, so a pasted line's own marker literal still wins + * over the host's redundant glyph. + * + * Scoped by KEY, not by paragraph or by time, which is what keeps it from leaking into typed + * input: a key is only ever added while `pasteRebuildArmed` says the current update IS a paste's + * own; it is consumed (deleted) by the first settle that routes it to a rebuild, so no later + * rebuild of the same paragraph sees it; and the plugin's per-commit update listener prunes + * every key that is no longer pending, so provenance never outlives the pend it decorates. + * Typing beside the pasted bytes pends its OWN key, which carries no provenance and keeps the + * engine's existing split-with-empty behavior. + * + * Optional for the same reason `pasteRebuildArmed` is: a bare `Tier2Context` built directly by a + * test harness has no pends of its own to decorate. + */ + pastePendedKeys?: Set; } /** @@ -1102,9 +1125,9 @@ function $withoutRedundantOwnPrefix( * call site, every TYPED-input rebuild, and the read-only virtual settle): the own-marker-prefix * dedup recognizes a PASTE shape (a whole-paragraph copy's own glyph riding along with the pasted * text) and must not also apply to a user typing the same byte sequence — see that function's doc - * comment for why. `$rebuildParas` is the only caller that ever passes `true`, and only when - * `Tier2Context.pasteRebuildArmed` says the triggering rebuild is inside an external paste's own - * update. + * comment for why. `$rebuildParas` is the only caller that ever passes `true`, and only for a + * rebuild a paste asked for: one inside the paste's own update (`Tier2Context.pasteRebuildArmed`), + * or one the paste's update deferred to a later settle (`Tier2Context.pastePendedKeys`). */ export function $buildParaFragment( para: ParaNode, @@ -1574,13 +1597,21 @@ function $restoreSelectionInNoteContent( * the resulting USJ byte-for-byte (pinned by settledGetUsj.test.tsx and * settleDifferential.test.tsx). * + * `isPasteRebuild` says whether a PASTE asked for this rebuild — see `$buildParaFragment`'s own + * parameter of the same name. It defaults to "is this the paste's own update" + * (`Tier2Context.pasteRebuildArmed`); a settle performing a rebuild the paste deferred passes the + * provenance it consumed from `Tier2Context.pastePendedKeys` instead. + * * Mutating: call inside `editor.update()` (dispatched from the Tier-2 trigger transform, the * caret-departure and commit paths in MarkerEditPlugin.tsx, and `$requestTier2ForNode`). */ -export function $rebuildParas(paras: ParaNode[], context: Tier2Context): boolean { +export function $rebuildParas( + paras: ParaNode[], + context: Tier2Context, + isPasteRebuild = context.pasteRebuildArmed?.current ?? false, +): boolean { if (paras.length === 0) return false; - const { viewOptions, getMarker: getMarkerFn, logger, pasteRebuildArmed } = context; - const isPasteRebuild = pasteRebuildArmed?.current ?? false; + const { viewOptions, getMarker: getMarkerFn, logger } = context; const combined: FragmentAccumulator = { text: "", spans: [], sentinels: [] }; for (const para of paras) { @@ -2282,13 +2313,18 @@ export function $settleScopeForNode( /** Route a Tier-1-unexpressible edit to Tier 2 via its scope ({@link $settleScopeForNode}). * Returns whether the routed rebuild actually SPLICED — a guard-rail or fixed-point refusal mutates * nothing, and the deferred-resolution history bookkeeping ($resolvePendingMarkers callers) needs to - * tell the two apart. */ -export function $requestTier2ForNode(node: LexicalNode, context: Tier2Context): boolean { + * tell the two apart. `isPasteRebuild` is forwarded to {@link $rebuildParas} (omit it to take that + * function's own default). */ +export function $requestTier2ForNode( + node: LexicalNode, + context: Tier2Context, + isPasteRebuild?: boolean, +): boolean { const scope = $settleScopeForNode(node); if (!scope) return false; if ($isNoteNode(scope)) return $rebuildNoteContent(scope, context); if ($isChapterNode(scope)) return $rebuildChapter(scope, context); - return $rebuildParas([scope], context); + return $rebuildParas([scope], context, isPasteRebuild); } /** USJ marker-object keys that are never attribute-list display bytes. `closed` is a USJ key but From 1de2516b22f73ff3f3268d0f188d1280ea23973c Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 14:13:28 -0500 Subject: [PATCH 11/44] fix: make the copy carriers agree at a construct's own start boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review follow-ups on the opaque-construct clipboard cluster. `UnknownNode.excludeFromCopy`'s rationale still read as an optbreak husk rule after the guard was generalized to child count. It now names what the rule actually excludes, including the case it was under-describing: a hidden-marker view builds no display children at all, so ANY content-less construct there — a caption-less figure, an empty ref — is childless and dropped. Recorded as a deferred item rather than changed, since a marker-free view copying marker-only constructs is a view-semantics decision. Pinning the `isSelected` boundary on a second kind turned up a real disagreement rather than confirming the shared rule. A construct whose display bytes lead starts with a decorator, so an element-type boundary point on the wrapper stays one and reaches no child; a construct with no display bytes (a `ref`) starts with a real TextNode, and Lexical normalizes that same point into a text point at the child's offset 0 — the child is then in `getNodes()` while contributing zero characters, so the wrapper and its whole content went into the lexical flavor while `text/plain` had none of it. `isSelected` now asks whether the selection covers a child's CONTENT, which is what the copy walker computes. Pinned for figure and ref in both directions. The tokenizer's character-token cell arm gains sheet-backed unit coverage for the align-infix names and both of its guards; a ranged cell is not a sheet entry at all and still arrives as a paragraph token, so that is pinned too. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 8 +- .../converters/usfm/usfmFragmentToUsj.test.ts | 94 +++++++++++++++++++ libs/shared/src/nodes/features/UnknownNode.ts | 87 ++++++++++++----- .../markerEdit/figurePasteFidelity.test.tsx | 4 +- .../unknownClipboardFidelity.test.tsx | 84 ++++++++++++++++- 5 files changed, 249 insertions(+), 28 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 64e15961e..ec0b75ce8 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -26,7 +26,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). -- **An opaque construct copies out on ONE line:** the walk's ordinary rule puts a `\n` at every non-inline element boundary it crosses, which would spread a single construct over several lines — a sidebar's nested `\p`, and every row and cell of a table, are real block-level nodes. Pasting that text replays each `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct in one pass. `$startsBlockLine` (`whitespaceDisplay.plugin.utils.ts`) therefore suppresses the line break for a block boundary with an opaque ancestor ABOVE it — the construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document; it is simply not the line-per-marker layout a USFM writer emits, which is the deliberate trade for a paste that survives. Pinned by the corpus sweep's now-unskipped `"sidebar (esb)"` and `"table with header and cells"` fixtures and by `unknownClipboardFidelity.test.tsx`. Tables needed one more thing on the paste side: `usfm.sty` classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the fragment tokenizer as a char token and its table-cell assembly — which only ran for paragraph tokens — never saw it, turning every cell into a loose `char` span inside an empty row. ParatextData derives a cell from the marker NAME, so the assembly now recognizes a cell marker on either token kind, guarded on an open `\tr` row. +- **An opaque construct copies out on ONE line:** the walk's ordinary rule puts a `\n` at every non-inline element boundary it crosses, which would spread a single construct over several lines — a sidebar's nested `\p`, and every row and cell of a table, are real block-level nodes. Pasting that text replays each `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct in one pass. `$startsBlockLine` (`whitespaceDisplay.plugin.utils.ts`) therefore suppresses the line break for a block boundary with an opaque ancestor ABOVE it — the construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document; it is simply not the line-per-marker layout a USFM writer emits, which is the deliberate trade for a paste that survives. Pinned by the corpus sweep's now-unskipped `"sidebar (esb)"` and `"table with header and cells"` fixtures and by `unknownClipboardFidelity.test.tsx`. Tables needed one more thing on the paste side: `usfm.sty` classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the fragment tokenizer as a char token and its table-cell assembly — which only ran for paragraph tokens — never saw it, turning every cell into a loose `char` span inside an empty row. ParatextData derives a cell from the marker NAME, so the assembly now recognizes a cell marker on either token kind, guarded on an open `\tr` row. Pinned sheet-backed (`createMarkerLookup(defaultStyleInfo)`, where the cell markers really are Character) in `usfmFragmentToUsj.test.ts`'s `"cell markers under a real stylesheet"` describe, including the align-infix names (`\thc3` → center, `\thr5` → end) and both guards — a cell-named span with no open row stays an ordinary char span, and a nested `\+tc1` is never a cell. ### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) @@ -34,7 +34,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked a CHILD-BEARING optbreak "selected" under Lexical's default `isSelected`, producing a CHILDLESS `{type:"optbreak"}` placeholder in the copy even though the live node has a child — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx`, plus direct unit pins on the predicate in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override asks whether the selection covers any CONTENT of a child, not merely whether a child is in `getNodes()`, because that boundary reaches the children two different ways: a construct whose display bytes lead (a figure's `\fig ` glyph) starts with a DECORATOR, so the element point stays one and no child is in `getNodes()` at all, while a construct with no display bytes (a `ref`) starts with a real `TextNode` and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, exactly as the copy walker computes it. Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"carrier agreement at a construct's own start boundary"` describe pins the `isSelected` boundary for `figure` and `ref` in BOTH directions (touching the construct's start excludes it from both carriers; extending over its first child puts it back in both), covering a decorator-led and a text-led construct rather than leaving the shared predicate inferred from optbreak alone — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -224,6 +224,8 @@ The following items are recorded as deferred and not addressed in this plan: 11. **`\periph` has no tokenizer assembly, so a copied periph pastes back as an ordinary paragraph:** the sidebar/table paste-rebuild scoping gaps that used to sit in this list are closed — an opaque construct now copies out on ONE line (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) so a single-paragraph rebuild can reassemble it. `\periph` does not benefit, for two reasons of its own. `usfmFragmentToUsj.ts` has no periph assembly at all (it has `\esb`/`\esbe` for sidebars and `\tr`/`\th`/`\tc` for tables), so nothing turns `\periph …` back into a `periph` object; and `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` run on a marker with NO closing bytes, so nothing in the text marks where the attribute run ends and the content resumes. The paste therefore comes back as a paragraph whose marker is `periph`, carrying its own attribute bytes as literal content. Closing this means giving the tokenizer a periph assembly AND deciding how a closer-less division marker carries attributes at all — a construct-modelling question rather than a clipboard one, so it is recorded rather than guessed at. Pinned as a loss in `unknownClipboardFidelity.test.tsx`. See "Known Lossy Constructs" item 3 above. +12. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. + ## Test Mapping (S1–S7) @@ -239,7 +241,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"carrier agreement at a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`) and a text-led one (`ref`), in both directions | | **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts index fbb9e5b26..7d0b73dab 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts @@ -5,6 +5,7 @@ import { regularizeSpaces, } from "./usfmFragmentToUsj.js"; import { NBSP } from "../../nodes/usj/node-constants.js"; +import { defaultStyleInfo } from "../../utils/usfm/defaultStyleInfo.js"; import { createMarkerLookup, StyleInfo } from "../../utils/usfm/styleInfo.js"; describe("usfmFragmentToUsjContent — core", () => { @@ -1504,6 +1505,99 @@ describe("stylesheet-first classification", () => { }); }); +describe("cell markers under a real stylesheet, which classifies them as Character", () => { + // usfm.sty declares every `\th…`/`\tc…` as a Character style, so with a project sheet in play a + // cell marker reaches assembly as a `charOpen` token, not a paragraph one — the shape the app + // actually runs. ParatextData derives a cell from the marker NAME either way, and the alignment + // comes from the name's infix (`thc3` → center, `thr5` → end), so the two token kinds must land + // on the same cell. + const sheetLookup = createMarkerLookup(defaultStyleInfo); + + it("assembles align-infix cells that arrive as character tokens", () => { + expect( + usfmFragmentToUsjContent("\\tr \\thc3 middle\\thr5 right", { getMarker: sheetLookup }), + ).toEqual([ + { + type: "table", + content: [ + { + type: "table:row", + marker: "tr", + content: [ + { type: "table:cell", marker: "thc3", align: "center", content: ["middle"] }, + { type: "table:cell", marker: "thr5", align: "end", content: ["right"] }, + ], + }, + ], + }, + ]); + }); + + it("a RANGED cell marker is not in the sheet at all, so it still arrives as a paragraph token", () => { + // usfm.sty declares `\tcr1`, never `\tcr1-4` — a span is spelled by ParatextData's range rule, + // not by a stylesheet entry — so a ranged cell is an UNKNOWN marker even under a project sheet + // and reaches assembly through the paragraph arm. Colspan assembly is therefore untouched by + // the character-token arm above; both arms build the cell through the same `pushTableCell`. + expect(usfmFragmentToUsjContent("\\tr \\tcr1-4 wide", { getMarker: sheetLookup })).toEqual([ + { + type: "table", + content: [ + { + type: "table:row", + marker: "tr", + content: [ + { type: "table:cell", marker: "tcr1", align: "end", colspan: "4", content: ["wide"] }, + ], + }, + ], + }, + ]); + }); + + it("a cell-named character marker with NO open row stays an ordinary char span", () => { + // The cell arm is guarded on an open `\tr`; only that opens a row. Without the guard, a span + // that merely shares a cell marker's name would be assembled into a table that does not exist. + expect( + usfmFragmentToUsjContent("\\p before \\thc3 middle", { getMarker: sheetLookup }), + ).toEqual([ + { + type: "para", + marker: "p", + content: [ + "before ", + { type: "char", marker: "thc3", content: ["middle"], closed: "false" }, + ], + }, + ]); + }); + + it("a NESTED cell-named marker inside a cell is a char span, not a second cell", () => { + expect( + usfmFragmentToUsjContent("\\tr \\thc3 middle \\+tc1 inner\\+tc1*", { + getMarker: sheetLookup, + }), + ).toEqual([ + { + type: "table", + content: [ + { + type: "table:row", + marker: "tr", + content: [ + { + type: "table:cell", + marker: "thc3", + align: "center", + content: ["middle ", { type: "char", marker: "tc1", content: ["inner"] }], + }, + ], + }, + ], + }, + ]); + }); +}); + describe("PT9 unknown-marker handling", () => { it("unknown marker in body context becomes a paragraph (UsfmParser.DetermineUnknownTokenType)", () => { const content = usfmFragmentToUsjContent("\\p before \\zfoo after"); diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index 7aa8b320f..48ddb07ca 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -3,6 +3,8 @@ import { MarkerObject } from "@eten-tech-foundation/scripture-utilities"; import { $applyNodeReplacement, $getSelection, + $isRangeSelection, + $isTextNode, BaseSelection, DOMConversionMap, DOMConversionOutput, @@ -225,40 +227,81 @@ export class UnknownNode extends ElementNode { // construct's full USFM bytes while its USJ has lost the node and every attribute on it — a // convincing display over missing data, which a save then persists with no error. // - // The `getChildrenSize() > 0` guard keeps a genuinely childless husk excluded, matching - // `text/plain`'s "nothing here" for the same node (`markerEditTier1.utils.ts`'s own - // husk-removal check recognizes an emptied optbreak by that same zero-child shape). It does NOT, - // on its own, cover a DIFFERENT carrier-agreement gap: a selection whose ending boundary resolves - // to an ELEMENT-type point ON this node at offset 0 (touching the wrapper without covering any of - // its content) still marks a CHILD-BEARING node "selected" under Lexical's default `isSelected` - // (key-membership in `selection.getNodes()`, unaffected by this node's own child count), so - // `$appendNodesToJSON` would serialize a CHILDLESS placeholder for a node that has children, - // disagreeing with `text/plain` (`$selectionToUsfmText`, which walks that same `getNodes()` list - // and correctly emits nothing for this boundary). The `isSelected` override below closes that - // second gap at its actual source, since `excludeFromCopy` has no visibility into which of a - // node's children a given selection will include. + // The `getChildrenSize() > 0` guard excludes a node with no children of its OWN — of any kind, + // and for whatever reason it has none — because that is exactly when `text/plain` also emits + // nothing for it, and the two carriers must agree. Three shapes reach it, and only the first is + // a husk: an optbreak whose `//` display child was deleted (`markerEditTier1.utils.ts`'s + // husk-removal check recognizes it by that same zero-child shape); a content-less construct in + // an editable-marker view, where the marker/attribute display bytes `createUnknown` + // (`usj-editor.adaptor.ts`) prepends are themselves children, so only a kind with no display + // bytes at all can be childless; and, in a HIDDEN-marker view, ANY content-less construct — a + // caption-less `figure`, an empty `ref`, every optbreak — because `createUnknown` builds no + // display children there at all. The last is prose-copy semantics rather than a husk: a + // hidden-marker view copies what it shows, and it shows no bytes for a construct with no + // content. + // + // The guard does NOT, on its own, cover a DIFFERENT carrier-agreement gap: a selection whose + // ending boundary resolves to an ELEMENT-type point ON this node at offset 0 (touching the + // wrapper without covering any of its content) still marks a CHILD-BEARING node "selected" under + // Lexical's default `isSelected` (key-membership in `selection.getNodes()`, unaffected by this + // node's own child count), so `$appendNodesToJSON` would serialize a CHILDLESS placeholder for a + // node that has children, disagreeing with `text/plain` (`$selectionToUsfmText`, which walks that + // same `getNodes()` list and correctly emits nothing for this boundary). The `isSelected` + // override below closes that second gap at its actual source, since `excludeFromCopy` has no + // visibility into which of a node's children a given selection will include. override excludeFromCopy(destination: "clone" | "html"): boolean { return this.getChildrenSize() > 0 ? false : destination !== "clone"; } // An `UnknownNode` is only meaningfully "selected" (and so only copy-included, per the - // `excludeFromCopy` guard above) when at least one of its own children is — mirrors - // `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at the same selection - // boundary, the same way Lexical's own base `isSelected` (`LexicalNode.prototype.isSelected`) - // already special-cases an inline DECORATOR node sitting as a parent's last child at an - // exactly-there boundary point, for the identical reason. Without this, a selection ending - // exactly at this node's own start (an ElementNode touch-boundary that covers none of its - // content) still counts the WRAPPER as selected via the default `ElementNode.isSelected` - // (key-membership in `selection.getNodes()`), while no child is — producing a childless entry in - // the `application/x-lexical-editor` copy with nothing corresponding to it in `text/plain`. + // `excludeFromCopy` guard above) when the selection actually covers CONTENT of one of its own + // children — mirrors `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at + // the same selection boundary, the same way Lexical's own base `isSelected` + // (`LexicalNode.prototype.isSelected`) already special-cases an inline DECORATOR node sitting as + // a parent's last child at an exactly-there boundary point, for the identical reason. Without + // this, a selection ending exactly at this node's own start counts the WRAPPER as selected via + // the default `ElementNode.isSelected` (key-membership in `selection.getNodes()`) while none of + // its content is covered — producing an entry in the `application/x-lexical-editor` copy with + // nothing corresponding to it in `text/plain`. + // + // Membership alone is not the whole test, because a boundary reaches the children two different + // ways depending on what the first child IS. A construct whose display bytes lead (a `figure`'s + // `\fig ` glyph) starts with a DECORATOR, so an element-type boundary point on the wrapper stays + // one and no child is in `getNodes()` at all. A construct with no display bytes (a `ref`, whose + // container USFM never carried) starts with a real `TextNode`, and Lexical normalizes that same + // boundary point into a TEXT point at that child's offset 0 — so the child IS in `getNodes()` + // while contributing zero characters, exactly as the copy walker computes it. Hence the + // zero-width check: a boundary point that touches a child at its very start or very end covers + // none of it. override isSelected(selection?: BaseSelection | null): boolean { const targetSelection = selection ?? $getSelection(); if (!targetSelection) return false; const selectedNodes = targetSelection.getNodes(); - return this.getChildren().some((child) => selectedNodes.some((node) => node.is(child))); + return this.getChildren().some( + (child) => + selectedNodes.some((node) => node.is(child)) && + $selectionCoversContent(child, targetSelection), + ); } } +/** + * Whether `selection` covers at least one character of `child`, as opposed to merely touching it at + * a boundary. Only a range selection's own endpoint can be zero-width against a text node — a child + * reached any other way is genuinely inside the range — so every other shape answers `true`. + */ +function $selectionCoversContent(child: LexicalNode, selection: BaseSelection): boolean { + if (!$isRangeSelection(selection) || !$isTextNode(child)) return true; + const { anchor, focus } = selection; + const [start, end] = anchor.isBefore(focus) ? [anchor, focus] : [focus, anchor]; + if (end.type === "text" && end.key === child.getKey() && end.offset === 0) return false; + return !( + start.type === "text" && + start.key === child.getKey() && + start.offset === child.getTextContentSize() + ); +} + function $convertUnknownElement(element: HTMLElement): DOMConversionOutput { const tag = element.getAttribute("data-tag") ?? ""; const marker = element.getAttribute("data-marker") ?? ""; diff --git a/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx index 0eedfb691..735cfefff 100644 --- a/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx @@ -36,7 +36,9 @@ if (typeof globalStubs.DragEvent === "undefined") if (typeof globalStubs.ClipboardEvent === "undefined") globalStubs.ClipboardEvent = class ClipboardEvent extends Event {}; -/** The exact bytes from the live report. */ +/** One figure's full USFM: the caption, then its USFM 3 attribute run, then the closer. The + * caption sitting BEFORE the attributes is what makes the construct's own content the thing a + * rebuild has to keep inside the wrapper. */ const FIGURE_USFM = `\\fig At once they left their nets.|src="avnt016.jpg" size="span" ref="1.18"\\fig*`; const CAPTION = "At once they left their nets."; diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx index cef1176d5..2082f47b9 100644 --- a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -28,8 +28,23 @@ import { mountStandardViewEditor } from "../settledGetUsj.test-helpers"; import { corpusFixtures } from "../adaptors/corpus/corpus-data"; import { MarkerObject, Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; import { act } from "@testing-library/react"; -import { $getRoot, COPY_COMMAND, PASTE_COMMAND, RootNode } from "lexical"; -import { $isChapterNode, $isImmutableChapterNode } from "shared"; +import { + $createPoint, + $createRangeSelection, + $getRoot, + $setSelection, + COPY_COMMAND, + PASTE_COMMAND, + RootNode, +} from "lexical"; +import { + $isChapterNode, + $isImmutableChapterNode, + $isParaNode, + $isUnknownNode, + ParaNode, + UnknownNode, +} from "shared"; // jsdom implements neither `ClipboardEvent` nor `DragEvent`; Lexical's own rich-paste fallback — // the path a lexical-flavor payload takes once the Standard-view handler declines — duck-types @@ -157,6 +172,44 @@ function objectsOfType(usj: Usj, type: string): MarkerObject[] { return found; } +/** The document's first paragraph PAST the header — the one a user would be selecting in. */ +function $firstContentPara(): ParaNode { + const root = $getRoot(); + const para = root.getChildren().slice($contentStartIndex(root)).find($isParaNode); + if (!para) throw new Error("no content paragraph past the header"); + return para; +} + +/** The first `UnknownNode` of `tag` in that paragraph. */ +function $constructOfTag(tag: string): UnknownNode { + const construct = $firstContentPara() + .getChildren() + .filter($isUnknownNode) + .find((node) => node.getTag() === tag); + if (!construct) throw new Error(`no ${tag} UnknownNode in the first content paragraph`); + return construct; +} + +/** Copies a selection running from the paragraph's content start to an ELEMENT-type point ON the + * `tag` construct at `offset`, and hands back what both carriers wrote. `offset` 0 TOUCHES the + * wrapper without covering any of its children; `offset` 1 covers the first one. */ +async function copyToConstructBoundary(usj: Usj, tag: string, offset: number): Promise { + const { lexical } = await mountStandardViewEditor(usj); + await act(async () => + lexical.update(() => { + const para = $firstContentPara(); + const construct = $constructOfTag(tag); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(para.getKey(), 2, "element"); // past the para's own glyph + selection.focus = $createPoint(construct.getKey(), offset, "element"); + $setSelection(selection); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => lexical.dispatchCommand(COPY_COMMAND, event)); + return { [PLAIN]: getData(PLAIN), [HTML]: getData(HTML), [LEXICAL]: getData(LEXICAL) }; +} + const SHAPES = ["plain", "plain+html", "full"] as const; describe("figure (UnknownNode) copy→paste across all three payload shapes", () => { @@ -225,6 +278,33 @@ describe("ref (UnknownNode) copy→paste across all three payload shapes", () => }); }); +describe("carrier agreement at a construct's own start boundary (the shared isSelected rule)", () => { + // `UnknownNode.isSelected` asks whether any of the node's OWN CHILDREN are selected, rather than + // taking Lexical's default key-membership answer — so the lexical-JSON copy agrees with + // `text/plain`, whose walker reads the same `getNodes()` list. The rule is written once for every + // kind and keys on nothing kind-specific, so it is pinned on more than the optbreak that + // surfaced it (`optbreakClipboardFidelity.test.tsx`); these two constructs have real attributes + // and real content, which an optbreak does not. + const BOUNDARY_KINDS = [ + { tag: "figure", fixture: "figure (USFM 3 attributes)", bytes: "\\fig" }, + { tag: "ref", fixture: "cross-reference ref target", bytes: "Genesis 1:1" }, + ]; + + BOUNDARY_KINDS.forEach(({ tag, fixture, bytes }) => { + it(`a selection ending exactly at the ${tag}'s own start excludes it from BOTH carriers`, async () => { + const payload = await copyToConstructBoundary(corpusUsj(fixture), tag, 0); + expect(payload[PLAIN]).not.toContain(bytes); + expect(payload[LEXICAL]).not.toContain(`"${tag}"`); + }); + + it(`extending that same selection over the ${tag}'s first child puts it back in BOTH carriers`, async () => { + const payload = await copyToConstructBoundary(corpusUsj(fixture), tag, 1); + expect(payload[PLAIN]).toContain(bytes); + expect(payload[LEXICAL]).toContain(`"${tag}"`); + }); + }); +}); + describe("table (ImmutableTable* nodes, not UnknownNode) copy→paste across all three payload shapes", () => { const usj = corpusUsj("table with header and cells"); SHAPES.forEach((shape) => { From d393a7cd0d4aa05fea1f86c513e6707975af1a19 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 14:42:04 -0500 Subject: [PATCH 12/44] fix: keep a touched construct whole in the copy instead of hoisting its text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The content-coverage `isSelected` predicate closed the `ref` boundary's carrier disagreement by trading it for something worse. Excluding the wrapper does not drop it quietly: `$appendNodesToJSON` hoists its children in its place, and `createUnknown` stamps `mode:"token"` on every text child, which `$sliceSelectedTextNodeContent` refuses to slice — so the zero-width child kept its full text and the copy carried `Genesis 1:1` with the `ref` node and its `loc` attribute silently gone. That is the convincing-lie shape the suite exists to prevent, and the tag-only assertion could not see it. Revert to child membership, which keeps the construct whole and makes the lexical flavor a superset of `text/plain` at that one boundary. Making it appear in neither carrier is not reachable from the node API: the children are plain TextNodes, and `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot exclude itself AND its children. The residual is recorded in the deferred list with both measured payloads. Every boundary assertion now reads the construct's own content bytes, and the pins state the behavior per kind — a decorator-led construct is excluded from both carriers, a text-led one is carried whole — so the lossy variant turns them red instead of passing. Also scoped the provenance doc to what was measured: typing into a DIFFERENT node pends its own key and takes the typed path, but typing into the still- pending node itself adds no key, so the paste path runs and the dedup applies to the line those bytes are now part of. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 8 +- libs/shared/src/nodes/features/UnknownNode.ts | 77 ++++++++----------- .../editor/markerEdit/tier2Rebuild.utils.ts | 25 +++--- .../unknownClipboardFidelity.test.tsx | 71 ++++++++++++----- 4 files changed, 107 insertions(+), 74 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index ec0b75ce8..0871f427a 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -34,7 +34,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override asks whether the selection covers any CONTENT of a child, not merely whether a child is in `getNodes()`, because that boundary reaches the children two different ways: a construct whose display bytes lead (a figure's `\fig ` glyph) starts with a DECORATOR, so the element point stays one and no child is in `getNodes()` at all, while a construct with no display bytes (a `ref`) starts with a real `TextNode` and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, exactly as the copy walker computes it. Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"carrier agreement at a construct's own start boundary"` describe pins the `isSelected` boundary for `figure` and `ref` in BOTH directions (touching the construct's start excludes it from both carriers; extending over its first child puts it back in both), covering a decorator-led and a text-led construct rather than leaving the shared predicate inferred from optbreak alone — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 13 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -55,7 +55,7 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra - **A multi-line payload is replayed line by line through `INSERT_PARAGRAPH_COMMAND`**, not through a bare `selection.insertParagraph()` per newline. Going through the command is what makes a paste landing inside a character-style stack close and reopen that stack at every line break, exactly as Enter does — a raw split tears the span, dropping its closing marker and leaving every line after the first outside the reopened style. The selection is removed up front rather than relying on the first `insertText` to replace it, so a payload whose first line is empty (a leading newline) cannot split a still-selected range. `MarkerEditContext.splitExpected` is armed before the first line goes in — the command's own handler arms it for each dispatch, but the first line is inserted before any dispatch runs — and `Tier2Context.pasteRebuildArmed` is armed unconditionally, single-line pastes included, since a single line can trigger the own-marker-prefix dedup just as easily. Foreign HTML formatting is dropped by design — Standard view is markers-as-text, exactly like P9's Standard view where the reformat pipeline re-tokenizes everything. This generalizes the existing NBSP-gated handler and kills both live-observed corruptions plus the latent doubled-glyph path (re-imported Standard-view HTML producing a `CharNode` via `data-marker` importDOM *and* literal `\nd` text, since MarkerNode has no importDOM). -- **A paste's provenance outlives its own commit, keyed to the pended work rather than to the clock:** the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, `tier2Rebuild.utils.ts`) drops a host paragraph's now-redundant `\p ` glyph when the pasted line carries its own, and it is deliberately a PASTE-shape recognition — typing the same bytes keeps the engine's P9-parity split-with-empty behavior. `Tier2Context.pasteRebuildArmed` answers "is this the paste's own update", which is not enough on its own: a paragraph whose re-tokenization would EJECT content out of a milestone deliberately does not settle in the commit that produced it (`markerEditTier2Trigger.utils.ts` — ejection rearranges the line under a caret the user is still on), so its rebuild reaches a later, caret-departure settle with the flag long since reset, and the host was left with a stray empty paragraph ahead of the correct one. `Tier2Context.pastePendedKeys` closes that: a pend recorded while the paste's own update is armed also records its KEY, and the settle that finally routes that key to a rebuild consumes the provenance and passes it down. Scoping it by key is what keeps it out of typed input — a key is only ever added inside a paste's own update, it is consumed by the first rebuild it reaches, the plugin's per-commit update listener prunes any key that has stopped pending, and text typed beside the pasted bytes pends its OWN key, which carries no provenance. Answering "what if the user types between the paste and the departure": the typed bytes get an ordinary typed rebuild, and the paste's deferred rebuild still gets the paste's. +- **A paste's provenance outlives its own commit, keyed to the pended work rather than to the clock:** the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, `tier2Rebuild.utils.ts`) drops a host paragraph's now-redundant `\p ` glyph when the pasted line carries its own, and it is deliberately a PASTE-shape recognition — typing the same bytes keeps the engine's P9-parity split-with-empty behavior. `Tier2Context.pasteRebuildArmed` answers "is this the paste's own update", which is not enough on its own: a paragraph whose re-tokenization would EJECT content out of a milestone deliberately does not settle in the commit that produced it (`markerEditTier2Trigger.utils.ts` — ejection rearranges the line under a caret the user is still on), so its rebuild reaches a later, caret-departure settle with the flag long since reset, and the host was left with a stray empty paragraph ahead of the correct one. `Tier2Context.pastePendedKeys` closes that: a pend recorded while the paste's own update is armed also records its KEY, and the settle that finally routes that key to a rebuild consumes the provenance and passes it down. Scoping it by key is what keeps it out of typed input: a key is only ever added inside a paste's own update, it is consumed by the first rebuild it reaches, and the plugin's per-commit update listener prunes any key that has stopped pending. Answering "what if the user types between the paste and the departure" precisely, since the two cases differ (both measured): typing into a DIFFERENT node pends that node's own key, which carries no provenance, so its rebuild takes the ordinary typed split-with-empty path; typing into the still-pending node ITSELF adds no key at all — the paste's key is already pended and is still what the departure settles — so the paste path runs and the dedup applies to the line those typed bytes are now part of. The second is the intended reading of provenance (the line is still the pasted line), not an escape from it. - **Positional NBSP normalization (replaces the blanket NBSP→`~`), three-part rule:** (1) a leading NBSP — string-start or right after a newline — is a structural separator with nothing in front of it to match against and becomes a space; (2) an NBSP immediately FOLLOWING a marker token (`\marker`, nested `\+marker`, either's closer, or a milestone's anonymous `\*` self-closer) is the required opener/closer separator and becomes a space; (3) an NBSP immediately PRECEDING a marker token is a structural spacer with no source counterpart and is DROPPED entirely (neither spaced nor kept as data) — `createNote` (`usj-editor.adaptor.ts`) appends a spacer after EVERY note child, not just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller; a browser-hop/html-derived copy of a collapsed note therefore carries structural NBSPs on BOTH sides of its interior markers, not only after the opener. Every remaining NBSP is user data → `~`. This is P9's `PostprocessUsfm` model. Live repro that mandates it (2026-08-07): pasting P10's own copied footnote back produced `\f~ \fr~1:1 ~ \ft~Caller test.~ \f*` (every display-NBSP became `~`, breaking marker recognition); a browser-hop paste of `\nd …\nd*` produced `\nd~light … \nd*` with an unmatched pair. **Rule precedence when both position rules match the same NBSP:** a closed-children note shape (`\xo*` immediately followed by NBSP immediately followed by `\xt`) sits after a closing marker AND before the next opening marker at once; the after-marker rule wins and the NBSP becomes a space rather than being dropped, because `$normalizePastedNbsp`'s `.replace` chain applies `AFTER_MARKER_NBSP` before `BEFORE_MARKER_NBSP`, consuming the NBSP into a space before the before-marker pass ever sees it. The tie-break is correct, not incidental: an after-closer NBSP in ordinary body text (`\nd*and`) is a real content space that dropping would silently lose, and the case is reachable only via a foreign NBSP-preserving carrier — P10's own `text/plain` copy (S1) never emits an NBSP in this position to begin with. @@ -226,6 +226,8 @@ The following items are recorded as deferred and not addressed in this plan: 12. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. +13. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. + ## Test Mapping (S1–S7) @@ -241,7 +243,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"carrier agreement at a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`) and a text-led one (`ref`), in both directions | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 13), asserting content BYTES rather than tags | | **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index 48ddb07ca..d8b282673 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -3,8 +3,6 @@ import { MarkerObject } from "@eten-tech-foundation/scripture-utilities"; import { $applyNodeReplacement, $getSelection, - $isRangeSelection, - $isTextNode, BaseSelection, DOMConversionMap, DOMConversionOutput, @@ -248,60 +246,53 @@ export class UnknownNode extends ElementNode { // node that has children, disagreeing with `text/plain` (`$selectionToUsfmText`, which walks that // same `getNodes()` list and correctly emits nothing for this boundary). The `isSelected` // override below closes that second gap at its actual source, since `excludeFromCopy` has no - // visibility into which of a node's children a given selection will include. + // visibility into which of a node's children a given selection will include — for every + // construct whose display bytes lead, which is all of them but `ref`; see that override for the + // one shape it deliberately does not close, and why. override excludeFromCopy(destination: "clone" | "html"): boolean { return this.getChildrenSize() > 0 ? false : destination !== "clone"; } // An `UnknownNode` is only meaningfully "selected" (and so only copy-included, per the - // `excludeFromCopy` guard above) when the selection actually covers CONTENT of one of its own - // children — mirrors `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at - // the same selection boundary, the same way Lexical's own base `isSelected` - // (`LexicalNode.prototype.isSelected`) already special-cases an inline DECORATOR node sitting as - // a parent's last child at an exactly-there boundary point, for the identical reason. Without - // this, a selection ending exactly at this node's own start counts the WRAPPER as selected via - // the default `ElementNode.isSelected` (key-membership in `selection.getNodes()`) while none of - // its content is covered — producing an entry in the `application/x-lexical-editor` copy with - // nothing corresponding to it in `text/plain`. + // `excludeFromCopy` guard above) when at least one of its own children is — mirrors + // `$selectionToUsfmText`'s copy walker so both clipboard carriers agree at the same selection + // boundary, the same way Lexical's own base `isSelected` (`LexicalNode.prototype.isSelected`) + // already special-cases an inline DECORATOR node sitting as a parent's last child at an + // exactly-there boundary point, for the identical reason. Without this, a selection ending + // exactly at this node's own start (an ElementNode touch-boundary that covers none of its + // content) still counts the WRAPPER as selected via the default `ElementNode.isSelected` + // (key-membership in `selection.getNodes()`), while no child is — producing a childless entry in + // the `application/x-lexical-editor` copy with nothing corresponding to it in `text/plain`. // - // Membership alone is not the whole test, because a boundary reaches the children two different - // ways depending on what the first child IS. A construct whose display bytes lead (a `figure`'s - // `\fig ` glyph) starts with a DECORATOR, so an element-type boundary point on the wrapper stays - // one and no child is in `getNodes()` at all. A construct with no display bytes (a `ref`, whose - // container USFM never carried) starts with a real `TextNode`, and Lexical normalizes that same - // boundary point into a TEXT point at that child's offset 0 — so the child IS in `getNodes()` - // while contributing zero characters, exactly as the copy walker computes it. Hence the - // zero-width check: a boundary point that touches a child at its very start or very end covers - // none of it. + // Child MEMBERSHIP is the test, deliberately, and not the narrower "does the selection cover a + // child's CONTENT". The two differ for exactly one shape, because that boundary reaches the + // children two ways depending on what the first child IS. A construct whose display bytes lead + // (a `figure`'s `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR: the element-type + // point stays one, no child is in `getNodes()`, and both carriers agree. A construct with no + // display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and + // Lexical normalizes that same point into a TEXT point at the child's offset 0 — the child is in + // `getNodes()` contributing zero characters, so `text/plain` emits nothing for it while this + // predicate still answers true and the construct rides along in the lexical flavor. + // + // Answering false there is WORSE, not better, and measurably so. Excluding the wrapper does not + // drop it quietly: `$appendNodesToJSON` HOISTS its children in its place, and `createUnknown` + // stamps `mode:"token"` on every text child, which `$sliceSelectedTextNodeContent` refuses to + // slice — so the zero-width child keeps its full text, the emptied-text reset never fires, and + // the copy ends up carrying the construct's CHARACTERS with the wrapper and its attributes + // silently gone. That is the convincing-lie hazard this whole pair exists to prevent. Membership + // keeps the construct whole, so the lexical flavor is a SUPERSET of `text/plain` at that one + // boundary rather than a structural loss. Recorded as a residual (the clipboard semantics doc's + // "Deferred / Out of Scope" list) rather than papered over; closing it needs a lever Lexical does + // not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a + // `children` array, so a node cannot say "drop me AND my children". override isSelected(selection?: BaseSelection | null): boolean { const targetSelection = selection ?? $getSelection(); if (!targetSelection) return false; const selectedNodes = targetSelection.getNodes(); - return this.getChildren().some( - (child) => - selectedNodes.some((node) => node.is(child)) && - $selectionCoversContent(child, targetSelection), - ); + return this.getChildren().some((child) => selectedNodes.some((node) => node.is(child))); } } -/** - * Whether `selection` covers at least one character of `child`, as opposed to merely touching it at - * a boundary. Only a range selection's own endpoint can be zero-width against a text node — a child - * reached any other way is genuinely inside the range — so every other shape answers `true`. - */ -function $selectionCoversContent(child: LexicalNode, selection: BaseSelection): boolean { - if (!$isRangeSelection(selection) || !$isTextNode(child)) return true; - const { anchor, focus } = selection; - const [start, end] = anchor.isBefore(focus) ? [anchor, focus] : [focus, anchor]; - if (end.type === "text" && end.key === child.getKey() && end.offset === 0) return false; - return !( - start.type === "text" && - start.key === child.getKey() && - start.offset === child.getTextContentSize() - ); -} - function $convertUnknownElement(element: HTMLElement): DOMConversionOutput { const tag = element.getAttribute("data-tag") ?? ""; const marker = element.getAttribute("data-marker") ?? ""; diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index c44f48892..7953b29dc 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -94,13 +94,14 @@ export interface Tier2Context { * Armed by `$handlePasteForStandardView` (whitespaceDisplay.plugin.utils.ts), alongside * `MarkerEditContext.splitExpected`, for the duration of one external paste's update — reset by * the same per-commit update listener that resets `splitExpected`. A rebuild that update DEFERS - * outlives the flag; {@link pastePendedKeys} is what carries the same provenance to it. `$rebuildParas` reads it to - * decide whether `$buildParaFragment`'s own-marker-prefix dedup (`$withoutRedundantOwnPrefix`) - * may run: that dedup is a PASTE-shape recognition ("a whole-paragraph copy's own glyph rides - * along with the pasted text"), not a general typed-retag rule — applying it unconditionally - * made typing a paragraph-kind marker literal at an existing paragraph's content start silently - * DELETE or no-op the paragraph's prior marker instead of the engine's existing (P9-parity) - * split-with-empty behavior, a product decision this task must not make for typed input. + * outlives the flag; {@link pastePendedKeys} is what carries the same provenance to it. + * `$rebuildParas` reads it to decide whether `$buildParaFragment`'s own-marker-prefix dedup + * (`$withoutRedundantOwnPrefix`) may run: that dedup is a PASTE-shape recognition ("a + * whole-paragraph copy's own glyph rides along with the pasted text"), not a general + * typed-retag rule — applying it unconditionally made typing a paragraph-kind marker literal at + * an existing paragraph's content start silently DELETE or no-op the paragraph's prior marker + * instead of the engine's existing (P9-parity) split-with-empty behavior, a product decision + * this module must not make for typed input. * Optional so bare `Tier2Context` objects built directly (tests exercising the tokenizer/rebuild * machinery without the full marker-edit engine) default to "not a paste rebuild" — the * pre-existing, dedup-free behavior. @@ -120,8 +121,14 @@ export interface Tier2Context { * own; it is consumed (deleted) by the first settle that routes it to a rebuild, so no later * rebuild of the same paragraph sees it; and the plugin's per-commit update listener prunes * every key that is no longer pending, so provenance never outlives the pend it decorates. - * Typing beside the pasted bytes pends its OWN key, which carries no provenance and keeps the - * engine's existing split-with-empty behavior. + * + * What that does NOT mean is "any typing between the paste and the departure is unaffected". + * Typing into a DIFFERENT node pends that node's own key, which carries no provenance, and its + * rebuild takes the engine's existing split-with-empty behavior. Typing into the still-pending + * node ITSELF adds no key at all — the paste's key is already pended and is still the one the + * departure settles — so the paste path runs and the dedup applies to the line the typed bytes + * are now part of. That is the intended reading of provenance (the line is still the pasted + * line), not an escape from it. * * Optional for the same reason `pasteRebuildArmed` is: a bare `Tier2Context` built directly by a * test harness has no pends of its own to decorate. diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx index 2082f47b9..245134bc5 100644 --- a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -278,29 +278,62 @@ describe("ref (UnknownNode) copy→paste across all three payload shapes", () => }); }); -describe("carrier agreement at a construct's own start boundary (the shared isSelected rule)", () => { - // `UnknownNode.isSelected` asks whether any of the node's OWN CHILDREN are selected, rather than - // taking Lexical's default key-membership answer — so the lexical-JSON copy agrees with - // `text/plain`, whose walker reads the same `getNodes()` list. The rule is written once for every - // kind and keys on nothing kind-specific, so it is pinned on more than the optbreak that - // surfaced it (`optbreakClipboardFidelity.test.tsx`); these two constructs have real attributes - // and real content, which an optbreak does not. - const BOUNDARY_KINDS = [ - { tag: "figure", fixture: "figure (USFM 3 attributes)", bytes: "\\fig" }, - { tag: "ref", fixture: "cross-reference ref target", bytes: "Genesis 1:1" }, - ]; +describe("a construct's own start boundary (the shared isSelected rule)", () => { + // `UnknownNode.isSelected` asks whether any of the node's OWN CHILDREN are in the selection, + // rather than taking Lexical's default key-membership answer for the WRAPPER — so the + // lexical-JSON copy agrees with `text/plain`, whose walker reads the same `getNodes()` list. The + // rule is written once for every kind and keys on nothing kind-specific, so it is pinned on more + // than the optbreak that surfaced it (`optbreakClipboardFidelity.test.tsx`): these constructs + // have real attributes and real content, which an optbreak does not. + // + // Every assertion below reads the construct's own CONTENT BYTES, never just its tag. A tag-only + // assertion cannot see the shape that matters here — a copy that dropped the wrapper while + // HOISTING its characters out has no tag in it and its content bytes are all still there. + const FIGURE = { tag: "figure", fixture: "figure (USFM 3 attributes)" }; + const REF = { tag: "ref", fixture: "cross-reference ref target" }; + const FIGURE_BYTES = "At once they left their nets."; + const REF_BYTES = "Genesis 1:1"; - BOUNDARY_KINDS.forEach(({ tag, fixture, bytes }) => { - it(`a selection ending exactly at the ${tag}'s own start excludes it from BOTH carriers`, async () => { - const payload = await copyToConstructBoundary(corpusUsj(fixture), tag, 0); - expect(payload[PLAIN]).not.toContain(bytes); - expect(payload[LEXICAL]).not.toContain(`"${tag}"`); - }); + it("a selection ending exactly at a DISPLAY-BYTE-LED construct's start excludes it from both carriers", async () => { + // A figure's first child is the `\fig ` display decorator, so the element-type focus point + // stays an element point and reaches none of the figure's children: neither carrier carries + // the caption, and the lexical flavor has no figure node either. + const payload = await copyToConstructBoundary(corpusUsj(FIGURE.fixture), FIGURE.tag, 0); + expect(payload[PLAIN]).not.toContain(FIGURE_BYTES); + expect(payload[PLAIN]).not.toContain("\\fig"); + expect(payload[LEXICAL]).not.toContain(FIGURE_BYTES); + expect(payload[LEXICAL]).not.toContain('"figure"'); + }); + + it("a selection ending exactly at a TEXT-LED construct's start carries it WHOLE in the lexical flavor — a superset of text/plain, deliberately, because the alternative loses the node", async () => { + // A `ref` has no display bytes of its own (USJ invented the container), so its first child is a + // real TextNode and Lexical normalizes the same focus point into a TEXT point at that child's + // offset 0. The child is then in `getNodes()` contributing zero characters: `text/plain` + // correctly emits none of it, while `isSelected` answers true and the lexical flavor keeps the + // construct. + // + // That residual disagreement is the DELIBERATE half of the trade. Answering false instead makes + // `$appendNodesToJSON` hoist the construct's children in place of the excluded wrapper, and a + // token-mode child is never sliced, so the copy carries `Genesis 1:1` with the `ref` node and + // its `loc` attribute silently gone — the same convincing-lie shape this suite's header + // describes. Measured both ways; this pin fixes which one ships. + const payload = await copyToConstructBoundary(corpusUsj(REF.fixture), REF.tag, 0); + expect(payload[PLAIN]).not.toContain(REF_BYTES); + expect(payload[LEXICAL]).toContain(REF_BYTES); + // Whole, not hoisted: the wrapper and the attribute it carries travel with those bytes. + expect(payload[LEXICAL]).toContain('"ref"'); + expect(payload[LEXICAL]).toContain('"loc":"GEN 1:1"'); + }); - it(`extending that same selection over the ${tag}'s first child puts it back in BOTH carriers`, async () => { + [ + { ...FIGURE, bytes: FIGURE_BYTES }, + { ...REF, bytes: REF_BYTES }, + ].forEach(({ tag, fixture, bytes }) => { + it(`extending that selection over the ${tag}'s first child puts its content in BOTH carriers`, async () => { const payload = await copyToConstructBoundary(corpusUsj(fixture), tag, 1); - expect(payload[PLAIN]).toContain(bytes); expect(payload[LEXICAL]).toContain(`"${tag}"`); + expect(payload[LEXICAL]).toContain(bytes); + expect(payload[PLAIN]).toContain(tag === "figure" ? "\\fig" : bytes); }); }); }); From bef61e422b57edfb2ba9825c143fcb847854c228 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 15:31:28 -0500 Subject: [PATCH 13/44] fix: reassemble a copied \periph division on paste MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A copied `\periph` came back as an ordinary paragraph carrying its own attribute bytes as literal content: `usfmFragmentToUsj.ts` had assemblies for `\esb`/`\esbe` and `\tr`/`\th`/`\tc` but none for `\periph`. The suspected second cause — that the one-line opaque-construct copy removes the attribute run's only delimiter — does not hold. Measured: the one-line and line-per-marker spellings tokenize identically, because a line break is ordinary whitespace and the next `\marker` is what ends a division's marker line either way. The one-lining is in fact required: a paste replays every line break as a paragraph split and Tier 2 re-tokenizes one paragraph at a time, so only the one-line form lets a single pass see the division together with the blocks inside it. Pasting the line-per-marker spelling was measured to come back as an empty division with its paragraph beside it, and that measurement is now a pin. `\periph` therefore assembles like a sidebar: the marker line splits at its first `|` into the division title (`alt`) and a named-attribute list, and the division takes every following block until the next `\periph`, a chapter, or the fragment end. A division has no closing bytes, so there is no unterminated state to record. An attribute list that does not parse degrades the whole division to a literal paragraph, as an unfoldable figure span does. A sidebar now lands in an open division instead of beside it, which required pushing it before assigning `sidebar` — `blockTarget()` answers "the open sidebar" whenever one is open, so assigning first nested it inside itself. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 30 ++-- .../converters/usfm/usfmFragmentToUsj.test.ts | 119 ++++++++++++++++ .../src/converters/usfm/usfmFragmentToUsj.ts | 132 +++++++++++++++--- .../unknownClipboardFidelity.test.tsx | 43 +++--- 4 files changed, 266 insertions(+), 58 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 0871f427a..5772b3cf4 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -26,7 +26,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). -- **An opaque construct copies out on ONE line:** the walk's ordinary rule puts a `\n` at every non-inline element boundary it crosses, which would spread a single construct over several lines — a sidebar's nested `\p`, and every row and cell of a table, are real block-level nodes. Pasting that text replays each `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct in one pass. `$startsBlockLine` (`whitespaceDisplay.plugin.utils.ts`) therefore suppresses the line break for a block boundary with an opaque ancestor ABOVE it — the construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document; it is simply not the line-per-marker layout a USFM writer emits, which is the deliberate trade for a paste that survives. Pinned by the corpus sweep's now-unskipped `"sidebar (esb)"` and `"table with header and cells"` fixtures and by `unknownClipboardFidelity.test.tsx`. Tables needed one more thing on the paste side: `usfm.sty` classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the fragment tokenizer as a char token and its table-cell assembly — which only ran for paragraph tokens — never saw it, turning every cell into a loose `char` span inside an empty row. ParatextData derives a cell from the marker NAME, so the assembly now recognizes a cell marker on either token kind, guarded on an open `\tr` row. Pinned sheet-backed (`createMarkerLookup(defaultStyleInfo)`, where the cell markers really are Character) in `usfmFragmentToUsj.test.ts`'s `"cell markers under a real stylesheet"` describe, including the align-infix names (`\thc3` → center, `\thr5` → end) and both guards — a cell-named span with no open row stays an ordinary char span, and a nested `\+tc1` is never a cell. +- **An opaque construct copies out on ONE line:** the walk's ordinary rule puts a `\n` at every non-inline element boundary it crosses, which would spread a single construct over several lines — a sidebar's nested `\p`, and every row and cell of a table, are real block-level nodes. Pasting that text replays each `\n` as a paragraph split, and Tier 2 re-tokenizes strictly one paragraph at a time, so the `\esb`/`\esbe` and `\tr`/`\th`/`\tc` assembly the fragment tokenizer already implements never saw the whole construct in one pass. `$startsBlockLine` (`whitespaceDisplay.plugin.utils.ts`) therefore suppresses the line break for a block boundary with an opaque ancestor ABOVE it — the construct's own outermost node still starts its own line, so a table stays separated from the prose around it. USFM markers self-delimit, so the one-line form is valid USFM of the same document; it is simply not the line-per-marker layout a USFM writer emits, which is the deliberate trade for a paste that survives. Pinned by the corpus sweep's now-unskipped `"sidebar (esb)"` and `"table with header and cells"` fixtures and by `unknownClipboardFidelity.test.tsx`. A `\periph` peripheral division needs the rule just as much, for the same reason and with nothing periph-specific about it: the division marker carries no closing bytes at all, so what ends its attribute run and what opens its content are the SAME thing — the next marker — and both spellings tokenize identically (pinned in `usfmFragmentToUsj.test.ts`). It is the paste's line splitting, not the byte layout, that decides whether one re-tokenization pass ever sees the division together with the blocks inside it; measured both ways and pinned in `unknownClipboardFidelity.test.tsx`. Tables needed one more thing on the paste side: `usfm.sty` classifies `\th1`/`\tc1`/… as CHARACTER styles, so with a real project stylesheet a cell marker reached the fragment tokenizer as a char token and its table-cell assembly — which only ran for paragraph tokens — never saw it, turning every cell into a loose `char` span inside an empty row. ParatextData derives a cell from the marker NAME, so the assembly now recognizes a cell marker on either token kind, guarded on an open `\tr` row. Pinned sheet-backed (`createMarkerLookup(defaultStyleInfo)`, where the cell markers really are Character) in `usfmFragmentToUsj.test.ts`'s `"cell markers under a real stylesheet"` describe, including the align-infix names (`\thc3` → center, `\thr5` → end) and both guards — a cell-named span with no open row stays an ordinary char span, and a nested `\+tc1` is never a cell. ### S2. Paste (Standard view, internal) — same-namespace `application/x-lexical-editor` payload reconstructs the exact node tree (existing) @@ -34,7 +34,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 13 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 12 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -143,8 +143,7 @@ shared USJ round-trip corpus (`corpus-data.ts`), select the chapter's content, c fresh editor holding the same chapter header, and compare the resulting USJ to the source. Two of the remaining failures are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes capable of representing the construct, so no paste-side fix is possible without a different carrier. One is -a construct the fragment tokenizer has no assembly for. One is ACCEPTED normalization that matches -Paratext 9's own behavior, not a bug at all. Every one of them stays in the sweep as `it.skip`, not +ACCEPTED normalization that matches Paratext 9's own behavior, not a bug at all. Every one of them stays in the sweep as `it.skip`, not deleted, so a future engine change un-skips it automatically instead of the gap going unnoticed — which is how the sidebar, table and milestone entries that used to sit here left the list. @@ -168,19 +167,7 @@ which is how the sidebar, table and milestone entries that used to sit here left confirming the ambiguity is specifically about trailing content after an implicit close, not `closed="false"` itself. -3. **`\periph` degrades to an ordinary paragraph (tokenizer gap):** the copy is byte-faithful and - now lands on ONE line (`\periph Title Page|id="title"\mt1 The Title`), so this is no longer the - block-split the sidebar and table entries used to describe. Two `\periph`-specific gaps remain. - The fragment tokenizer has no periph assembly at all — nothing turns `\periph …` back into a - `periph` object, so the paste comes back as an ordinary paragraph whose marker happens to be - `periph`. And `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` - run on a marker that has NO closing bytes, so even with an assembly there is nothing in the text - marking where the attribute run ends and the content resumes. Closing this means giving the - tokenizer a periph assembly and deciding how a closer-less division marker carries attributes at - all — a construct-modelling question, not a clipboard one. Pinned as a loss (not left unpinned) - in `unknownClipboardFidelity.test.tsx`'s periph describe. - -4. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a +3. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a bug):** isolated with a minimal non-corpus repro: pasting the literal text `"\p X"` (marker, its own required separator, and a SECOND, real content-leading space) into a fresh empty `"\p"` host produces `"\p X"` — one space, not two. The mechanism is `consumeSeparator()` @@ -222,11 +209,9 @@ The following items are recorded as deferred and not addressed in this plan: 10. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. -11. **`\periph` has no tokenizer assembly, so a copied periph pastes back as an ordinary paragraph:** the sidebar/table paste-rebuild scoping gaps that used to sit in this list are closed — an opaque construct now copies out on ONE line (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) so a single-paragraph rebuild can reassemble it. `\periph` does not benefit, for two reasons of its own. `usfmFragmentToUsj.ts` has no periph assembly at all (it has `\esb`/`\esbe` for sidebars and `\tr`/`\th`/`\tc` for tables), so nothing turns `\periph …` back into a `periph` object; and `unknownDisplayParts` renders periph's non-`alt` attributes as a `|name="value"` run on a marker with NO closing bytes, so nothing in the text marks where the attribute run ends and the content resumes. The paste therefore comes back as a paragraph whose marker is `periph`, carrying its own attribute bytes as literal content. Closing this means giving the tokenizer a periph assembly AND deciding how a closer-less division marker carries attributes at all — a construct-modelling question rather than a clipboard one, so it is recorded rather than guessed at. Pinned as a loss in `unknownClipboardFidelity.test.tsx`. See "Known Lossy Constructs" item 3 above. +11. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. -12. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. - -13. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. +12. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. ## Test Mapping (S1–S7) @@ -241,9 +226,10 @@ The following items are recorded as deferred and not addressed in this plan: | **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | +| **S3 (peripheral divisions)** `\periph` re-tokenizes back into a `periph` division — its title, its attribute list, and the blocks it contains | `libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts` — `"usfmFragmentToUsjContent — peripheral divisions (\\periph)"` describe: the marker line split into `alt` plus attributes, the line-per-marker spelling reading identically, following blocks taken into the division, one division ending at the next, a chapter-free contentless division, a nested sidebar, the unparseable-attribute degrade to a literal paragraph, and note context opening no division at all | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar and table round-trip on all three shapes; `ref` and `periph` round-trip on the lexical flavor and have their plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper; `\periph` has no tokenizer assembly — "Known Lossy Constructs" item 3). Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 13), asserting content BYTES rather than tags | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 12), asserting content BYTES rather than tags | | **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts index 7d0b73dab..e84d8f211 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts @@ -1941,3 +1941,122 @@ describe("usfmFragmentToUsjContent — figures parse without a project styleshee ]); }); }); + +describe("usfmFragmentToUsjContent — peripheral divisions (\\periph)", () => { + const PERIPH_USJ = { + type: "periph", + alt: "Title Page", + id: "title", + content: [{ type: "para", marker: "mt1", content: ["The Title"] }], + }; + + it("assembles a periph division, splitting its marker-line text into alt and attributes", () => { + expect(usfmFragmentToUsjContent('\\periph Title Page|id="title"\\mt1 The Title')).toEqual([ + PERIPH_USJ, + ]); + }); + + it("reads the line-per-marker spelling identically — the next marker delimits the attribute run, not the line break", () => { + // A USFM writer puts `\periph` on its own line, but the newline is ordinary whitespace to a + // tokenizer: the marker line's text ends where the next `\marker` begins either way. That is + // what lets an opaque construct copy out on ONE line and still parse (`$startsBlockLine`, + // whitespaceDisplay.plugin.utils.ts) — and it is why a periph's content can be reassembled at + // all, since a Tier-2 rebuild only ever re-tokenizes a single paragraph's bytes. + expect(usfmFragmentToUsjContent('\\periph Title Page|id="title"\n\\mt1 The Title')).toEqual([ + PERIPH_USJ, + ]); + }); + + it("takes every following block into the division — periph has no closing marker", () => { + expect( + usfmFragmentToUsjContent('\\periph Title Page|id="title"\\mt1 The Title\\p Body.'), + ).toEqual([ + { + type: "periph", + alt: "Title Page", + id: "title", + content: [ + { type: "para", marker: "mt1", content: ["The Title"] }, + { type: "para", marker: "p", content: ["Body."] }, + ], + }, + ]); + }); + + it("ends one division at the next, since peripheral divisions never nest", () => { + expect(usfmFragmentToUsjContent("\\periph One\\p a\\periph Two\\p b")).toEqual([ + { type: "periph", alt: "One", content: [{ type: "para", marker: "p", content: ["a"] }] }, + { type: "periph", alt: "Two", content: [{ type: "para", marker: "p", content: ["b"] }] }, + ]); + }); + + it("keeps a marker-line with no attributes as pure alt text, its structural line break included", () => { + expect(usfmFragmentToUsjContent("\\periph Title Page\n\\mt1 The Title")).toEqual([ + { + type: "periph", + alt: "Title Page", + content: [{ type: "para", marker: "mt1", content: ["The Title"] }], + }, + ]); + }); + + it("keeps an attributes-only marker-line free of an empty alt", () => { + expect(usfmFragmentToUsjContent('\\periph |id="title"\\mt1 The Title')).toEqual([ + { + type: "periph", + id: "title", + content: [{ type: "para", marker: "mt1", content: ["The Title"] }], + }, + ]); + }); + + it("emits a contentless division for a marker line with nothing after it", () => { + expect(usfmFragmentToUsjContent('\\periph Title Page|id="title"')).toEqual([ + { type: "periph", alt: "Title Page", id: "title" }, + ]); + }); + + it("nests a sidebar inside the open division rather than beside it", () => { + expect( + usfmFragmentToUsjContent("\\periph One\\esb \\cat History\\cat*\\p in sidebar\\esbe"), + ).toEqual([ + { + type: "periph", + alt: "One", + content: [ + { + type: "sidebar", + marker: "esb", + category: "History", + content: [{ type: "para", marker: "p", content: ["in sidebar"] }], + }, + ], + }, + ]); + }); + + it("degrades to an ordinary paragraph when the attribute list does not parse, keeping every byte", () => { + // Same refusal as every other attribute list here: `|id=""` is not a reading Paratext agrees + // with, so the bytes stay literal text where the author can see and fix them. + expect(usfmFragmentToUsjContent('\\periph Title Page|id=""\\mt1 The Title')).toEqual([ + { type: "para", marker: "periph", content: ['Title Page|id=""'] }, + { type: "para", marker: "mt1", content: ["The Title"] }, + ]); + }); + + it("opens no division inside note content, where peripheral divisions do not occur", () => { + expect( + usfmFragmentToUsjContent("\\ft text \\periph Title\\mt1 The Title", { isNoteContext: true }), + ).toEqual([ + { + type: "para", + marker: "p", + content: [ + { type: "char", marker: "ft", content: ["text "], closed: "false" }, + { type: "char", marker: "periph", content: ["Title"], closed: "false" }, + ], + }, + { type: "para", marker: "mt1", content: ["The Title"] }, + ]); + }); +}); diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts index 4fcbc0549..f7622e180 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts @@ -28,17 +28,21 @@ * element. A stray `\*` with no milestone to close is NOT literal either — it * becomes an unmatched element (see above). * - * Figures, tables, and sidebars assemble to their faithful USJ shapes at the - * assembly level, marker-name driven (they are parser-level structures in - * ParatextData, independent of stylesheet classification): `\fig …\fig*` folds - * to an inline `figure` object (USFM's `src` attribute renamed to USX/USJ's - * `file`), `\tr` plus `t[hc][rc]#(-#)` cell markers build `table` → - * `table:row` → `table:cell` with name-derived `align`/`colspan`, and + * Figures, tables, sidebars, and peripheral divisions assemble to their faithful + * USJ shapes at the assembly level, marker-name driven (they are parser-level + * structures in ParatextData, independent of stylesheet classification): + * `\fig …\fig*` folds to an inline `figure` object (USFM's `src` attribute + * renamed to USX/USJ's `file`), `\tr` plus `t[hc][rc]#(-#)` cell markers build + * `table` → `table:row` → `table:cell` with name-derived `align`/`colspan`, * `\esb`…`\esbe` wraps the following blocks in a `sidebar` (`\cat` directly - * after `\esb` folds to its `category`). Anything off the clean shapes — - * nested markup or positional (USFM 2.0) attributes in a figure, a missing - * `\fig*`, a cell marker with no open row — degrades to the plain char/para - * output the marker classification produces on its own. + * after `\esb` folds to its `category`), and `\periph` opens a `periph` + * division that takes every following block until the next `\periph`, a + * chapter, or the fragment end — its marker line splitting at the first `|` + * into the division title (`alt`) and an ordinary attribute list. Anything off + * the clean shapes — nested markup or positional (USFM 2.0) attributes in a + * figure, a missing `\fig*`, a cell marker with no open row, a periph + * attribute list that does not parse — degrades to the plain char/para output + * the marker classification produces on its own. * * Input is USFM text: `~` means NBSP; U+FFFC sentinels (atomic-node placeholders * from the Tier 2 fragment builder) ride through as ordinary text characters. @@ -69,6 +73,7 @@ const FIGURE_MARKER = "fig"; const TABLE_ROW_MARKER = "tr"; const SIDEBAR_MARKER = "esb"; const SIDEBAR_END_MARKER = "esbe"; +const PERIPH_MARKER = "periph"; /** * Table cell marker names: `t` + header/cell (`h`/`c`) + optional alignment infix (`r`/`c`) + @@ -814,10 +819,20 @@ export function usfmFragmentToUsjContent( // tables). Implicit close (fragment end or a chapter token) marks it closed="false" — // ParatextData auto-closes sidebars at the chapter boundary. let sidebar: ClosableMarkerObject | undefined; + // Current open peripheral division: `\periph` opens one and takes every following block into + // it. Nothing closes a division but the next `\periph`, a chapter boundary, or the end of the + // fragment — USFM gives a division marker no closing bytes at all, so unlike a sidebar there is + // no terminated/implicit distinction to record. The marker LINE's own text (the division title + // and its attribute list) is captured first; see `periphCapture`. + let periph: MarkerObject | undefined; - /** Where top-level blocks (paragraphs, tables) land: an open sidebar's content, else the - * fragment result. */ - const blockTarget = (): MarkerContent[] => (sidebar ? getContent(sidebar) : result); + /** Where top-level blocks (paragraphs, tables) land: an open sidebar's content, else an open + * peripheral division's content, else the fragment result. */ + const blockTarget = (): MarkerContent[] => { + if (sidebar) return getContent(sidebar); + if (periph) return getContent(periph); + return result; + }; // True between a chapter token and the next opened block: loose content there sits at the // DOCUMENT ROOT in ParatextData's output, not in an implied paragraph — text typed after @@ -912,6 +927,11 @@ export function usfmFragmentToUsjContent( sidebar = undefined; }; + /** End the open division. Nothing to record: a division has no closing bytes to be missing. */ + const closePeriph = () => { + periph = undefined; + }; + // ---- attribute-marker folding state (see ATTRIBUTE_MARKERS) ---- // The most recent chapter/verse/note object, still "receptive": an adjacent attribute // marker folds onto it as an attribute. Any real content clears it. @@ -980,6 +1000,47 @@ export function usfmFragmentToUsjContent( attrCapture = undefined; }; + // ---- peripheral-division capture state ---- + // The text of a `\periph` marker line, collected between the marker and the next marker token. + // `\periph` has no closing bytes, so the next marker is the only thing that ends the line — + // the same delimiter whether a USFM writer put the division on its own line or a copy laid the + // whole construct out on one (a line break is ordinary whitespace to a tokenizer). That is what + // lets a Tier-2 rebuild, which only ever re-tokenizes a single paragraph's bytes, reassemble a + // whole division from one fragment. + let periphCapture: { value: string } | undefined; + /** + * Close the captured marker line and open the division it describes: everything before the + * line's first `|` is the division title (USX/USJ's `alt`), the rest an ordinary named-attribute + * list. + * + * A list that does not parse degrades the whole division to an ordinary paragraph carrying the + * literal bytes, exactly as an unfoldable figure span degrades — every byte the author wrote + * stays where they can see and fix it. + * + * @param atBlockBoundary - Whether the token that ended the line starts a new block (or the + * fragment ended). The line's trailing line break is structural there, the same rule the text + * case applies: the line ends where the next block marker begins, and ParatextData emits no + * content for it. + */ + const finishPeriphLine = (atBlockBoundary: boolean) => { + if (!periphCapture) return; + let { value } = periphCapture; + periphCapture = undefined; + if (atBlockBoundary && value.endsWith("\n")) value = value.slice(0, -1); + const pipeIndex = value.indexOf("|"); + const attributes = + pipeIndex >= 0 ? parseAttributeText(value.slice(pipeIndex + 1), PERIPH_MARKER) : undefined; + if (pipeIndex >= 0 && !attributes) { + startParagraph(PERIPH_MARKER, value); + return; + } + const title = pipeIndex >= 0 ? value.slice(0, pipeIndex) : value; + periph = { type: "periph", ...(title ? { alt: toUsjText(title) } : {}), ...attributes }; + periph.content = []; + result.push(periph); + para = undefined; + }; + // ---- figure capture state ---- // A `\fig …\fig*` span being collected for faithful `figure` emission (ParatextData turns // the span into `{ type: "figure", … }`). Only a clean span folds: plain-text content with @@ -1111,6 +1172,20 @@ export function usfmFragmentToUsjContent( } } + if (periphCapture) { + // The marker line runs to the next marker token. An optbreak rejoins as the literal `//` + // the author typed, for the same reason the figure capture rejoins one: `//` inside an + // attribute value (a URL, say) is plain value bytes to ParatextData, which strips the + // attribute segment out at tokenize-time before its `//` pass ever runs. + if (token.kind === "text" || token.kind === "optbreak") { + periphCapture.value += token.kind === "text" ? token.text : "//"; + continue; + } + finishPeriphLine(token.kind === "para" || token.kind === "chapter"); + tokenIndex--; + continue; + } + if (figCapture) { if (token.kind === "text" || token.kind === "optbreak") { // The tokenizer splits `//` into optbreak tokens spec-blind — including inside what @@ -1276,8 +1351,15 @@ export function usfmFragmentToUsjContent( closeNote(false); // Sidebars never nest: an unterminated previous sidebar closes implicitly. closeSidebar(false); - sidebar = { type: "sidebar", marker: SIDEBAR_MARKER, content: [] }; - result.push(sidebar); + // Landed BEFORE `sidebar` is set: `blockTarget()` answers "the open sidebar's content" + // whenever one is open, so assigning first would nest the new sidebar inside itself. + const opened: ClosableMarkerObject = { + type: "sidebar", + marker: SIDEBAR_MARKER, + content: [], + }; + blockTarget().push(opened); + sidebar = opened; para = undefined; attrTarget = sidebar; // receptive to \cat (directly after \esb only) atChapterRootScope = false; @@ -1292,6 +1374,19 @@ export function usfmFragmentToUsjContent( para = undefined; break; } + // ---- peripheral-division assembly ---- + if (!isNoteContext && token.marker === PERIPH_MARKER) { + closeCharStack(); + closeNote(false); + // A division ends every block structure open inside it, including a sidebar the author + // never terminated; divisions themselves never nest. + closeSidebar(false); + closePeriph(); + periphCapture = { value: "" }; + para = undefined; + atChapterRootScope = false; + break; + } startParagraph(token.marker); break; } @@ -1312,9 +1407,11 @@ export function usfmFragmentToUsjContent( closeCharStack(); closeNote(false); // A chapter boundary ends any open table and implicitly closes an open sidebar - // (ParatextData auto-closes sidebars at the end of the chapter). + // (ParatextData auto-closes sidebars at the end of the chapter). A peripheral division + // ends there too: chapters are book body, which is never division content. endTable(); closeSidebar(false); + closePeriph(); para = undefined; const chapter: MarkerObject = { type: "chapter", @@ -1406,6 +1503,9 @@ export function usfmFragmentToUsjContent( break; } } + // Fragment ended on a `\periph` marker line: the division has its title and attributes but no + // content, which is a complete division — a periph never had a closer to be missing. + if (periphCapture) finishPeriphLine(true); // Fragment ended mid-figure: no `\fig*` closer — degrade to the plain char/para span. if (figCapture) materializeFigCapture(); if (attrCapture) { diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx index 245134bc5..c7cddc502 100644 --- a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -232,32 +232,35 @@ describe("sidebar (UnknownNode) copy→paste across all three payload shapes", ( describe("periph (UnknownNode) copy→paste across all three payload shapes", () => { const usj = corpusUsj("periph"); - it("full payload (lexical flavor) round-trips the periph construct whole — its id/alt attributes and nested paragraph included", async () => { + SHAPES.forEach((shape) => { // Compared construct-to-construct rather than document-to-document: `periph` is book-level // front matter with no chapter, so this sweep's "header plus one empty `\\p` host" target // leaves the pasted block nested inside that host paragraph. Where the block LANDS is generic - // Lexical insertion, not construct fidelity; what this pin is about is that nothing inside the - // construct was lost on the way. - const pasted = await pastedUsjFor(usj, "full"); - expect(objectsOfType(pasted, "periph")).toEqual(objectsOfType(usj, "periph")); - }); - - (["plain", "plain+html"] as const).forEach((shape) => { - // Not a block split: an opaque construct copies out on ONE line (`$startsBlockLine`, - // `whitespaceDisplay.plugin.utils.ts`), which is what lets the sidebar above round-trip. Two - // `\periph`-specific gaps stop periph taking the same route: the fragment tokenizer has no - // periph assembly (nothing turns `\periph …` back into a `periph` object), and periph's display - // renders its non-`alt` attributes as a `|name="value"` run on a marker that has no closing - // bytes, so nothing marks where the attribute run ends and the content resumes. The paste - // therefore comes back as an ORDINARY paragraph whose marker is `periph`, carrying the - // attribute bytes as literal text. - it(`${shape} payload loses the periph construct — it re-tokenizes as an ordinary paragraph carrying its own attribute bytes as text`, async () => { + // Lexical insertion, not construct fidelity; what these pins are about is that nothing inside + // the construct was lost on the way. + it(`${shape} payload round-trips the periph construct whole — its id/alt attributes and nested paragraph included`, async () => { const pasted = await pastedUsjFor(usj, shape); - expect(objectsOfType(pasted, "periph")).toEqual([]); - const degraded = objectsOfType(pasted, "para").find((item) => item.marker === "periph"); - expect(degraded?.content).toEqual(['Title Page|id="title"']); + expect(objectsOfType(pasted, "periph")).toEqual(objectsOfType(usj, "periph")); }); }); + + it("loses the division's content if the copy is laid out line-per-marker instead of on one line", async () => { + // Why the one-line copy rule (`$startsBlockLine`, `whitespaceDisplay.plugin.utils.ts`) has to + // cover a peripheral division too, measured rather than argued. A `\periph` line and the + // blocks it contains are separate USFM lines the way a writer emits them, and a paste replays + // every line break as a paragraph split — after which Tier 2 re-tokenizes each paragraph on + // its own and no single pass ever sees the division together with its content. The tokenizer + // reads both spellings identically (`usfmFragmentToUsj.test.ts`); it is the paste's line + // splitting, not the byte form, that decides whether the content can be reassembled. + const payload = await copyContentPayload(usj); + const lineBroken = payload[PLAIN].replace("\\mt1", "\n\\mt1"); + const pasted = await pasteIntoFreshHost(usj, { [PLAIN]: lineBroken }); + if (!pasted) throw new Error("editor produced no USJ"); + expect(objectsOfType(pasted, "periph")).toEqual([ + { type: "periph", alt: "Title Page", id: "title" }, + ]); + expect(objectsOfType(pasted, "para").map((item) => item.marker)).toContain("mt1"); + }); }); describe("ref (UnknownNode) copy→paste across all three payload shapes", () => { From f70df0005130ae3f75d1d1a52b913e7108d494ce Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 15:45:52 -0500 Subject: [PATCH 14/44] fix: give a mixed attribute-context paste body content's paste rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred item 9's two attribute-context paste residuals, characterized and closed. (a) does not reproduce and is not paste-specific. A mixed selection reaching past a char span's closing glyph into following sibling text settles to byte-identical USJ whether the replacement is typed or pasted, across four selection shapes (forward and backward, run-start-to-trailing-end and run-end-to-trailing-start included). The span coming back `closed="false"` with its attribute run collapsed into content is the arithmetic of a range that covered the closer, not a defect: deleting a range containing the closing glyph deletes the closing glyph. (b) reproduces, from BOTH directions, and is a data-fidelity hole. The value-byte carve-outs — no `\c`/`\id` strip, no positional NBSP mapping — were applied to any selection that merely TOUCHED an attribute run. Those carve-outs are earned by one fact only: the bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` skips. A touching selection loses that the moment its removal takes the range out of the run, so pasting `\c 5` over one produced a real chapter node mid-paragraph with the rest of the paragraph stranded outside it — the save-loop poisoning the strip exists to prevent — and a pasted interior NBSP was read as display whitespace and saved as a plain space instead of surviving as data. Measured with the range starting in body text and reaching into the run, and with it starting inside the run and reaching past the closer. The carve-outs now apply only to a selection wholly inside one attribute node. The insertion MECHANISM is unchanged for every touching selection — one `insertText`, never the paragraph-splitting line replay and never Lexical's rich-paste node insertion — so a multi-line payload landing in body content this way still collapses per newline rather than splitting a paragraph the removal has cut a char span in half in. The single-paragraph fixture the new pins use is what made this measurable at all: the earlier probing needed a cross-paragraph range, which does not hold in jsdom. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 20 +- .../attributeContextPasteFidelity.test.tsx | 194 ++++++++++++++++++ .../whitespaceDisplay.plugin.utils.ts | 65 ++++-- 3 files changed, 254 insertions(+), 25 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 5772b3cf4..1c6696d39 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -34,7 +34,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 12 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 11 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -69,11 +69,13 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra - **Pasted `\c` / `\id`:** Strip during paste normalization (approved). Live-verified harm (2026-08-07): pasting `\c 2` mid-chapter put a chapter node in the editor and poisoned the save loop — PDP rejects every save with "Multiple chapter markers present", the error surfaces only in the renderer log, disk and other editors silently stop updating. (P9 destroys pasted `\c` at save with a user-facing error.) -- **Paste inside an attribute display run — paste ≡ typing:** a selection that TOUCHES attribute-display text at either end — a char span's bare `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value — is inserted exactly as the SAME characters typed at the SAME caret (or over the same selection) would be: each `\n` becomes a single space, per-newline, not run-collapsed (attribute values are single-line; there is no multi-line attribute byte shape to collapse INTO — `"a\n\nb"` pastes as `"a b"`, two spaces), and NO chapter/book-id strip, NO positional NBSP mapping, and NO marker tokenization run against the pasted bytes — a pasted `\c 5` or `\p` here is literal value text, not a structural marker, and a pasted NBSP passes through unchanged (matching `$displayWhitespaceTransform`'s own skip of attribute-tagged text). A selection that only PARTLY sits inside attribute-display text (one end in, one end out) is included too — see below. The existing attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, `$resolvePendingMarkers`) then re-tokenizes the displayed bytes back into node state identically whether they arrived by typing or paste. +- **Paste inside an attribute display run — paste ≡ typing:** a selection that TOUCHES attribute-display text at either end — a char span's bare `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value — is inserted exactly as the SAME characters typed at the SAME caret (or over the same selection) would be: one `selection.insertText`, with each `\n` becoming a single space, per-newline, not run-collapsed (attribute values are single-line; there is no multi-line attribute byte shape to collapse INTO — `"a\n\nb"` pastes as `"a b"`, two spaces). A selection that only PARTLY sits inside attribute-display text (one end in, one end out) takes this same INSERTION MECHANISM — see below for the two corruption shapes that earns it. The existing attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, `$resolvePendingMarkers`) then re-tokenizes the displayed bytes back into node state identically whether they arrived by typing or paste. + + **The two BODY-content paste rules are skipped only for a selection wholly inside ONE attribute node** — the `\c`/`\id` strip and the positional NBSP mapping, which are the only two places a paste deliberately diverges from typing the same bytes. That carve-out is earned by exactly one fact: the bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` skips, so a pasted `\c 5` or `\p` there really is literal value text and never a structural marker, and a pasted NBSP passes through unchanged (matching `$displayWhitespaceTransform`'s own skip of attribute-tagged text). A selection that merely TOUCHES an attribute run does not have that property, and pretending it does was a real data-fidelity hole: removing such a range takes the bytes out of the run — and when the range covers a char span's closing glyph it deletes the closer, so the whole span re-tokenizes around whatever just arrived. Measured both ways round (a range starting in body text and reaching INTO the run, and one starting inside the run and reaching PAST the closer), pasting `\c 5` over such a selection produced a real chapter node mid-paragraph with the rest of the paragraph stranded outside it — the exact save-loop poisoning the strip exists to prevent — and a pasted interior NBSP was read as display whitespace and saved as a plain space instead of surviving as data. So a touching-but-not-wholly-inside selection gets body content's rules for the bytes, while keeping the attribute path's insertion mechanism. One deliberate residual comes with that split: a MULTI-LINE payload landing in body content this way still collapses per newline rather than splitting the paragraph, because splitting a paragraph whose char span the removal has just cut in half is the corruption this path exists to avoid. TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the saved file diverged from the editor. `sid="things"` carries no NBSP; the pre-branch build's paste handler still had its OLD NBSP-gated form (see the "positional NBSP normalization" bullet above — that gate was generalized to claim every external paste, NBSP or not, on 2026-08-07), so the most plausible mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion — NOT a same-namespace `application/x-lexical-editor` flavor on the clipboard (this document's own "private Lexical flavor is dead on Ctrl+V" fact, above, still holds for the reconstructed-`DataTransfer` paste path this repro most likely used). - What matters for THIS branch's code is the SHAPE the corruption takes once ANY handler declines an attribute-context paste to Lexical's default rich-paste node insertion — reproduced directly here (`attributeContextPasteFidelity.test.tsx`'s "root cause" describe): it has no notion that an attribute run's text must stay inside its one tagged TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph's sibling text into one plain node, destroying the attribute display and the closing marker and leaving the pasted bytes loose in body content. On this branch's CURRENT code (a single-line, NBSP-free, flavor-free external paste is already safe since 2026-08-07's generalization), the confirmed regression classes this paste-≡-typing rule closes are: (1) a live native paste event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for when that reaches a real handler); (2) a multi-line plain-text payload, which the ordinary pipeline would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND`; (3) a marker-bearing payload (`\c 5`), which the ordinary pipeline's `\c`/`\id` strip would eat bytes out of an attribute VALUE that were never a chapter token; (4) a MIXED selection (one end in the run, one end out) combined with either (1) or (2) — the selection touches attribute context, so it must not reach either risky branch merely because its other end sits outside. + What matters for THIS branch's code is the SHAPE the corruption takes once ANY handler declines an attribute-context paste to Lexical's default rich-paste node insertion — reproduced directly here (`attributeContextPasteFidelity.test.tsx`'s "root cause" describe): it has no notion that an attribute run's text must stay inside its one tagged TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph's sibling text into one plain node, destroying the attribute display and the closing marker and leaving the pasted bytes loose in body content. On this branch's CURRENT code (a single-line, NBSP-free, flavor-free external paste is already safe since 2026-08-07's generalization), the confirmed regression classes this paste-≡-typing rule closes are: (1) a live native paste event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for when that reaches a real handler); (2) a multi-line plain-text payload, which the ordinary pipeline would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND`; (3) a marker-bearing payload (`\c 5`) landing wholly INSIDE one attribute node, where the ordinary pipeline's `\c`/`\id` strip would eat bytes out of an attribute VALUE that were never a chapter token (bytes merely TOUCHING a run do get that strip — see the body-rules paragraph above); (4) a MIXED selection (one end in the run, one end out) combined with either (1) or (2) — the selection touches attribute context, so it must not reach either risky branch merely because its other end sits outside. Pre-existing precedence, UNCHANGED by this rule: the CRITICAL-priority in-note multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs before the Standard-view external-paste handler and still wins for a multi-line payload whose selection touches EXPANDED note content — an attribute run that happens to sit inside an expanded note's content is reached by this rule only when that in-note claim itself declines. See `$handlePasteForStandardView`'s doc comment (`whitespaceDisplay.plugin.utils.ts`) for the full mechanism. @@ -205,13 +207,11 @@ The following items are recorded as deferred and not addressed in this plan: 8. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. -9. **Attribute-context paste residuals (2026-08-11, tracked follow-up):** two uncharacterized residuals from the attribute-run paste fix are recorded for investigation, not fixed here: (a) a mixed selection reaching past a char span's closing glyph into following sibling text may still corrupt on replacement — the available evidence suggests this class is a pre-existing selection-edit/engine behavior (typed input over the same selection showed the same anecdotal shape), not paste-specific; (b) a wide selection with one end merely touching an attribute run now takes the attribute paste path for the whole replacement, skipping the `\c`/`\id` strip and NBSP normalization for the body-side text — unverified (cross-paragraph mixed selections would not hold in jsdom during probing). - -10. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. +9. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. -11. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. +10. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. -12. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. +11. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. ## Test Mapping (S1–S7) @@ -221,7 +221,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"` describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | | **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"` and `"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"` (external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction); `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | | **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | -| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (confirmed regression class on this branch, not TJ's literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"milestone attribute run paste"`, and `"verse \\va run paste"` describes | +| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (confirmed regression class on this branch, not TJ's literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | | **S4** Paste-as-plain-text (Ctrl+Shift+V) — equivalence to external paste | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"paste-as-plain-text equivalence (S4): no literal mode, plain always wins"` describe | | **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | @@ -229,7 +229,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S3 (peripheral divisions)** `\periph` re-tokenizes back into a `periph` division — its title, its attribute list, and the blocks it contains | `libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts` — `"usfmFragmentToUsjContent — peripheral divisions (\\periph)"` describe: the marker line split into `alt` plus attributes, the line-per-marker spelling reading identically, following blocks taken into the division, one division ending at the next, a chapter-free contentless division, a nested sidebar, the unparseable-attribute degrade to a literal paragraph, and note context opening no division at all | | **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 12), asserting content BYTES rather than tags | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 11), asserting content BYTES rather than tags | | **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | diff --git a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx index 9d4c91f50..c5d4e7c21 100644 --- a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx @@ -34,6 +34,14 @@ * caret (or over the same selection). Every "paste ≡ typed" pin below proves paste and the * character-by-character TYPED equivalent settle to the identical USJ, not merely to * individually-plausible-looking results. + * + * That equivalence governs the insertion MECHANISM for every selection touching a run. It does NOT + * extend to the two rules a body-content paste applies that typing does not — the `\c`/`\id` strip + * and the positional NBSP mapping — which are skipped only for a selection that stays wholly inside + * ONE attribute node, the only shape whose bytes end up in a node that stays tagged "attribute" and + * therefore never re-tokenizes. The "mixed selection: the pasted bytes are BODY content" describe + * pins that split from both directions and pins the wholly-inside boundary that keeps value-byte + * semantics. */ import { @@ -671,6 +679,167 @@ describe("mixed selection: spans BOTH attribute and non-attribute content", () = }); }); +// --------------------------------------------------------------------------------------------- +// Single-paragraph mixed-selection fixture: `\p before \nd asdf|who="hi"\nd* after`, with body +// text on BOTH sides of the span in the SAME paragraph, plus a second paragraph to depart to. A +// mixed selection needs ordinary content and attribute-display text in one range, and a +// cross-paragraph range does not hold in jsdom — within one paragraph it does. +// --------------------------------------------------------------------------------------------- + +function $inlineCharFixture(): void { + const char = $createCharNode("nd"); + char.setUnknownAttributes({ who: "hi" }); + const run = $createTextNode('|who="hi"'); + $setState(run, textTypeState, "attribute"); + char.append( + $createMarkerNode("nd"), + $createTextNode(`${NBSP}asdf`), + run, + $createMarkerNode("nd", "closing"), + ); + const [glyph, separator] = $createMarkerPrefix("p"); + const [glyph2, separator2] = $createMarkerPrefix("p"); + $getRoot().append( + $createParaNode("p").append( + glyph, + separator, + $createTextNode("before "), + char, + $createTextNode(" after"), + ), + $createParaNode("p").append(glyph2, separator2, $createTextNode("body")), + ); +} + +const $leadingBodyText = (): TextNode => { + const node = $firstPara().getChildren()[2]; + if (!$isTextNode(node)) throw new Error("leading body text missing"); + return node; +}; +const $trailingBodyText = (): TextNode => { + const node = $firstPara().getLastChild(); + if (!$isTextNode(node)) throw new Error("trailing body text missing"); + return node; +}; + +/** anchor in the leading BODY text, focus inside the attribute run — the range crosses the span's + * opening glyph on its way in. */ +function $selectBodyIntoRun(): void { + $leadingBodyText().select(3, 3); + const selection = $getSelection(); + if ($isRangeSelection(selection)) + selection.focus.set($charAttributeRun($firstChar()).getKey(), 5, "text"); +} + +/** anchor inside the attribute run, focus in the trailing BODY text — the range covers the span's + * CLOSING glyph, so removing it leaves the span unterminated. */ +function $selectRunPastCloser(): void { + const run = $charAttributeRun($firstChar()); + run.select(1, 1); + const selection = $getSelection(); + if ($isRangeSelection(selection)) selection.focus.set($trailingBodyText().getKey(), 3, "text"); +} + +describe("mixed selection: the pasted bytes are BODY content, not attribute-value bytes", () => { + // Attribute-value semantics (no `\c`/`\id` strip, no positional NBSP mapping) are earned by one + // fact: the bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` + // skips, so they never re-tokenize as markers. Only a selection wholly inside ONE such node has + // that property. A selection that merely TOUCHES one loses it — removing the range takes the + // bytes out of the run (and, when the range covers the span's closing glyph, deletes the closer + // so the whole span re-tokenizes around them) — so the bytes are ordinary body content and get + // body content's paste rules. The INSERTION MECHANISM is unchanged either way: one + // `selection.insertText`, never the paragraph-splitting line replay and never Lexical's + // rich-paste node insertion, both of which corrupt the attribute-run end of the range. + + it("does not create a chapter node when the range starts in body text — the \\c strip applies", async () => { + const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => editor.update($selectBodyIntoRun)); + + await pasteAndFlush(editor, { "text/plain": "\\c 5 X" }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + const content = usjOf(editor).content; + expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( + [], + ); + // The paragraph also stays whole: a chapter token closes the enclosing paragraph, stranding + // everything after it as bare top-level USJ. + expect(content.every((item) => typeof item !== "string" && item.type === "para")).toBe(true); + }); + + it("does not create a chapter node when the range STARTS inside the run and reaches past the closing glyph", async () => { + // The bytes land at the run's own offset, yet still re-tokenize: the range took the closer with + // it, so the span is no longer a span with an attribute run to be inside of. + const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => editor.update($selectRunPastCloser)); + + await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + const content = usjOf(editor).content; + expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( + [], + ); + expect(content.every((item) => typeof item !== "string" && item.type === "para")).toBe(true); + }); + + it("keeps a pasted interior NBSP as DATA — the positional rule applies to body-bound bytes", async () => { + // Body rule: every NBSP that is not a marker-adjacent display separator is user data and + // becomes `~`, which the tokenizer reads back as a real NBSP. Inserted raw instead, the same + // byte is read as display whitespace and saves as a plain space — the data is gone. + const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => editor.update($selectBodyIntoRun)); + + await pasteAndFlush(editor, { "text/plain": `X${NBSP}Y` }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + expect(JSON.stringify(usjOf(editor).content)).toContain(`X${NBSP}Y`); + }); + + it("still settles a plain payload identically to the same character typed over the same range", async () => { + // The strip and the NBSP rule are the only two places body paste diverges from typing; a + // payload carrying neither must still land exactly where a keystroke would. This also answers + // the recorded suspicion that a range reaching past the closing glyph corrupts on replacement: + // it does not, and typed input over the same range produces the identical bytes — the span + // losing its closer is the arithmetic of a range that covered the closer, not a paste defect. + const typed = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => typed.editor.update($selectRunPastCloser)); + await typeCharByChar(typed.editor, "Z"); + await departAndSettle(typed.editor, () => $bodyTextNode().select(0, 0)); + + const pasted = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => pasted.editor.update($selectRunPastCloser)); + await pasteAndFlush(pasted.editor, { "text/plain": "Z" }); + await departAndSettle(pasted.editor, () => $bodyTextNode().select(0, 0)); + + expect(usjOf(pasted.editor)).toEqual(usjOf(typed.editor)); + }); + + it("leaves a range wholly INSIDE the run on attribute-value semantics — a pasted \\c stays literal there", async () => { + // The boundary of the rule above, in its non-collapsed form (the collapsed-caret form is pinned + // in the "marker-bearing payload" describe). The run survives this replacement still tagged + // "attribute", so the bytes really are value text and the strip must not touch them. + const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + const text = run.getTextContent(); // '|who="hi"' + run.select(text.indexOf('"') + 1, text.lastIndexOf('"')); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + + editor.getEditorState().read(() => { + expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="\\c 5"'); + }); + const content = usjOf(editor).content; + expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( + [], + ); + }); +}); + // --------------------------------------------------------------------------------------------- // Milestone attribute run: one paste case, matching milestoneAttributeSettle.test.tsx's idiom. // --------------------------------------------------------------------------------------------- @@ -725,6 +894,31 @@ describe("milestone attribute run paste", () => { expect(milestone.getUnknownAttributes()).toEqual({ who: "ed" }); }); }); + + it("a mixed selection reaching into the milestone's run takes body rules too — the rule is not char-span-specific", async () => { + // `$isNodeInAttributeContext` covers all three run kinds through one predicate, so the + // wholly-inside-versus-touching distinction is pinned on a second kind rather than left + // inferred from the char span that surfaced it. + const { editor } = await testEnvironmentWithSpacing($milestoneFixture); + await act(async () => + editor.update(() => { + const leading = $firstPara().getChildren()[2]; + if (!$isTextNode(leading)) throw new Error("leading body text missing"); + leading.select(3, 3); + const selection = $getSelection(); + if ($isRangeSelection(selection)) + selection.focus.set($milestoneAttributeRun().getKey(), 2, "text"); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + await departAndSettle(editor, () => $milestoneBodyText().select(0, 0)); + + const content = usjOf(editor).content; + expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( + [], + ); + }); }); // --------------------------------------------------------------------------------------------- diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 7962ed523..4b1c1c419 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -318,6 +318,29 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { ); } +/** + * True when the WHOLE of `selection` stays inside ONE attribute-display node — a collapsed caret in + * attribute context, or a range whose anchor and focus are the same node and that node is in + * attribute context. + * + * This is the only shape whose pasted bytes are genuinely attribute VALUE bytes. The value-byte + * carve-outs ({@link $insertPastedTextIntoAttributeContext}) are earned by exactly one fact: the + * bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` skips, so + * they are never read as markers and never re-tokenize. + * + * A selection that merely TOUCHES attribute context does not have that property. Removing the + * range takes the bytes out of the run, and when the range covers a char span's closing glyph it + * deletes the closer too, so the whole span re-tokenizes around whatever just arrived. Measured + * both ways round — a range starting in body text and reaching INTO the run, and a range starting + * inside the run and reaching PAST the closer — a pasted `\c 5` produced a real chapter node + * mid-paragraph, which is exactly the save-loop poisoning {@link $stripPastedChapterAndBookId} + * exists to prevent (`attributeContextPasteFidelity.test.tsx`). + */ +function $isSelectionWithinOneAttributeNode(selection: RangeSelection): boolean { + const { anchor, focus } = selection; + return anchor.key === focus.key && $isNodeInAttributeContext(anchor.getNode()); +} + /** * Paste normalization for a selection touching attribute-display text — the binding design * principle: paste ≡ typing the same characters at the same caret. Each `\n` in `text` becomes a @@ -325,21 +348,31 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { * two spaces, matching what two individual Enter-less keystrokes over an attribute run would * produce — attribute values are single-line, so there is no multi-line attribute byte shape to * collapse INTO). `text` arrives with bare `\n` line endings ({@link getPastePayload} normalizes - * them for every paste claim) — no `\r` reaches this function. The result is inserted via the exact - * `selection.insertText` a keystroke uses — no chapter/book-id strip, no positional NBSP mapping, - * no marker tokenization, all of which exist for BODY content's marker-recognition and - * whitespace-display invariants and apply to none of it: a pasted `\c 5` or `\p` here is literal - * value text, not a structural marker, and + * them for every paste claim) — no `\r` reaches this function. For a non-collapsed (mixed or + * fully-inside) selection, `selection.insertText` removes the selected range before inserting, the + * same as it does for a typed keystroke over that selection — no separate removal step is needed + * here. The existing attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged + * early return, `$resolvePendingMarkers`) takes over identically whether the text arrived by typing + * or this call. + * + * The two BODY-content paste rules — the `\c`/`\id` strip and the positional NBSP mapping, the only + * two places a paste deliberately diverges from typing the same bytes — are skipped only for a + * selection that stays wholly inside one attribute node ({@link $isSelectionWithinOneAttributeNode}). + * There the bytes really are value text: a pasted `\c 5` is literal, not a structural marker, and * `$displayWhitespaceTransform` (this file) already skips textType "attribute" nodes outright, so a - * literal NBSP a user TYPES into an attribute run is never touched — a pasted one must not be - * treated any differently. For a non-collapsed (mixed or fully-inside) selection, - * `selection.insertText` removes the selected range before inserting, the same as it does for a - * typed keystroke over that selection — no separate removal step is needed here. The existing - * attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, - * `$resolvePendingMarkers`) takes over identically whether the text arrived by typing or this call. + * literal NBSP a user TYPES into an attribute run is never touched and a pasted one must not be + * either. A selection that only TOUCHES an attribute run does not qualify: its bytes end up as + * ordinary paragraph content that re-tokenizes, so they get body content's rules. What does NOT + * change with them is the insertion MECHANISM — one `insertText`, never the paragraph-splitting + * line replay and never Lexical's rich-paste node insertion, both of which corrupt the + * attribute-run end of the range. A multi-line payload landing in body content this way therefore + * still collapses per newline instead of splitting the paragraph; splitting a paragraph the removal + * has just cut a char span in half in is the corruption this path exists to avoid. */ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: string): void { - selection.insertText(text.replace(/\n/g, " ")); + const valueBytes = $isSelectionWithinOneAttributeNode(selection); + const resolved = valueBytes ? text : $normalizePastedNbsp($stripPastedChapterAndBookId(text)); + selection.insertText(resolved.replace(/\n/g, " ")); } /** @@ -400,9 +433,11 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * documented case for when that CAN still reach a handler, unlike the reconstructed-`DataTransfer` * path); (2) a multi-line plain-text payload, which the ordinary external-paste pipeline below * would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND` — inside an attribute run that - * is exactly as destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`), which - * the ordinary pipeline's `$stripPastedChapterAndBookId` would eat bytes out of an attribute VALUE - * that were never a chapter token to begin with; (4) a MIXED selection (one end inside the run, one + * is exactly as destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`) landing + * wholly INSIDE one attribute node, where the ordinary pipeline's `$stripPastedChapterAndBookId` + * would eat bytes out of an attribute VALUE that were never a chapter token to begin with — bytes + * merely TOUCHING a run get that strip, since they end up as re-tokenizing paragraph content + * ({@link $insertPastedTextIntoAttributeContext}); (4) a MIXED selection (one end inside the run, one * end outside) combined with either (1) or (2) above — the selection touches attribute context, so * it must not be allowed to reach either risky branch merely because its OTHER end sits outside. * Attribute value bytes are never rich content — a user cannot "type formatting" into one either — From fb0be85d499afe657d6bd973a2ae67c40a94df54 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 16:08:25 -0500 Subject: [PATCH 15/44] fix: sweep the real-world 2SA fixture, closing two copy-side attribute losses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred item 7 excluded `2sa.usj.ts` from the copy→paste corpus sweep because it did not round-trip clean when last measured. Re-measured against the same harness: three classes of divergence remained, two of them clipboard bugs of the family this round has been closing. A collapsed note's `category` was dropped. `createNote` builds the `\cat` display run only for the editable EXPANDED layout — a deliberate view decision — so a collapsed note has no category bytes on screen for the copy walker to pick up. Same shape as the note caller, which renders as an empty glyph and is recovered by the walker for exactly this reason, so the category is recovered the same way, placed where the file has it (`\f + \cat People\cat*\fr …`, no trailing space: a space there re-tokenizes into a stray text child inside the note). A spanning table cell's `colspan` was dropped. USFM tables have no pipe attributes at all — a cell's width lives in its marker NAME (`\thc3-4`), which the tokenizer trims off into `colspan` on the way in — and the display rendered the bare marker, so the editor showed, copied, and re-tokenized a one-column cell. `tableCellMarkerWithSpan` already existed for the `UnknownNode` byte builder, whose table branch went dead when cells became `ImmutableTable*` nodes; it is now exported and shared by both. The committed 2SA Lexical fixtures are regenerated: six lines, the three spanning cells in editable and visible modes. The third class is inherent and stays: a verse's `sid` is derived from book + chapter + verse when ParatextData produces USX, and USFM has no bytes for it anywhere. Measured rather than assumed — a plain load→save of this fixture keeps all 26 of them, so the loss belongs to the carrier, not the adaptors. With those two fixed, 2SA round-trips to full USJ equality modulo the 26 verse sids, so it joins the sweep as a running case rather than a skip: skipping would trade 143 items of real-world coverage for one attribute. The carve-out is asserted on both sides (the source really carries them, the paste carries none) so it can neither be vacuous nor outlive the gap, and the re-stamped document USJ version is asserted rather than waived. The whole sweep file, 2SA included, runs in about a second. Two gaps the old exclusion reason listed are already closed and were confirmed so: the empty `\b` blank-line paragraph and the `\ref` target both round-trip. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 51 ++++-- .../src/nodes/features/unknownUsfm.utils.ts | 6 +- .../src/data/2sa.lexical.editable.ts | 6 +- .../test-data/src/data/2sa.lexical.visible.ts | 6 +- .../src/editor/adaptors/usj-editor.adaptor.ts | 9 +- .../markerEdit/clipboardCopyFidelity.test.tsx | 83 ++++++++++ .../clipboardCorpusRoundTrip.test.tsx | 154 ++++++++++++------ .../whitespaceDisplay.plugin.utils.ts | 31 +++- 8 files changed, 269 insertions(+), 77 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 1c6696d39..b10646f3f 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -20,6 +20,8 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Byte fidelity around note-internal markers:** Copied text must not contain spaces the source USFM lacks. Live repro (2026-08-07): copying `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*Den` produced `… \xo* \xt 2Cor 4:6\xt* \x*Den` — phantom spaces after each closing marker (display-separator NBSPs leaking through the blanket NBSP→space mapping). The copy walker must be source-faithful, mirroring the serialization inverse, not a blanket `replaceAll`. +- **An attribute the display does not show is still copied, whenever the file has bytes for it:** the same rule the note-caller bullet above states, applied wherever the display and the file disagree. A COLLAPSED note deliberately renders no `\cat` run (`createNote`, `usj-editor.adaptor.ts` — the category displays only in the editable EXPANDED layout, mirroring `\va`/`\vp`), so the copy walker contributes the file's own `\cat …\cat*` span itself, placed where the file has it (`\f + \cat People\cat*\fr 1:1 …`, with no trailing space — a space there re-tokenizes into a stray text child inside the note). A spanning TABLE CELL was a display gap rather than a view decision: USFM tables carry no pipe attributes at all, so a cell's width lives in its MARKER NAME (`\thc3-4`) and the tokenizer trims it off into `colspan` on the way in — the cell's display glyph now re-encodes it (`tableCellMarkerWithSpan`, shared with the `UnknownNode` byte builder), so what is shown, copied, and re-tokenized is the cell the file describes rather than a one-column cell. Both were found by sweeping the repo's real-world fixture (`2sa.usj.ts`) through the copy→paste round trip; both are pinned on the copied bytes in `clipboardCopyFidelity.test.tsx`. + - **Multi-block selections:** One `\n` between blocks; a selection starting mid-paragraph omits that paragraph's own `\p ` glyph (matches P9's rendered-text copy). Pinned, not changed. - **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection *before* calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it *likelier*: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers, both of which decide INSIDE the update where the selection is authoritative — a check in front of the dispatch would read the last COMMITTED selection, which Lexical is a microtask behind, and would silently no-op a `copy()` made in the same tick as the selection it is copying. (1) `registerEmptyCopyGuard` (`clipboard.utils.ts`, shared-react), a `COPY_COMMAND`/`CUT_COMMAND` listener `ClipboardPlugin` registers at `COMMAND_PRIORITY_LOW` — above `@lexical/rich-text`'s EDITOR fallback, which is what synthesizes the event, and below every feature handler. One registration covers every leg (the keyboard shortcuts, the context menu's Cut/Copy, `EditorRef.copy()`/`.cut()`), since all of them dispatch the same command against the same editor. (2) `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into that fallback. A real clipboard event is still declined by layer 2 at a collapsed caret (the browser already writes nothing, and declining leaves it un-`preventDefault`ed). A NodeSelection is claimed by neither layer — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it. Such a selection — measured at the Lexical level as two ELEMENT points on the parent paragraph either side of the block — reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. **Unverified assumption:** that a real drag or click over a `contentEditable=false` block is what a browser resolves INTO that selection shape (on the reasoning that it will not place a selection endpoint inside one). jsdom cannot observe DOM-range → Lexical selection resolution — the same boundary S6 already records for the WI-2 quirk — so this needs a real-browser check before it is treated as fact. If a browser instead leaves the caret collapsed beside the block, the leak is still closed (nothing is written) but the user gets nothing rather than the figure, which would be a selection-affordance gap to file separately. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, including the node-selection carve-out, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view where no Standard-view handler is registered at all, and through `EditorRef.copy()`/`.cut()`), and shared-react's `ClipboardPlugin.test.tsx` (the shortcuts, and the same-tick select-then-copy case) and `ContextMenuPlugin.test.tsx` (the menu leg). @@ -34,7 +36,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio `UnknownNode.excludeFromCopy` returned `destination !== "clone"`, excluding every `UnknownNode` unconditionally from the lexical-JSON flavor. An excluded node is not dropped silently — `$appendNodesToJSON` hoists its own children into the parent's list in its place. For an optbreak, whose only child is the `//` `ImmutableTypedTextNode` display token (a content-free `DecoratorNode` with no meaning once separated from its owning `UnknownNode`), that stranded a loose decorator on paste: `$parseSerializedNode` reconstructed the bare decorator, not a recognized optbreak, and nothing re-tokenizes a decorator's text — the discretionary line break silently vanished. Live report (TJ, 2026-08-13, most likely a pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the clipboard correctly, but pasting that same clipboard back did not restore it. -Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 11 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. +Fixed by narrowing `UnknownNode.excludeFromCopy` (`UnknownNode.ts`) to leave a CHILD-BEARING "optbreak" out of the exclusion (a genuinely childless live optbreak — an emptied husk mid-settle — still falls back to excluded, matching `text/plain`'s "nothing here" for that shape), plus an `isSelected` override closing a SECOND, independent carrier-disagreement gap measured while verifying the fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the construct at offset 0 (touching the wrapper without covering any of its content) still marked a CHILD-BEARING node "selected" under Lexical's default `isSelected`, producing an entry in the copy that `text/plain` has nothing corresponding to — `excludeFromCopy` alone cannot see this (it has no visibility into which children a given selection will include), so the override closes it at its actual source, mirroring `$selectionToUsfmText`'s own `getNodes()`-based walk so both carriers agree. The override tests child MEMBERSHIP in `getNodes()`, deliberately, and NOT the narrower "does the selection cover a child's content" — the two differ for exactly one shape, and the narrower rule is the worse of the two. That boundary reaches the children two different ways depending on what the first child IS: a construct whose display bytes lead (a figure's `\fig ` glyph, an optbreak's `//`) starts with a DECORATOR, so the element point stays one, no child is in `getNodes()`, and both carriers agree; a construct with no display bytes (a `ref`, whose container USFM never carried) starts with a real `TextNode`, and Lexical normalizes the same point into a TEXT point at that child's offset 0 — the child is then in `getNodes()` contributing zero characters, so `text/plain` emits nothing while this predicate still answers true. Answering false there was measured, and it is a SILENT STRUCTURAL LOSS rather than the agreement it looks like: excluding the wrapper makes `$appendNodesToJSON` hoist its children in its place, `createUnknown` stamps `mode:"token"` on every text child, and `$sliceSelectedTextNodeContent` refuses to slice a token node — so the copy carries the construct's characters with the node and its attributes gone (`[verse, "See ", {"type":"text","mode":"token","text":"Genesis 1:1"}]`, no `ref`, no `loc`), which is precisely the convincing-lie hazard this pair exists to prevent. Membership keeps the construct whole (`[verse, "See ", {"type":"unknown","tag":"ref","unknownAttributes":{"loc":"GEN 1:1"},"children":["Genesis 1:1"]}]`), making the lexical flavor a SUPERSET of `text/plain` at that one boundary; the residual disagreement is recorded as Deferred item 10 rather than traded for a loss. Closing it properly needs a lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so a node cannot say "drop me AND my children". Both rules key on the node's own CHILD COUNT rather than on its kind, so they cover every `UnknownNode` construct alike — the kinds share one display-decorator shape (`unknownDisplayParts` builds each kind's marker/attribute bytes as the same content-free `ImmutableTypedTextNode`s), and so shared one hoisting failure. Pinned end-to-end for optbreak (all three real-world payload shapes: plain-only, plain+html without the lexical flavor, and the full sync/native payload carrying it; plus both childless-shape edges — a genuine husk, and the boundary-selection case) in `optbreakClipboardFidelity.test.tsx`, per kind for figure/sidebar/periph/ref across the same three shapes in `unknownClipboardFidelity.test.tsx` — whose own `"a construct's own start boundary"` describe pins the `isSelected` boundary for a decorator-led kind (`figure`: excluded from both carriers) and a text-led one (`ref`: carried WHOLE in the lexical flavor, node and `loc` attribute included), plus both kinds with the selection extended over their first child, rather than leaving the shared predicate inferred from optbreak alone. Every one of those assertions reads the construct's own CONTENT BYTES, never just its tag — a tag-only assertion cannot see a copy that dropped the wrapper while hoisting its characters out, which is exactly how the lossy variant passed review once — plus direct unit pins on `excludeFromCopy` in `UnknownNode.test.ts`. Tables are NOT part of this: `table`/`table:row`/`table:cell` are their own `ImmutableTable*` nodes now, never `UnknownNode`s, so they never took the exclusion at all — measured, and pinned in the same per-kind sweep. Otherwise unchanged, EXCEPT: a paste whose selection touches an attribute display run (a char span's `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value) never takes this path, even when @@ -141,13 +143,18 @@ These differences are documented and intentional; they are not implemented as wo ## Known Lossy Constructs — Copy→Paste Round Trip Found by a corpus-style sweep (`clipboardCorpusRoundTrip.test.tsx`): for every fixture in the -shared USJ round-trip corpus (`corpus-data.ts`), select the chapter's content, copy, paste into a -fresh editor holding the same chapter header, and compare the resulting USJ to the source. Two of -the remaining failures are INHERENT — the plain-text `text/plain` carrier (S3) has no bytes capable -of representing the construct, so no paste-side fix is possible without a different carrier. One is -ACCEPTED normalization that matches Paratext 9's own behavior, not a bug at all. Every one of them stays in the sweep as `it.skip`, not -deleted, so a future engine change un-skips it automatically instead of the gap going unnoticed — -which is how the sidebar, table and milestone entries that used to sit here left the list. +shared USJ round-trip corpus (`corpus-data.ts`) and for the repo's one real-world fixture +(`2sa.usj.ts`), select the chapter's content, copy, paste into a fresh editor holding the same +chapter header, and compare the resulting USJ to the source. Three of the remaining failures are +INHERENT — the plain-text `text/plain` carrier (S3) has no bytes capable of representing the +construct or the attribute, so no paste-side fix is possible without a different carrier. One is +ACCEPTED normalization that matches Paratext 9's own behavior, not a bug at all. Items 1–2 and 4 +stay in the sweep as `it.skip`, not deleted, so a future engine change un-skips one automatically +instead of the gap going unnoticed — which is how the sidebar, table, milestone and `\periph` +entries that used to sit here left the list. Item 3 is not a skip: it is a single attribute inside a +fixture that otherwise sweeps clean, so the sweep asserts it as a NAMED carve-out — both its +presence in the source and its absence after the paste — rather than skipping 143 items' worth of +coverage over it. 1. **Cross-reference `` target wrapper (inherent):** USJ's `ref` element is a wrapper USFM itself never carried (`unknownUsfm.utils.ts`'s own doc comment: "USJ invented this container, @@ -169,7 +176,17 @@ which is how the sidebar, table and milestone entries that used to sit here left confirming the ambiguity is specifically about trailing content after an implicit close, not `closed="false"` itself. -3. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a +3. **A verse's derived `sid` (inherent):** USJ/USX carry `sid="2SA 1:1"` on a verse; USFM carries + `\v 1` and nothing else — the identifier is DERIVED from book + chapter + verse when + ParatextData produces USX, so there are no bytes anywhere for the plain-text carrier to copy and + nothing for paste to rebuild one from. Measured rather than assumed: a plain load→save of the + 2SA fixture keeps all 26 of its verse sids, so the loss belongs to the carrier and not to the + adaptors. The CHAPTER's `sid` survives the same round trip only because the sweep seeds the + target editor with the chapter header. Asserted by name in the sweep's + `"real-world multi-construct fixture (2sa)"` describe, on both sides (the source really carries + them; the paste carries none), so the carve-out can neither be vacuous nor outlive the gap. + +4. **Paragraph-leading space swallowed on paste (ACCEPTED normalization — matches Paratext 9, not a bug):** isolated with a minimal non-corpus repro: pasting the literal text `"\p X"` (marker, its own required separator, and a SECOND, real content-leading space) into a fresh empty `"\p"` host produces `"\p X"` — one space, not two. The mechanism is `consumeSeparator()` @@ -203,22 +220,20 @@ The following items are recorded as deferred and not addressed in this plan: 6. **Popover expanded-note `isStandardView` gating issue:** An issue with note-expansion behavior in popovers has an open tracking document at `docs/superpowers/specs/2026-07-07-standard-view-followups.md`. This is not addressed in the current plan. -7. **2SA corpus (`libs/test-data/src/data/2sa.usj.ts`) not included in the copy→paste corpus sweep:** this fixture IS single-chapter (one top-level chapter object), so the earlier "multi-chapter/book-level" exclusion reason recorded in an earlier draft of this work was wrong. It was tried against the sweep's harness directly and did not round-trip clean: it hit the sidebar gap three times — since closed — plus several additional, unrelated fidelity gaps (an empty `\b` blank-line paragraph, a verse's derived `sid` attribute, a `\ref` target) that this sweep's one-construct-per-fixture design has no clean way to itemize as a single byte diff. It has not been re-measured since the sidebar gap closed, so how much of the residual survives is unknown; re-running it against the harness is the first step whenever this is picked up. `tier2Rebuild.corpus.test.tsx` already exercises this fixture, but for a narrower, different property (an unedited `$rebuildParas` call refusing as a fixed point) that does not cover the paste path this sweep does — so the exclusion is a real coverage gap, not a redundant re-test. Recorded here rather than force-fit into the sweep. - -8. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. +7. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. -9. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. +8. **A foreign clipboard that kept an NBSP only in `text/html` loses it (accepted trade-off in the paste carrier choice):** `getPastePayload` resolves a paste to `text/plain` whenever the clipboard carries any, falling back to the decoded `text/html` only when it does not. A stronger rule is possible and genuinely better for FOREIGN sources — choose the carrier the NBSP survived in (`plainText.includes(NBSP) ? plainText : htmlText`) — because some sources collapse ` ` to a plain space in their `text/plain` while keeping the real NBSP in their `text/html`; with the presence rule, `{"text/plain": "3 000", "text/html": "

3 000

"}` inserts `3 000` where the stronger rule would insert `3~000`. It is not usable here, because it inverts on P10's OWN copy: Standard view's `text/plain` deliberately carries no NBSP at all (display NBSPs invert to plain spaces; a genuine data NBSP displays and copies as `~`), while its `text/html` still ships NBSPs wherever a plain space would not survive a rich-text consumer (a run of 2+, a fragment edge) — so "the plain text has no NBSP" is true of EVERY P10 copy, and an NBSP-presence test would route P10's own round trip through the html carrier, whose decoded text drops a collapsed note's caller entirely (accepted asymmetry #3 above: the caller rides as a `data-caller` attribute, never as text). Losing a note caller on the editor's own copy is both worse and far likelier than losing a foreign clipboard's data-NBSP, so the presence rule stands and the foreign-source loss is accepted. Closing it properly means a per-carrier reconciliation (take the plain text's words and the html's NBSP positions) rather than a choice between the two — more machinery than the gap justifies today. -10. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. +9. **A hidden-marker view's copy drops a CONTENT-LESS opaque construct entirely:** `UnknownNode.excludeFromCopy` keeps a construct in the `application/x-lexical-editor` flavor only while it has children of its own, which is exactly when `text/plain` has bytes for it — the two carriers agreeing is the point of the rule. In an editable-marker view a construct's own marker and attribute display bytes ARE children, so only a genuinely emptied husk is childless. In a HIDDEN-marker view `createUnknown` (`usj-editor.adaptor.ts`) builds no display children at all, so a construct with no content — a caption-less `figure`, an empty `ref`, every optbreak — has zero children and is dropped from both carriers; an internal (lexical-flavor) copy→paste inside such a view therefore loses the node and every attribute on it. This is consistent with S5 (a hidden-marker view copies the prose it shows, and it shows nothing for a construct with no content) rather than a regression in the copy rule, and closing it would mean deciding that a marker-free view copies marker-only constructs anyway — a view-semantics question, not a clipboard one. Recorded rather than guessed at. -11. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. +10. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. ## Test Mapping (S1–S7) | Semantic | Pinning Test | |----------|--------------| -| **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"` describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | +| **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"`, `"attributes a construct carries in bytes the display had to reconstruct"` (a collapsed note's `\cat` run; a spanning table cell's span suffix) describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | | **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"` and `"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"` (external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction); `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | | **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | | **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (confirmed regression class on this branch, not TJ's literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | @@ -227,9 +242,9 @@ The following items are recorded as deferred and not addressed in this plan: | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | | **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | | **S3 (peripheral divisions)** `\periph` re-tokenizes back into a `periph` division — its title, its attribute list, and the blocks it contains | `libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts` — `"usfmFragmentToUsjContent — peripheral divisions (\\periph)"` describe: the marker line split into `alt` plus attributes, the line-per-marker spelling reading identically, following blocks taken into the division, one division ending at the next, a chapter-free contentless division, a nested sidebar, the unparseable-attribute degrade to a literal paragraph, and note context opening no division at all | -| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped | +| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus AND the repo's real-world fixture | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped. Its `"real-world multi-construct fixture (2sa)"` describe sweeps `libs/test-data/src/data/2sa.usj.ts` (143 top-level items, every supported construct in the shapes a real project produced) to full USJ equality, with the one inherent verse-`sid` loss asserted by name on both sides. The whole file runs in about a second | | **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 11), asserting content BYTES rather than tags | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 10), asserting content BYTES rather than tags | | **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | | **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | | **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | diff --git a/libs/shared/src/nodes/features/unknownUsfm.utils.ts b/libs/shared/src/nodes/features/unknownUsfm.utils.ts index 8880f4203..c275ac511 100644 --- a/libs/shared/src/nodes/features/unknownUsfm.utils.ts +++ b/libs/shared/src/nodes/features/unknownUsfm.utils.ts @@ -128,7 +128,9 @@ function renameFigureFileToSrc(attributes: UnknownAttributes): UnknownAttributes } /** - * The cell's opening marker with its span suffix re-encoded from `colspan`. The tokenizer + * The cell's opening marker with its span suffix re-encoded from `colspan`. Exported because a + * cell's display bytes are built by `ImmutableTableCellNode`'s own adaptor path (the kind stopped + * being an `UnknownNode`), and both builders must spell a spanning cell the same way. The tokenizer * (usfmFragmentToUsj.ts, table-cell assembly) splits a spanning cell marker apart on the way in — * `\thc3-4` becomes marker `thc3` (span suffix trimmed off after the start column) plus colspan * `"2"` (the COUNT of columns spanned, end − start + 1) — so rendering the stored marker bare @@ -137,7 +139,7 @@ function renameFigureFileToSrc(attributes: UnknownAttributes): UnknownAttributes * marker with no trailing start column to count from yields the bare marker rather than a garbage * suffix. */ -function tableCellMarkerWithSpan( +export function tableCellMarkerWithSpan( marker: string | undefined, colspan: string | undefined, ): string | undefined { diff --git a/libs/test-data/src/data/2sa.lexical.editable.ts b/libs/test-data/src/data/2sa.lexical.editable.ts index 09c65c184..41afcf710 100644 --- a/libs/test-data/src/data/2sa.lexical.editable.ts +++ b/libs/test-data/src/data/2sa.lexical.editable.ts @@ -9695,7 +9695,7 @@ export const lexicalEditable2Sa: SerializedEditorState = { children: [ { type: "marker", - marker: "thc3", + marker: "thc3-4", markerSyntax: "opening", text: "", detail: 0, @@ -9944,7 +9944,7 @@ export const lexicalEditable2Sa: SerializedEditorState = { children: [ { type: "marker", - marker: "thr4", + marker: "thr4-5", markerSyntax: "opening", text: "", detail: 0, @@ -10021,7 +10021,7 @@ export const lexicalEditable2Sa: SerializedEditorState = { children: [ { type: "marker", - marker: "tcr1", + marker: "tcr1-4", markerSyntax: "opening", text: "", detail: 0, diff --git a/libs/test-data/src/data/2sa.lexical.visible.ts b/libs/test-data/src/data/2sa.lexical.visible.ts index 3b68c856c..55c3148af 100644 --- a/libs/test-data/src/data/2sa.lexical.visible.ts +++ b/libs/test-data/src/data/2sa.lexical.visible.ts @@ -6393,7 +6393,7 @@ export const lexicalVisible2Sa: SerializedEditorState = { children: [ { type: "immutable-typed-text", - text: "\\thc3 ", + text: "\\thc3-4 ", textType: "marker", version: 1, }, @@ -6540,7 +6540,7 @@ export const lexicalVisible2Sa: SerializedEditorState = { children: [ { type: "immutable-typed-text", - text: "\\thr4 ", + text: "\\thr4-5 ", textType: "marker", version: 1, }, @@ -6583,7 +6583,7 @@ export const lexicalVisible2Sa: SerializedEditorState = { children: [ { type: "immutable-typed-text", - text: "\\tcr1 ", + text: "\\tcr1-4 ", textType: "marker", version: 1, }, diff --git a/packages/platform/src/editor/adaptors/usj-editor.adaptor.ts b/packages/platform/src/editor/adaptors/usj-editor.adaptor.ts index 3d82d0870..ca6d4c741 100644 --- a/packages/platform/src/editor/adaptors/usj-editor.adaptor.ts +++ b/packages/platform/src/editor/adaptors/usj-editor.adaptor.ts @@ -121,6 +121,7 @@ import { UNKNOWN_MARKER_OBJECT_PROPS, UNKNOWN_VERSION, UnknownAttributes, + tableCellMarkerWithSpan, unknownDisplayParts, UnknownNode, unmatchedGlyphText, @@ -633,16 +634,20 @@ function createTableCell( const { marker, align, colspan } = markerObject as ImmutableTableCellMarker; const children: SerializedLexicalNode[] = []; const cellMarker = marker ?? TABLE_CELL_DEFAULT_MARKER; + // USFM tables carry no pipe attributes at all: a spanning cell's width IS part of its marker + // name (`\thc3-4`), which the tokenizer trimmed off into `colspan` on the way in. Displaying + // the bare marker would show — and copy, and re-tokenize as — a one-column cell. + const displayMarker = tableCellMarkerWithSpan(cellMarker, colspan) ?? cellMarker; if (_viewOptions?.markerMode === "editable") children.push( - createMarker(cellMarker), + createMarker(displayMarker), createText(NBSP, MARKER_TRAILING_SPACE_TEXT_TYPE, "token"), ); else if (_viewOptions?.markerMode === "visible" || _viewOptions?.hasGutterParaMarkers) children.push( createImmutableTypedText( "marker", - openingMarkerText(cellMarker) + NBSP, + openingMarkerText(displayMarker) + NBSP, _viewOptions?.hasGutterParaMarkers, ), ); diff --git a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx index 075a87dbd..389860beb 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx @@ -240,6 +240,89 @@ describe("phantom-space live-repro pins (2026-08-07) — collapsed note, byte-id }); }); +describe("attributes a construct carries in bytes the display had to reconstruct", () => { + /** A collapsed footnote carrying a `category`. In the file the category is a `\cat` span on the + * note's own marker line, directly after the caller (`\f + \cat People\cat*\fr …`); a COLLAPSED + * note deliberately does not DISPLAY it (`createNote`, usj-editor.adaptor.ts), which is a view + * decision, not a licence for the copy to drop the attribute. */ + function categorizedNoteUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "para", + marker: "p", + content: [ + { + type: "note", + marker: "f", + caller: "-", + category: "People", + content: [ + { type: "char", marker: "fr", content: ["1:1 "], closed: "false" }, + { type: "char", marker: "ft", content: ["Caller test."], closed: "false" }, + ], + }, + ], + }, + ], + } as unknown as Usj; + } + + /** A one-row table whose first cell spans two columns. USFM tables have no attribute syntax at + * all: a cell's width lives in its MARKER NAME (`\thc3-4`), which the tokenizer splits into + * marker `thc3` + `colspan` "2" on the way in. */ + function spanningCellTableUsj(): Usj { + return { + type: "USJ", + version: "3.1", + content: [ + { + type: "table", + content: [ + { + type: "table:row", + marker: "tr", + content: [ + { + type: "table:cell", + marker: "thc3", + align: "center", + colspan: "2", + content: ["wide"], + }, + { type: "table:cell", marker: "th5", align: "start", content: ["last"] }, + ], + }, + ], + }, + ], + } as unknown as Usj; + } + + it("copies a collapsed note's category as the `\\cat` span the file carries", async () => { + // Same rule as the note caller above: what you copy > what you see. The category has no bytes + // on screen in a collapsed note, so the walker contributes them, exactly as it contributes the + // caller a collapsed note renders as an empty glyph. + const { editor } = await renderUsjEditor(categorizedNoteUsj()); + await act(async () => editor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe( + "\\p \\f - \\cat People\\cat*\\fr 1:1 \\ft Caller test.\\f*", + ); + }); + + it("copies a spanning cell's width as the span suffix its marker name carries", async () => { + const { editor } = await renderUsjEditor(spanningCellTableUsj()); + await act(async () => editor.update($selectWholeDocument)); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe("\\tr \\thc3-4 wide\\th5 last"); + }); +}); + describe("multi-paragraph selections", () => { it("joins a full multi-paragraph selection with a single \\n, each paragraph keeping its own \\marker", async () => { let secondText: TextNode; diff --git a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx index 1b774b504..3ef437957 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx @@ -18,22 +18,16 @@ * `"periph"` is a named `it.skip` rather than a swept case: it is book-level front matter with no * chapter at all, so it does not fit "a single-chapter editor state" — there is no chapter-content * selection for it to exercise. Every other fixture is either swept clean or recorded in - * `KNOWN_LOSSY` below with the exact byte-level divergence — none are silently dropped. All 21 - * fixtures run in well under a second; no sampling is needed. + * `KNOWN_LOSSY` below with the exact byte-level divergence — none are silently dropped. * - * The OTHER USJ corpus in this repo, `libs/test-data/src/data/2sa.usj.ts` (141 paragraphs; IS - * single-chapter — one top-level chapter object, despite the size), is deliberately NOT included - * here. It was tried against this exact harness and did not round-trip clean: it hit the sidebar - * gap three times (2SA's own sidebar coverage) — since closed — PLUS several additional, unrelated - * fidelity gaps this sweep's one-construct-per-fixture design has no clean way to itemize as a - * single byte diff (an empty `\b` blank-line paragraph folds into literal text inside a `p` - * paragraph; a verse's derived `sid` attribute is dropped; a `\ref` cross-reference target degrades - * differently than the corpus's own dedicated `ref` fixture above). It has not been re-measured - * since the sidebar gap closed, so how much of that residual survives is unknown. `tier2Rebuild.corpus - * .test.tsx` already exercises this exact fixture, but for a narrower, DIFFERENT property (an - * unedited `$rebuildParas` call refusing as a fixed point) — that coverage does not include the - * paste path (NBSP normalization, `\c`/`\id` strip, own-marker-wins dedup) this sweep does, so - * excluding 2sa here is a real coverage gap, not a redundant re-test. + * The repo's OTHER USJ corpus, `libs/test-data/src/data/2sa.usj.ts`, is swept too — see the + * `"real-world multi-construct fixture (2sa)"` describe at the bottom of this file for what it adds + * and for the one attribute it cannot carry. `tier2Rebuild.corpus.test.tsx` already exercises that + * fixture, but for a narrower, DIFFERENT property (an unedited `$rebuildParas` call refusing as a + * fixed point) that does not cover the paste path (NBSP normalization, `\c`/`\id` strip, + * own-marker-wins dedup) this sweep does. + * + * The whole file, 2sa included, runs in about a second; no sampling is needed. */ import { MarkerEditPlugin } from "./MarkerEditPlugin"; import { @@ -51,9 +45,16 @@ import { act } from "@testing-library/react"; // Reaching inside only for tests (same pattern as markerEdit.test-helpers). // eslint-disable-next-line @nx/enforce-module-boundaries import { baseTestEnvironment } from "../../../../../libs/shared-react/src/plugins/usj/react-test.utils"; -import { MarkerObject, Usj, usxStringToUsj } from "@eten-tech-foundation/scripture-utilities"; +import { + MarkerContent, + MarkerObject, + Usj, + USJ_VERSION, + usxStringToUsj, +} from "@eten-tech-foundation/scripture-utilities"; import { $getRoot, COPY_COMMAND, PASTE_COMMAND, RootNode } from "lexical"; import { $isChapterNode, $isImmutableChapterNode } from "shared"; +import { usj2Sa } from "test-data"; /** The index of the top-level child right after the document's (LAST) chapter node — every corpus * fixture but `"periph"` has exactly one, but a document could in principle carry more than one @@ -159,6 +160,40 @@ const KNOWN_LOSSY: { name: string; reason: string }[] = [ }, ]; +/** Loads `usj` into a Standard-view editor, copies the chapter's own content, pastes the resulting + * `text/plain` into a fresh editor holding the same header, settles, and returns the USJ a host + * would save. */ +async function copyPasteRoundTrip(usj: Usj): Promise { + initializeDeserialize(undefined); + const { editor: sourceEditor } = await baseTestEnvironment( + serializedState(usj), + , + ); + await act(async () => sourceEditor.update(() => $selectChapterContent($getRoot()))); + const { event, getData } = copyEvent(); + await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, event)); + const copiedText = getData("text/plain"); + + const { editor: targetEditor } = await baseTestEnvironment( + serializedState(chapterHeaderSkeletonUsj(usj)), + , + ); + await act(async () => + targetEditor.update(() => { + $getRoot().getLastChild()?.selectEnd(); + targetEditor.dispatchCommand(PASTE_COMMAND, plainTextPasteEvent(copiedText)); + }), + ); + // Settle: flush any reconciliation the paste-triggered Tier 2 re-tokenization schedules beyond + // the synchronous update above (the same double-microtask flush every paste-adjacent suite in + // this directory uses). + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + return deserializeSerializedEditorState(targetEditor.getEditorState().toJSON(), viewOptions); +} + describe("corpus copy/paste round trip (Standard view)", () => { for (const fixture of corpusFixtures) { if (fixture.name === "periph") { @@ -174,41 +209,64 @@ describe("corpus copy/paste round trip (Standard view)", () => { const lossy = KNOWN_LOSSY.find((entry) => entry.name === fixture.name); const run = lossy ? it.skip : it; run(`${fixture.name}${lossy ? ` (${lossy.reason})` : ""}`, async () => { - initializeDeserialize(undefined); const usj = usxStringToUsj(fixture.usx); + expect(await copyPasteRoundTrip(usj)).toEqual(usj); + }); + } +}); - const { editor: sourceEditor } = await baseTestEnvironment( - serializedState(usj), - , - ); - await act(async () => sourceEditor.update(() => $selectChapterContent($getRoot()))); - const { event, getData } = copyEvent(); - await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, event)); - const copiedText = getData("text/plain"); - - const { editor: targetEditor } = await baseTestEnvironment( - serializedState(chapterHeaderSkeletonUsj(usj)), - , - ); - await act(async () => - targetEditor.update(() => { - $getRoot().getLastChild()?.selectEnd(); - targetEditor.dispatchCommand(PASTE_COMMAND, plainTextPasteEvent(copiedText)); - }), - ); - // Settle: flush any reconciliation the paste-triggered Tier 2 re-tokenization schedules - // beyond the synchronous update above (the same double-microtask flush every paste-adjacent - // suite in this directory uses). - await act(async () => { - await Promise.resolve(); - await Promise.resolve(); +/** + * The one real-world fixture in the repo — 143 top-level items, every construct the editor supports + * in one document, in the shapes an actual project produced rather than one-construct-per-fixture. + * It is swept here rather than omitted: the corpus fixtures above each isolate one construct, so + * only this one can catch a loss that needs two constructs in the same paragraph, or a construct in + * a shape nobody wrote a fixture for. It costs about a second. + * + * It round-trips byte-for-byte with ONE exception, asserted by name below rather than waived: a + * verse's `sid`. That attribute is DERIVED (book + chapter + verse) when ParatextData produces + * USX/USJ; USFM has no bytes for it anywhere, so the plain-text clipboard carrier (S3) cannot carry + * one and the paste cannot invent one. Measured, not assumed: a plain load→save of this same fixture + * keeps all 26 of them, so the loss belongs to the carrier and not to the adaptors. The chapter's + * own `sid` survives only because the target editor is SEEDED with the chapter header. Same family + * as the `` wrapper above — see "Known Lossy Constructs" in the semantics doc. + */ +describe("real-world multi-construct fixture (2sa)", () => { + /** Every `verse` object's `sid` removed, everywhere, leaving the document otherwise untouched. */ + function withoutVerseSids(usj: Usj): Usj { + const stripItems = (items: MarkerContent[] | undefined): MarkerContent[] | undefined => + items?.map((item) => { + if (typeof item === "string") return item; + const stripped: MarkerObject = { ...item }; + const content = stripItems(item.content); + if (content) stripped.content = content; + if (stripped.type === "verse") delete stripped.sid; + return stripped; }); + return { ...usj, content: stripItems(usj.content) ?? [] }; + } - const pastedUsj = deserializeSerializedEditorState( - targetEditor.getEditorState().toJSON(), - viewOptions, - ); - expect(pastedUsj).toEqual(usj); - }); + /** How many `verse` objects anywhere in `usj` still carry a `sid`. */ + function countVerseSids(usj: Usj | undefined): number { + const count = (items: MarkerContent[] | undefined): number => + (items ?? []).reduce((total, item) => { + if (typeof item === "string") return total; + return ( + total + (item.type === "verse" && item.sid !== undefined ? 1 : 0) + count(item.content) + ); + }, 0); + return count(usj?.content); } + + it("round-trips every construct in the document, losing only the verse sid USFM has no bytes for", async () => { + const pasted = await copyPasteRoundTrip(usj2Sa); + // Both counts guard the carve-out: without them the equality could pass because the source + // never had a sid, or keep passing unchanged on the day the loss is closed. + expect(countVerseSids(usj2Sa)).toBe(26); + expect(countVerseSids(pasted)).toBe(0); + // The editor re-stamps the document's USJ version on export, and a plain load→save of this + // same fixture does exactly the same (it declares 3.0), so it is not a clipboard property — + // asserted on its own rather than waived, then normalized out of the whole-document compare. + expect(pasted?.version).toBe(USJ_VERSION); + expect({ ...pasted, version: usj2Sa.version }).toEqual(withoutVerseSids(usj2Sa)); + }); }); diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 4b1c1c419..72e23df22 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -529,6 +529,33 @@ function $isNoteInternalDisplaySeparator(node: TextNode): boolean { return !$isImmutableNoteCallerNode(node.getPreviousSibling()); } +/** + * The `\cat …\cat*` bytes a COLLAPSED note's category needs, contributed by the one display node + * that sits where those bytes belong: the separator directly after the caller. `""` for every + * other node, and for a note with no category. + * + * A collapsed note deliberately displays no category at all — `createNote` (`usj-editor.adaptor.ts`) + * builds the `\cat` run only for the editable EXPANDED layout, mirroring `\va`/`\vp`. That is a + * view decision, and it is not a licence for the copy to drop the attribute: this is the same rule + * that already recovers a collapsed note's caller from an `ImmutableNoteCallerNode` that renders as + * an empty glyph — what you COPY is the note's bytes, not the note's pixels. + * + * Scoped to the collapsed layout by construction: only that layout puts an + * `ImmutableNoteCallerNode` before this separator (the expanded one uses a plain caller TextNode + * and carries the real `\cat` run among its children), so an expanded note can never double-emit. + * Placed AFTER the separator's own space and with none of its own, which is where the file has the + * span: `\f + \cat People\cat*\fr 1:1 …` — a trailing space here would re-tokenize into a stray + * text child inside the note. + */ +function $collapsedNoteCategoryBytes(node: TextNode): string { + if (node.getTextContent() !== NBSP) return ""; + const note = node.getParent(); + if (!$isNoteNode(note)) return ""; + if (!$isImmutableNoteCallerNode(node.getPreviousSibling())) return ""; + const category = note.getCategory(); + return category ? `\\cat ${category}\\cat*` : ""; +} + /** The nearest enclosing `NoteNode`'s USJ caller value, falling back to the auto-generated-caller * marker (`+`) when the note has none set. */ function $noteCallerText(callerNode: LexicalNode): string { @@ -619,7 +646,9 @@ export function $selectionToUsfmText(selection: RangeSelection): string { } else if (node === lastNode) { nodeText = isBefore ? nodeText.slice(0, focusOffset) : nodeText.slice(0, anchorOffset); } - text += $isNoteInternalDisplaySeparator(node) ? "" : nodeText.replaceAll(NBSP, " "); + text += $isNoteInternalDisplaySeparator(node) + ? "" + : nodeText.replaceAll(NBSP, " ") + $collapsedNoteCategoryBytes(node); } else if ( ($isDecoratorNode(node) || $isLineBreakNode(node)) && (node !== lastNode || !selection.isCollapsed()) From 11082da5be271d2a580bbb69e56dd3ebad71f7ef Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 16:58:43 -0500 Subject: [PATCH 16/44] fix: correct the attribute-value carve-out's rationale and close the periph title loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The carve-out is earned by paste-equals-typing, not by the bytes never re-tokenizing: the "attribute" tag survives insertion, but the caret-departure settle re-tokenizes the paragraph, so a pasted \c in a value does become a chapter marker — identically to typing it there. Both halves are now pinned settled rather than read before the settle, so the residual is visible instead of looking safe. Periph's parsed attributes were spread after its title, and the title attribute is not reserved, so a periph line carrying that attribute silently lost its title. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 2 +- .../converters/usfm/usfmFragmentToUsj.test.ts | 23 +++ .../src/converters/usfm/usfmFragmentToUsj.ts | 27 ++- .../attributeContextPasteFidelity.test.tsx | 177 ++++++++++++++---- .../markerEdit/clipboardCopyFidelity.test.tsx | 22 +++ .../whitespaceDisplay.plugin.utils.ts | 51 +++-- 6 files changed, 245 insertions(+), 57 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index b10646f3f..992f0c5f1 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -73,7 +73,7 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra - **Paste inside an attribute display run — paste ≡ typing:** a selection that TOUCHES attribute-display text at either end — a char span's bare `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` value — is inserted exactly as the SAME characters typed at the SAME caret (or over the same selection) would be: one `selection.insertText`, with each `\n` becoming a single space, per-newline, not run-collapsed (attribute values are single-line; there is no multi-line attribute byte shape to collapse INTO — `"a\n\nb"` pastes as `"a b"`, two spaces). A selection that only PARTLY sits inside attribute-display text (one end in, one end out) takes this same INSERTION MECHANISM — see below for the two corruption shapes that earns it. The existing attribute pend/settle machinery (`$textNodeTier2Transform`'s attribute-tagged early return, `$resolvePendingMarkers`) then re-tokenizes the displayed bytes back into node state identically whether they arrived by typing or paste. - **The two BODY-content paste rules are skipped only for a selection wholly inside ONE attribute node** — the `\c`/`\id` strip and the positional NBSP mapping, which are the only two places a paste deliberately diverges from typing the same bytes. That carve-out is earned by exactly one fact: the bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` skips, so a pasted `\c 5` or `\p` there really is literal value text and never a structural marker, and a pasted NBSP passes through unchanged (matching `$displayWhitespaceTransform`'s own skip of attribute-tagged text). A selection that merely TOUCHES an attribute run does not have that property, and pretending it does was a real data-fidelity hole: removing such a range takes the bytes out of the run — and when the range covers a char span's closing glyph it deletes the closer, so the whole span re-tokenizes around whatever just arrived. Measured both ways round (a range starting in body text and reaching INTO the run, and one starting inside the run and reaching PAST the closer), pasting `\c 5` over such a selection produced a real chapter node mid-paragraph with the rest of the paragraph stranded outside it — the exact save-loop poisoning the strip exists to prevent — and a pasted interior NBSP was read as display whitespace and saved as a plain space instead of surviving as data. So a touching-but-not-wholly-inside selection gets body content's rules for the bytes, while keeping the attribute path's insertion mechanism. One deliberate residual comes with that split: a MULTI-LINE payload landing in body content this way still collapses per newline rather than splitting the paragraph, because splitting a paragraph whose char span the removal has just cut in half is the corruption this path exists to avoid. + **The two BODY-content paste rules are skipped only for a selection wholly inside ONE attribute node** — the `\c`/`\id` strip and the positional NBSP mapping, which are the only two places a paste deliberately diverges from typing the same bytes. What earns that carve-out is paste ≡ TYPING: the bytes land where the same keystrokes would land, and the attribute-context contract is that a paste in a value behaves as typing there does. It is NOT that the bytes never re-tokenize — measured, that is false. The `"attribute"` tag survives the insertion, but the caret-departure settle re-tokenizes the paragraph, so a pasted `\c 5` in a value DOES become a chapter marker, and a pasted interior NBSP is read as display whitespace and saved as a plain space. Both happen IDENTICALLY when the same bytes are TYPED at the same caret, which is why this is the typed-`\c` hole recorded as Deferred item 2 — inherited by the paste path, not introduced by it. Extending the strip to cover it would eat bytes out of an attribute value that were never a chapter token — the very regression the carve-out exists to prevent — and would break the equivalence. Both halves are pinned SETTLED, on concrete USJ, in `attributeContextPasteFidelity.test.tsx`. A selection that merely TOUCHES an attribute run does not have that property, and pretending it does was a real data-fidelity hole: removing such a range takes the bytes out of the run — and when the range covers a char span's closing glyph it deletes the closer, so the whole span re-tokenizes around whatever just arrived. Measured both ways round (a range starting in body text and reaching INTO the run, and one starting inside the run and reaching PAST the closer), pasting `\c 5` over such a selection produced a real chapter node mid-paragraph with the rest of the paragraph stranded outside it — the exact save-loop poisoning the strip exists to prevent — and a pasted interior NBSP was read as display whitespace and saved as a plain space instead of surviving as data. So a touching-but-not-wholly-inside selection gets body content's rules for the bytes, while keeping the attribute path's insertion mechanism. One deliberate residual comes with that split: a MULTI-LINE payload landing in body content this way still collapses per newline rather than splitting the paragraph, because splitting a paragraph whose char span the removal has just cut in half is the corruption this path exists to avoid. TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the saved file diverged from the editor. `sid="things"` carries no NBSP; the pre-branch build's paste handler still had its OLD NBSP-gated form (see the "positional NBSP normalization" bullet above — that gate was generalized to claim every external paste, NBSP or not, on 2026-08-07), so the most plausible mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion — NOT a same-namespace `application/x-lexical-editor` flavor on the clipboard (this document's own "private Lexical flavor is dead on Ctrl+V" fact, above, still holds for the reconstructed-`DataTransfer` paste path this repro most likely used). diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts index e84d8f211..ba280b7a9 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts @@ -2044,6 +2044,29 @@ describe("usfmFragmentToUsjContent — peripheral divisions (\\periph)", () => { ]); }); + it("refuses a marker line that spells its title twice, keeping every byte literal", () => { + // `alt` is the division TITLE, which periph spells as marker-line text rather than a pipe pair + // (`unknownUsfm.utils.ts` renders it that way, so the editor can never produce this shape) — a + // line carrying both spellings has two conflicting readings and no lossless one, so it refuses + // the list like any other ambiguous attribute list rather than silently dropping a title. + expect(usfmFragmentToUsjContent('\\periph Title|alt="Z"\\mt1 X')).toEqual([ + { type: "para", marker: "periph", content: ['Title|alt="Z"'] }, + { type: "para", marker: "mt1", content: ["X"] }, + ]); + }); + + it("reads a titleless marker line's `alt` pair as the division title", () => { + // No collision, so nothing is ambiguous and the pair is the only spelling present. + expect(usfmFragmentToUsjContent('\\periph |alt="Z" id="t"\\mt1 X')).toEqual([ + { + type: "periph", + alt: "Z", + id: "t", + content: [{ type: "para", marker: "mt1", content: ["X"] }], + }, + ]); + }); + it("opens no division inside note content, where peripheral divisions do not occur", () => { expect( usfmFragmentToUsjContent("\\ft text \\periph Title\\mt1 The Title", { isNoteContext: true }), diff --git a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts index f7622e180..9c36ddf14 100644 --- a/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts +++ b/libs/shared/src/converters/usfm/usfmFragmentToUsj.ts @@ -74,6 +74,9 @@ const TABLE_ROW_MARKER = "tr"; const SIDEBAR_MARKER = "esb"; const SIDEBAR_END_MARKER = "esbe"; const PERIPH_MARKER = "periph"; +/** The USJ property a periph's marker-line title lands on — the same name a pipe pair would use, + * which is why a line carrying both spellings is refused rather than resolved. */ +const PERIPH_TITLE_ATTRIBUTE = "alt"; /** * Table cell marker names: `t` + header/cell (`h`/`c`) + optional alignment infix (`r`/`c`) + @@ -1015,7 +1018,12 @@ export function usfmFragmentToUsjContent( * * A list that does not parse degrades the whole division to an ordinary paragraph carrying the * literal bytes, exactly as an unfoldable figure span degrades — every byte the author wrote - * stays where they can see and fix it. + * stays where they can see and fix it. A list that spells the TITLE a second time (`alt`, + * alongside a non-empty marker-line title) degrades for the same reason and by the same rule: + * the two spellings are the same USJ property, there is no reading that keeps both, and picking + * one would silently destroy bytes the author can still see. `alt` on a TITLELESS line collides + * with nothing and is read normally. The editor can never author the colliding shape — periph's + * display renders `alt` AS the marker-line title (`unknownUsfm.utils.ts`), never as a pipe pair. * * @param atBlockBoundary - Whether the token that ended the line starts a new block (or the * fragment ended). The line's trailing line break is structural there, the same rule the text @@ -1030,14 +1038,21 @@ export function usfmFragmentToUsjContent( const pipeIndex = value.indexOf("|"); const attributes = pipeIndex >= 0 ? parseAttributeText(value.slice(pipeIndex + 1), PERIPH_MARKER) : undefined; - if (pipeIndex >= 0 && !attributes) { + const title = pipeIndex >= 0 ? value.slice(0, pipeIndex) : value; + if (pipeIndex >= 0 && (!attributes || (title && attributes[PERIPH_TITLE_ATTRIBUTE]))) { startParagraph(PERIPH_MARKER, value); return; } - const title = pipeIndex >= 0 ? value.slice(0, pipeIndex) : value; - periph = { type: "periph", ...(title ? { alt: toUsjText(title) } : {}), ...attributes }; - periph.content = []; - result.push(periph); + const opened: MarkerObject = { + type: "periph", + ...(title ? { [PERIPH_TITLE_ATTRIBUTE]: toUsjText(title) } : {}), + ...attributes, + }; + opened.content = []; + // Landed BEFORE `periph` is set, for the same reason the sidebar assembly does it: an already + // open division would otherwise take the new one into its own content. + blockTarget().push(opened); + periph = opened; para = undefined; }; diff --git a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx index c5d4e7c21..c0bad5afe 100644 --- a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx @@ -70,6 +70,7 @@ import { $isRangeSelection, $isTextNode, $setState, + CONTROLLED_TEXT_INSERTION_COMMAND, CUT_COMMAND, LexicalEditor, PASTE_COMMAND, @@ -130,16 +131,23 @@ if (typeof globalStubs.ClipboardEvent === "undefined") type EditorHandle = LexicalEditor; -/** Types `text` one character at a time via `selection.insertText`, each in its own commit — the - * same update path a real keystroke takes (matches the idiom in markerEditTier2Trigger.utils.test.tsx). */ +/** + * Types `text` one character at a time through `CONTROLLED_TEXT_INSERTION_COMMAND`, each in its own + * commit — the command a real keystroke dispatches, not the `selection.insertText` primitive + * underneath it. + * + * The difference is load-bearing for every "paste ≡ typed" pin here. `MarkerEditPlugin` registers a + * NORMAL-priority listener on this command that runs `$prepareReplaceSelection` + * (`markerEditDeletion.utils.ts`), which performs the delete half itself whenever the replaced + * selection covers marker-glyph bytes — exactly the selections these pins use. Driving + * `insertText` directly would skip that listener and compare the paste path against the same + * primitive the paste path ends in, which no equivalence claim can rest on. + */ async function typeCharByChar(editor: EditorHandle, text: string): Promise { for (const character of text) { - await act(async () => - editor.update(() => { - const selection = $getSelection(); - if ($isRangeSelection(selection)) selection.insertText(character); - }), - ); + await act(async () => { + editor.dispatchCommand(CONTROLLED_TEXT_INSERTION_COMMAND, character); + }); } } @@ -523,15 +531,16 @@ describe("multi-line payload collapses to a single space, per newline (attribute }); }); -describe("marker-bearing payload: literal value text, no strip, no chapter node", () => { - it('paste "\\c 5" into the attribute run: the bytes survive literally, no chapter node is created', async () => { +describe("marker-bearing payload: literal value text, no strip — and what the settle then does with it", () => { + /** The run-end caret of TJ's repro, the shape the whole attribute-context path was built for. */ + function $selectRunEnd(): void { + const run = $charAttributeRun($firstChar()); + run.select(run.getTextContentSize(), run.getTextContentSize()); + } + + it('paste "\\c 5" at the run\'s end: the bytes survive the paste literally, un-stripped', async () => { const { editor } = await testEnvironmentWithCharSync($charFixture); - await act(async () => - editor.update(() => { - const run = $charAttributeRun($firstChar()); - run.select(run.getTextContentSize(), run.getTextContentSize()); - }), - ); + await act(async () => editor.update($selectRunEnd)); await pasteAndFlush(editor, { "text/plain": "\\c 5" }); @@ -546,6 +555,39 @@ describe("marker-bearing payload: literal value text, no strip, no chapter node" ).toEqual([]); expect(usj.content.filter((item): item is string => typeof item === "string")).toEqual([]); }); + + it("and at the settle those bytes DO re-tokenize — identically to the same bytes typed there", async () => { + // The value-byte carve-out is justified by paste ≡ TYPING, not by "these bytes never + // re-tokenize": the attribute tag survives the insertion, but the caret-departure settle + // re-tokenizes the whole paragraph and reads `\c 5` as the chapter marker it spells. Recorded + // here rather than left invisible behind an unsettled assertion. The residual — marker bytes + // reaching a value at all — is the TYPED `\c` hole (Deferred item 2 in the semantics doc), + // which paste inherits by design; closing it for paste alone would eat bytes out of an + // attribute value that were never a chapter token, and would break the equivalence below. + const pasted = await testEnvironmentWithCharSync($charFixture); + await act(async () => pasted.editor.update($selectRunEnd)); + await pasteAndFlush(pasted.editor, { "text/plain": "\\c 5" }); + await departAndSettle(pasted.editor, () => $bodyTextNode().select(0, 0)); + + const typed = await testEnvironmentWithCharSync($charFixture); + await act(async () => typed.editor.update($selectRunEnd)); + await typeCharByChar(typed.editor, "\\c 5"); + await departAndSettle(typed.editor, () => $bodyTextNode().select(0, 0)); + + // Asserted as the concrete settled shape, so the residual is visible in the suite rather than + // hidden inside an equivalence that would also pass if both arms were clean. + expect(usjOf(pasted.editor).content).toEqual([ + { + type: "para", + marker: "p", + content: [{ type: "char", marker: "nd", closed: "false", content: ['asdf|who="hi"'] }], + }, + { type: "chapter", marker: "c", number: "5" }, + { type: "unmatched", marker: "nd*" }, + { type: "para", marker: "p", content: [" body"] }, + ]); + expect(usjOf(pasted.editor)).toEqual(usjOf(typed.editor)); + }); }); describe("CUT of a selection inside the attribute value", () => { @@ -815,29 +857,98 @@ describe("mixed selection: the pasted bytes are BODY content, not attribute-valu expect(usjOf(pasted.editor)).toEqual(usjOf(typed.editor)); }); - it("leaves a range wholly INSIDE the run on attribute-value semantics — a pasted \\c stays literal there", async () => { + it("leaves a range wholly INSIDE the run on attribute-value semantics — a pasted \\c stays literal there, and settles as typing it would", async () => { // The boundary of the rule above, in its non-collapsed form (the collapsed-caret form is pinned - // in the "marker-bearing payload" describe). The run survives this replacement still tagged - // "attribute", so the bytes really are value text and the strip must not touch them. - const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); - await act(async () => - editor.update(() => { - const run = $charAttributeRun($firstChar()); - const text = run.getTextContent(); // '|who="hi"' - run.select(text.indexOf('"') + 1, text.lastIndexOf('"')); - }), - ); + // in the "marker-bearing payload" describe). The strip must not touch these bytes: they are + // value text at insertion time, and paste must put them where typing them would. + // + // Settled deliberately, and the settled shape asserted concretely: the attribute tag survives + // the insertion but not the caret-departure re-tokenization, so `\c 5` DOES become a chapter + // marker here — the typed `\c` hole (Deferred item 2), inherited by design rather than + // introduced. An assertion that stopped before the settle would read "safe" over a corrupt + // document, which is the shape this suite exists to refuse. + function $selectValueInterior(): void { + const run = $charAttributeRun($firstChar()); + const text = run.getTextContent(); // '|who="hi"' + run.select(text.indexOf('"') + 1, text.lastIndexOf('"')); + } - await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + const pasted = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => pasted.editor.update($selectValueInterior)); + await pasteAndFlush(pasted.editor, { "text/plain": "\\c 5" }); - editor.getEditorState().read(() => { + pasted.editor.getEditorState().read(() => { expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="\\c 5"'); }); - const content = usjOf(editor).content; - expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( - [], - ); + + await departAndSettle(pasted.editor, () => $bodyTextNode().select(0, 0)); + + const typed = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => typed.editor.update($selectValueInterior)); + await typeCharByChar(typed.editor, "\\c 5"); + await departAndSettle(typed.editor, () => $bodyTextNode().select(0, 0)); + + expect(usjOf(pasted.editor).content).toEqual([ + { + type: "para", + marker: "p", + content: [ + "before ", + { type: "char", marker: "nd", closed: "false", content: ['asdf|who="'] }, + ], + }, + { type: "chapter", marker: "c", number: '5"' }, + { type: "unmatched", marker: "nd*" }, + " after", + { type: "para", marker: "p", content: ["body"] }, + ]); + expect(usjOf(pasted.editor)).toEqual(usjOf(typed.editor)); }); + + it.each([ + ["a range ending at the trailing text's start", "runEndToTrailingStart"], + ["a BACKWARD range from the trailing text into the run", "backwardTrailingToRun"], + ["a range covering the run, the closer and all the trailing text", "runStartToTrailingEnd"], + ] as const)( + "strips a pasted \\c for %s too — the rule is about touching, not about which end touches", + async (_name, shape) => { + // The report for this branch measured four mixed-selection shapes; only one was pinned. The + // other three live here so the claim is in the suite rather than in a transcript. + const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); + await act(async () => + editor.update(() => { + const run = $charAttributeRun($firstChar()); + const trailing = $trailingBodyText(); + if (shape === "backwardTrailingToRun") { + trailing.select(3, 3); + const selection = $getSelection(); + if ($isRangeSelection(selection)) selection.focus.set(run.getKey(), 1, "text"); + return; + } + const start = shape === "runStartToTrailingEnd" ? 0 : run.getTextContentSize(); + run.select(start, start); + const selection = $getSelection(); + if ($isRangeSelection(selection)) + selection.focus.set( + trailing.getKey(), + shape === "runStartToTrailingEnd" ? trailing.getTextContentSize() : 0, + "text", + ); + }), + ); + + await pasteAndFlush(editor, { "text/plain": "\\c 5" }); + await departAndSettle(editor, () => $bodyTextNode().select(0, 0)); + + const content = usjOf(editor).content; + expect(content.filter((item) => typeof item !== "string" && item.type === "chapter")).toEqual( + [], + ); + // A chapter token also closes its paragraph, stranding the rest as bare top-level strings — + // asserted so the pin cannot pass on a document that merely renamed the damage. + expect(content.filter((item): item is string => typeof item === "string")).toEqual([]); + }, + ); }); // --------------------------------------------------------------------------------------------- diff --git a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx index 389860beb..0982604bc 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx @@ -314,6 +314,28 @@ describe("attributes a construct carries in bytes the display had to reconstruct ); }); + it("emits no category bytes for a selection that stops before the note's content", async () => { + // The category bytes belong to the region AFTER the caller separator, so a selection ending at + // that separator must not pull them in: it would put `\\cat People\\cat*` on the clipboard for a + // range the user never covered, with no `\\f` opener in front of the run that follows. + const { editor } = await renderUsjEditor(categorizedNoteUsj()); + await act(async () => + editor.update(() => { + const note = findOnlyNote($getRoot()); + const separator = note.getChildren()[2]; // opening glyph, caller, separator + const first = note.getFirstDescendant(); + if (!separator || !first) throw new Error("note is missing its caller separator"); + const selection = $createRangeSelection(); + selection.anchor = $createPoint(first.getKey(), 0, "text"); + selection.focus = $createPoint(separator.getKey(), 0, "text"); + $setSelection(selection); + }), + ); + const { event, getData } = copyEvent(); + await act(async () => editor.dispatchCommand(COPY_COMMAND, event)); + expect(getData("text/plain")).toBe("\\f -"); + }); + it("copies a spanning cell's width as the span suffix its marker name carries", async () => { const { editor } = await renderUsjEditor(spanningCellTableUsj()); await act(async () => editor.update($selectWholeDocument)); diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 72e23df22..b3cd049e6 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -323,18 +323,25 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { * attribute context, or a range whose anchor and focus are the same node and that node is in * attribute context. * - * This is the only shape whose pasted bytes are genuinely attribute VALUE bytes. The value-byte - * carve-outs ({@link $insertPastedTextIntoAttributeContext}) are earned by exactly one fact: the - * bytes land in a node that stays tagged "attribute", which `$textNodeTier2Transform` skips, so - * they are never read as markers and never re-tokenize. + * This is the only shape whose pasted bytes are genuinely attribute VALUE bytes, and therefore the + * only one whose value-byte carve-outs ({@link $insertPastedTextIntoAttributeContext}) are earned. + * What earns them is paste ≡ TYPING: the bytes land where the same keystrokes would land, and this + * editor's whole attribute-context contract is that a paste in a value behaves as typing there + * does. It is NOT that the bytes never re-tokenize — measured, that is false: the `"attribute"` tag + * survives the insertion, but the caret-departure settle re-tokenizes the paragraph, so a pasted + * `\c 5` in a value DOES become a chapter marker. It becomes one identically when TYPED, which is + * why this is the typed-`\c` hole the semantics doc defers (Deferred item 2), inherited rather than + * introduced. Extending the strip to cover it would eat bytes out of an attribute value that were + * never a chapter token — the very regression the carve-out exists to prevent — and would break the + * equivalence. Both halves are pinned settled, on concrete USJ, in + * `attributeContextPasteFidelity.test.tsx`. * - * A selection that merely TOUCHES attribute context does not have that property. Removing the + * A selection that merely TOUCHES attribute context does not land in a value at all. Removing the * range takes the bytes out of the run, and when the range covers a char span's closing glyph it - * deletes the closer too, so the whole span re-tokenizes around whatever just arrived. Measured - * both ways round — a range starting in body text and reaching INTO the run, and a range starting - * inside the run and reaching PAST the closer — a pasted `\c 5` produced a real chapter node - * mid-paragraph, which is exactly the save-loop poisoning {@link $stripPastedChapterAndBookId} - * exists to prevent (`attributeContextPasteFidelity.test.tsx`). + * deletes the closer too, so what arrives is ordinary paragraph content — measured both ways round, + * a range starting in body text and reaching INTO the run, and a range starting inside the run and + * reaching PAST the closer. Those bytes get body content's rules, which is a divergence from typing + * in exactly the two places body paste always diverges from it. */ function $isSelectionWithinOneAttributeNode(selection: RangeSelection): boolean { const { anchor, focus } = selection; @@ -358,11 +365,14 @@ function $isSelectionWithinOneAttributeNode(selection: RangeSelection): boolean * The two BODY-content paste rules — the `\c`/`\id` strip and the positional NBSP mapping, the only * two places a paste deliberately diverges from typing the same bytes — are skipped only for a * selection that stays wholly inside one attribute node ({@link $isSelectionWithinOneAttributeNode}). - * There the bytes really are value text: a pasted `\c 5` is literal, not a structural marker, and - * `$displayWhitespaceTransform` (this file) already skips textType "attribute" nodes outright, so a - * literal NBSP a user TYPES into an attribute run is never touched and a pasted one must not be - * either. A selection that only TOUCHES an attribute run does not qualify: its bytes end up as - * ordinary paragraph content that re-tokenizes, so they get body content's rules. What does NOT + * There the bytes go where the same keystrokes would put them, and paste must not diverge: a pasted + * `\c 5` is inserted literally exactly as a typed one is, and a pasted NBSP passes through exactly + * as a typed one does (`$displayWhitespaceTransform`, this file, skips textType "attribute" nodes + * outright). What the SETTLE then makes of those bytes is the same for both — a `\c 5` in a value + * re-tokenizes into a chapter marker whether it was typed or pasted, which is the deferred typed-`\c` + * hole rather than a paste defect; see {@link $isSelectionWithinOneAttributeNode}. A selection that + * only TOUCHES an attribute run does not qualify: its bytes end up as ordinary paragraph content, so + * they get body content's rules. What does NOT * change with them is the insertion MECHANISM — one `insertText`, never the paragraph-splitting * line replay and never Lexical's rich-paste node insertion, both of which corrupt the * attribute-run end of the range. A multi-line payload landing in body content this way therefore @@ -546,8 +556,15 @@ function $isNoteInternalDisplaySeparator(node: TextNode): boolean { * Placed AFTER the separator's own space and with none of its own, which is where the file has the * span: `\f + \cat People\cat*\fr 1:1 …` — a trailing space here would re-tokenize into a stray * text child inside the note. + * + * `isLast` is what keeps it inside the selection. These bytes stand for the region AFTER the + * separator, so a selection that STOPS there covers none of them: emitting anyway put + * `\cat People\cat*` on the clipboard for a range the user never selected, with no `\f` opener in + * front of the run that followed (measured: `\f -\cat People\cat*`). The sibling caller rule + * guards its own boundary the same way. */ -function $collapsedNoteCategoryBytes(node: TextNode): string { +function $collapsedNoteCategoryBytes(node: TextNode, isLast: boolean): string { + if (isLast) return ""; if (node.getTextContent() !== NBSP) return ""; const note = node.getParent(); if (!$isNoteNode(note)) return ""; @@ -648,7 +665,7 @@ export function $selectionToUsfmText(selection: RangeSelection): string { } text += $isNoteInternalDisplaySeparator(node) ? "" - : nodeText.replaceAll(NBSP, " ") + $collapsedNoteCategoryBytes(node); + : nodeText.replaceAll(NBSP, " ") + $collapsedNoteCategoryBytes(node, node === lastNode); } else if ( ($isDecoratorNode(node) || $isLineBreakNode(node)) && (node !== lastNode || !selection.isCollapsed()) From aab7618ded4551b53152dd32fd5e04428fde7150 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Thu, 10 Sep 2026 17:37:47 -0500 Subject: [PATCH 17/44] style: keep the clipboard comments forward-facing Comment-hygiene sweep of the clipboard/tokenizer work: rewrite backward-facing comments (dated live-repro narration, "used to"/"previously" change narration, personal attribution, "this branch" framing) added by this branch so they read as durable, forward-facing documentation instead of PR provenance. No logic, assertions, or names changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCjocx9uZcPqB2esPh9r6f --- .../specs/2026-08-06-clipboard-semantics.md | 4 +- .../editor/CommandMenuPlugin.gate.test.tsx | 8 +- .../attributeContextPasteFidelity.test.tsx | 76 ++++++------- .../markerEdit/clipboardCopyFidelity.test.tsx | 19 ++-- .../clipboardCorpusRoundTrip.test.tsx | 11 +- .../markerEdit/figurePasteFidelity.test.tsx | 11 +- .../markerEditDeletion.utils.test.tsx | 11 +- .../src/editor/markerEdit/markerKind.utils.ts | 4 +- .../markerEdit/markerPasteFidelity.test.tsx | 16 +-- .../editor/markerEdit/noteEnterFp.test.tsx | 21 ++-- .../optbreakClipboardFidelity.test.tsx | 106 +++++++++--------- .../editor/markerEdit/tier2Rebuild.utils.ts | 5 +- .../unknownClipboardFidelity.test.tsx | 4 +- .../whitespaceDisplay.plugin.utils.test.tsx | 52 +++++---- .../whitespaceDisplay.plugin.utils.ts | 90 +++++++-------- 15 files changed, 204 insertions(+), 234 deletions(-) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md index 992f0c5f1..a7286e8f1 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/superpowers/specs/2026-08-06-clipboard-semantics.md @@ -77,7 +77,7 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the saved file diverged from the editor. `sid="things"` carries no NBSP; the pre-branch build's paste handler still had its OLD NBSP-gated form (see the "positional NBSP normalization" bullet above — that gate was generalized to claim every external paste, NBSP or not, on 2026-08-07), so the most plausible mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion — NOT a same-namespace `application/x-lexical-editor` flavor on the clipboard (this document's own "private Lexical flavor is dead on Ctrl+V" fact, above, still holds for the reconstructed-`DataTransfer` paste path this repro most likely used). - What matters for THIS branch's code is the SHAPE the corruption takes once ANY handler declines an attribute-context paste to Lexical's default rich-paste node insertion — reproduced directly here (`attributeContextPasteFidelity.test.tsx`'s "root cause" describe): it has no notion that an attribute run's text must stay inside its one tagged TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph's sibling text into one plain node, destroying the attribute display and the closing marker and leaving the pasted bytes loose in body content. On this branch's CURRENT code (a single-line, NBSP-free, flavor-free external paste is already safe since 2026-08-07's generalization), the confirmed regression classes this paste-≡-typing rule closes are: (1) a live native paste event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for when that reaches a real handler); (2) a multi-line plain-text payload, which the ordinary pipeline would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND`; (3) a marker-bearing payload (`\c 5`) landing wholly INSIDE one attribute node, where the ordinary pipeline's `\c`/`\id` strip would eat bytes out of an attribute VALUE that were never a chapter token (bytes merely TOUCHING a run do get that strip — see the body-rules paragraph above); (4) a MIXED selection (one end in the run, one end out) combined with either (1) or (2) — the selection touches attribute context, so it must not reach either risky branch merely because its other end sits outside. + What matters for the current code is the SHAPE the corruption takes once ANY handler declines an attribute-context paste to Lexical's default rich-paste node insertion — reproduced directly here (`attributeContextPasteFidelity.test.tsx`'s "root cause" describe): it has no notion that an attribute run's text must stay inside its one tagged TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph's sibling text into one plain node, destroying the attribute display and the closing marker and leaving the pasted bytes loose in body content. On the current code (a single-line, NBSP-free, flavor-free external paste is already safe since 2026-08-07's generalization), the confirmed regression classes this paste-≡-typing rule closes are: (1) a live native paste event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for when that reaches a real handler); (2) a multi-line plain-text payload, which the ordinary pipeline would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND`; (3) a marker-bearing payload (`\c 5`) landing wholly INSIDE one attribute node, where the ordinary pipeline's `\c`/`\id` strip would eat bytes out of an attribute VALUE that were never a chapter token (bytes merely TOUCHING a run do get that strip — see the body-rules paragraph above); (4) a MIXED selection (one end in the run, one end out) combined with either (1) or (2) — the selection touches attribute context, so it must not reach either risky branch merely because its other end sits outside. Pre-existing precedence, UNCHANGED by this rule: the CRITICAL-priority in-note multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs before the Standard-view external-paste handler and still wins for a multi-line payload whose selection touches EXPANDED note content — an attribute run that happens to sit inside an expanded note's content is reached by this rule only when that in-note claim itself declines. See `$handlePasteForStandardView`'s doc comment (`whitespaceDisplay.plugin.utils.ts`) for the full mechanism. @@ -236,7 +236,7 @@ The following items are recorded as deferred and not addressed in this plan: | **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"`, `"attributes a construct carries in bytes the display had to reconstruct"` (a collapsed note's `\cat` run; a spanning table cell's span suffix) describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | | **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"` and `"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"` (external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction); `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | | **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | -| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (confirmed regression class on this branch, not TJ's literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | +| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (a confirmed regression class, not the literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | | **S4** Paste-as-plain-text (Ctrl+Shift+V) — equivalence to external paste | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"paste-as-plain-text equivalence (S4): no literal mode, plain always wins"` describe | | **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | | **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | diff --git a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx index afbcaa619..8c28735ab 100644 --- a/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx +++ b/packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx @@ -288,10 +288,10 @@ describe("MarkerEditPlugin's Standard-view clipboard handlers do not leak into F /** * The empty-copy rule is not Standard view's. It lives in `ClipboardPlugin` (shared-react), which * every view mounts, and this view has no Standard-view copy handler registered at all — so a - * regression here would leak the same way the reported one did, in a view where nothing else is - * watching. These exercise the whole real path from the public surface to the clipboard: a keydown - * on the root element where the plugin listens, and `EditorRef.copy()`/`.cut()`, the API a host - * app drives the editor through. + * regression here would leak silently, in a view where nothing else is watching. These exercise + * the whole real path from the public surface to the clipboard: a keydown on the root element + * where the plugin listens, and `EditorRef.copy()`/`.cut()`, the API a host app drives the editor + * through. * * `execCommandSpy` supplies the `document.execCommand` jsdom lacks and reports whether anything was * written. `ClipboardEvent` is stubbed for the same reason `formattedViewCopyEvent` stubs it: diff --git a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx index c0bad5afe..b159436a3 100644 --- a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx @@ -2,33 +2,28 @@ * Paste (and cut) fidelity for a selection that TOUCHES an attribute display run — a char span's * `|attrs` list, a milestone's attribute run, or a verse's `\va`/`\vp` run. * - * TJ's live repro (filed 2026-08-11, against a pre-branch build): existing span - * `\nd asdf|who="hi"\nd*`, caret at the end of the `who="hi"` run, paste plain text - * `sid="things"`. Observed: the `who` attribute display AND the closing `\nd*` glyph disappeared - * from the editor, the pasted text rendered visually outside the span, and the saved file diverged - * from the editor. `sid="things"` carries no NBSP, and the pre-branch build's paste handler still - * had its OLD NBSP-gated form (generalized to claim every external paste, NBSP or not, on - * 2026-08-07 — see `$handlePasteForStandardView`'s own doc comment) — so the most plausible - * mechanism is that OLD gate declining an NBSP-free paste outright and falling through to Lexical's - * default rich-paste node insertion, NOT a same-namespace `application/x-lexical-editor` payload on - * the clipboard. On THIS branch's current code, a single-line, NBSP-free, flavor-free external - * paste like TJ's literal repro was already safe (see the "typed characterization"/"paste ≡ typed" - * pins below, which pin that fact rather than a corruption). + * Concrete example of the failure mode: with existing span `\nd asdf|who="hi"\nd*`, caret at the + * end of the `who="hi"` run, pasting plain text `sid="things"` (which carries no NBSP) observably + * drops the `who` attribute display AND the closing `\nd*` glyph, renders the pasted text visually + * outside the span, and diverges the saved file from the editor. A single-line, NBSP-free, + * flavor-free external paste like this one is already safe on its own (see the "typed + * characterization"/"paste ≡ typed" pins below, which pin that fact rather than a corruption) — it + * takes one of the shapes below to reach the corruption class this file exists to catch. * * What this file's "root cause" describe block reproduces and fixes is the SHAPE that corruption * takes whenever ANY handler declines an attribute-context paste to Lexical's default rich-paste * node insertion: it has no notion that an attribute run's text must stay inside its ONE tagged * TextNode, and merges the run, the closing glyph, and even the FOLLOWING paragraph sibling's text * into one plain node — destroying the attribute display, the closing marker, and the paragraph - * boundary in one move. Confirmed regression classes on this branch, each pinned below: a live - * native paste event that still carries a same-namespace `application/x-lexical-editor` flavor; a - * multi-line plain-text payload (the ordinary pipeline splits it via `insertParagraph()`); a - * marker-bearing payload (the ordinary pipeline's `\c`/`\id` strip eats bytes out of an attribute - * VALUE that were never a chapter token); and a selection that only PARTLY touches the attribute - * run combined with either of the first two. Fixed by always routing a paste whose selection - * TOUCHES attribute-display text through plain-text insertion, regardless of what other MIME - * flavors the clipboard also carries or how much of the selection sits outside the run — see - * `$handlePasteForStandardView`'s doc comment for the full design. + * boundary in one move. Regression classes pinned below, each reaching that same failure mode a + * different way: a live native paste event that still carries a same-namespace + * `application/x-lexical-editor` flavor; a multi-line plain-text payload (the ordinary pipeline + * splits it via `insertParagraph()`); a marker-bearing payload (the ordinary pipeline's `\c`/`\id` + * strip eats bytes out of an attribute VALUE that were never a chapter token); and a selection that + * only PARTLY touches the attribute run combined with either of the first two. Fixed by always + * routing a paste whose selection TOUCHES attribute-display text through plain-text insertion, + * regardless of what other MIME flavors the clipboard also carries or how much of the selection + * sits outside the run — see `$handlePasteForStandardView`'s doc comment for the full design. * * Binding design principle: paste in attribute context ≡ typing the same characters at the same * caret (or over the same selection). Every "paste ≡ typed" pin below proves paste and the @@ -199,9 +194,10 @@ function usjOf(editor: EditorHandle): Usj { } // --------------------------------------------------------------------------------------------- -// Char span fixture: TJ's exact repro shape, `\nd asdf|who="hi"\nd*`, plus a second paragraph to -// depart to. Mounts BOTH CharNodePlugin (self-heal) and MarkerEditPlugin (pend/settle) — the real -// app's plugin stack — matching charAttributeDeletionSettle.test.tsx's idiom. +// Char span fixture: the shape this whole file is built around, `\nd asdf|who="hi"\nd*`, plus a +// second paragraph to depart to. Mounts BOTH CharNodePlugin (self-heal) and MarkerEditPlugin +// (pend/settle) — the real app's plugin stack — matching charAttributeDeletionSettle.test.tsx's +// idiom. // --------------------------------------------------------------------------------------------- function $charFixture(): void { @@ -313,8 +309,8 @@ describe("paste ≡ typed (TJ's repro shape): plain-text-only paste at the end o await pasteAndFlush(editor, { "text/plain": 'sid="things"' }); - // Pre-settle: no escaped text, closer intact, attribute display present (the exact live-repro - // symptoms — hidden display, escaped text — must not appear even transiently). + // Pre-settle: no escaped text, closer intact, attribute display present — the corruption + // symptoms (hidden display, escaped text) must not appear even transiently. editor.getEditorState().read(() => { const char = $firstChar(); expect($charCloser(char).getTextContent()).toBe("\\nd*"); @@ -352,10 +348,8 @@ describe("root cause: a native paste event carrying a same-namespace application // a live native paste event that still carries the flavor (a genuine same-page copy, not the // reconstructed-DataTransfer paste path S3's own doc comment shows can never carry it) keeps // Lexical's exact-node-tree fast path for ORDINARY content — but that guard must never win over - // an attribute-context destination; this reproduces the corruption class the guard used to - // cause there (a confirmed regression class on this branch — not necessarily TJ's own - // pre-branch repro, whose most plausible mechanism was the OLD NBSP-gated handler; see this - // file's header comment). + // an attribute-context destination, so this pins the corruption that fast path would otherwise + // cause there. const { editor } = await testEnvironmentWithCharSync(() => { $charFixture(); $getRoot().append( @@ -532,7 +526,7 @@ describe("multi-line payload collapses to a single space, per newline (attribute }); describe("marker-bearing payload: literal value text, no strip — and what the settle then does with it", () => { - /** The run-end caret of TJ's repro, the shape the whole attribute-context path was built for. */ + /** The run-end caret this whole attribute-context path was built for. */ function $selectRunEnd(): void { const run = $charAttributeRun($firstChar()); run.select(run.getTextContentSize(), run.getTextContentSize()); @@ -624,16 +618,14 @@ describe("CUT of a selection inside the attribute value", () => { }); describe("mixed selection: spans BOTH attribute and non-attribute content", () => { - // Design choice (brief-permitted either way): a selection that only PARTLY sits inside - // attribute-display text now TAKES the attribute-context path (widened from an earlier - // both-ends-must-qualify check) rather than declining it — under paste ≡ typing, a user typing a + // Design choice: a selection that only PARTLY sits inside attribute-display text TAKES the + // attribute-context path rather than declining it — under paste ≡ typing, a user typing a // character over this exact selection gets `selection.insertText`'s own removeText-then-insert // behavior regardless of which node the selection's OTHER end sits on, so paste must take the // identical path instead of falling through to a branch (the ordinary pipeline's // `insertParagraph()` for a multi-line payload, or Lexical's rich-node paste for a // same-namespace-flavored one) that CAN corrupt the attribute-run end of the range — reproduced - // in the two pins below this one, each of which used to reach exactly one of those two branches - // before the OR-widening. + // in the two pins below this one, each of which reaches exactly one of those two branches. function $selectRunThroughCloserStart(): void { const char = $firstChar(); const run = $charAttributeRun(char); @@ -659,10 +651,10 @@ describe("mixed selection: spans BOTH attribute and non-attribute content", () = }); it("mixed selection + same-namespace application/x-lexical-editor flavor: does not reach Lexical's rich-paste node insertion (regression for the corruption class this closes)", async () => { - // Before the OR-widening, this exact shape (one end in attribute context, one end not) failed - // the (then AND-based) attribute-context check, so the same-namespace-flavor decline above ran - // unconditionally and handed the paste to Lexical's default rich-paste node insertion — the - // SAME corruption class as the "root cause" describe block above, reached via a mixed selection + // This exact shape (one end in attribute context, one end not) must also take the + // attribute-context path: were the same-namespace-flavor decline above to run unconditionally + // here, it would hand the paste to Lexical's default rich-paste node insertion — the SAME + // corruption class as the "root cause" describe block above, reached via a mixed selection // instead of a fully-inside one. const { editor } = await testEnvironmentWithCharSync(() => { $charFixture(); @@ -912,8 +904,8 @@ describe("mixed selection: the pasted bytes are BODY content, not attribute-valu ] as const)( "strips a pasted \\c for %s too — the rule is about touching, not about which end touches", async (_name, shape) => { - // The report for this branch measured four mixed-selection shapes; only one was pinned. The - // other three live here so the claim is in the suite rather than in a transcript. + // All four mixed-selection shapes are covered: the rule is about whether the selection + // touches attribute-value context, not about which end of the range does the touching. const { editor } = await testEnvironmentWithCharSync($inlineCharFixture); await act(async () => editor.update(() => { diff --git a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx index 0982604bc..6d2e7bee9 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx @@ -1,8 +1,8 @@ /** - * Copy-fidelity pins for Standard-view `text/plain`: note callers (previously silently dropped), - * source-faithful NBSP handling around a collapsed note's internal markers (previously a blanket - * NBSP→space mapping produced phantom spaces the source USFM never had), multi-paragraph joining, - * and the copy→paste USJ round trip. Kept separate from `whitespaceDisplay.plugin.utils.test.tsx` + * Copy-fidelity pins for Standard-view `text/plain`: note callers render in the copied text, + * source-faithful NBSP handling around a collapsed note's internal markers (a blanket NBSP→space + * mapping would produce phantom spaces the source USFM never had), multi-paragraph joining, and + * the copy→paste USJ round trip. Kept separate from `whitespaceDisplay.plugin.utils.test.tsx` * (which already covers the NBSP display-run invariant and the plain payload-builder contract) so * this file can stay focused on `$selectionToUsfmText`'s USFM-shape behavior. */ @@ -124,9 +124,9 @@ function noteUsj(caller: string): Usj { } /** - * The 2026-08-07 live-repro footnote: `\f - \fr 1:1 \ft Caller test.\f*` — content chars carry - * `closed: "false"` (ParatextData's real shape for footnote content — `\fr`/`\ft` never get their - * own closer), matching the exact bytes the repro pinned. + * A footnote shaped like real ParatextData output: `\f - \fr 1:1 \ft Caller test.\f*` — content + * chars carry `closed: "false"` (ParatextData's real shape for footnote content — `\fr`/`\ft` + * never get their own closer). */ function footnoteReproUsj(): Usj { return { @@ -153,9 +153,8 @@ function footnoteReproUsj(): Usj { } /** - * The 2026-08-07 live-repro cross-reference: `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*` — content - * chars carry no `closed` flag (explicitly closed, real cross-reference shape), matching the exact - * bytes the repro pinned. + * A cross-reference shaped like real ParatextData output: `\x - \xo 1:3: \xo*\xt 2Cor 4:6\xt*\x*` + * — content chars carry no `closed` flag (explicitly closed, real cross-reference shape). */ function xrefReproUsj(): Usj { return { diff --git a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx index 3ef437957..102e2d5ab 100644 --- a/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx +++ b/packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx @@ -75,12 +75,11 @@ function $contentStartIndex(root: RootNode): number { * document — the chapter's own CONTENT, matching what Standard view actually copies out of an * open chapter (never the book/chapter header itself). Element-offset (`root.select`), not * node-anchored: this is used ONLY on the source/copy side (the paste side below always targets a - * fresh EMPTY host via `selectEnd()`), so it never hits the "replacing a large non-collapsed - * selection on paste" doubling bug a node-anchored `"text"`-typed focus point was once written to - * dodge here — and the element-offset form is strictly more general: it does not require the - * chapter's last node be a `TextNode` (a `"text"`-typed focus point does, and crashed with - * `TypeError: focusNode.selectionTransform is not a function` against a real corpus document, - * `libs/test-data/src/data/2sa.usj.ts`, whose last node isn't one). */ + * fresh EMPTY host via `selectEnd()`). A node-anchored `"text"`-typed focus point requires the + * chapter's last node be a `TextNode` — it crashes with `TypeError: + * focusNode.selectionTransform is not a function` against a real corpus document, + * `libs/test-data/src/data/2sa.usj.ts`, whose last node isn't one — so the element-offset form + * used here, which carries no such requirement, is strictly more general. */ function $selectChapterContent(root: RootNode): void { const start = $contentStartIndex(root); root.select(start, root.getChildrenSize()); diff --git a/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx index 735cfefff..205382716 100644 --- a/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx @@ -8,12 +8,11 @@ * surrounding prose — and an exported USJ whose `figure` object had no `content` at all, the * caption merged into the neighbouring text run instead. * - * Mechanism: `$textNodeInUnknownTransform` (`TextSpacingPlugin`, shared-react) ejects any TextNode - * that is NEW in the current update out of an `UnknownNode` parent, so a user typing into a - * read-only opaque block never plants prose inside it. A Tier-2 rebuild materializes the WHOLE - * figure — wrapper and caption together — in one update, so the caption looked exactly like typed - * text to that rule. The transform now ejects only when the wrapper predates the update, which is - * the shape a typed intrusion actually has. + * Mechanism: `$textNodeInUnknownTransform` (`TextSpacingPlugin`, shared-react) ejects a TextNode + * out of an `UnknownNode` parent only when the WRAPPER predates the current update — the shape a + * typed intrusion into an existing read-only opaque block actually has. A Tier-2 rebuild + * materializes the WHOLE figure — wrapper and caption together — in the same update, so the + * caption stays inside it rather than being ejected as though it were typed text. * * These pins mount the real `Editor` (via `mountStandardViewEditor`) rather than the marker-edit * plugin alone: the ejection lives in a sibling plugin, so a harness carrying only diff --git a/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx b/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx index 3ba7d44c4..d5cf0a4ff 100644 --- a/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx @@ -1047,12 +1047,11 @@ describe("load/engine para prefix drift pin", () => { }); describe("selection-delete at a settled char-span boundary (WI-2 filed)", () => { - // Regression armor for a QA-filed live repro: after typing settled a `\wj …\wj*` char span, a - // PROGRAMMATIC DOM selection built with `range.setEndAfter(spanElement)` — anchor in the text - // BEFORE the span, focus set immediately after the span's whole DOM element — was deleted, and - // the deletion absorbed one character AFTER the selection's own end: the closing curly quote - // `”` that immediately follows the span. Mouse-drawn selections never reproduced it, so this - // mirrors the DOM shape at the Lexical level with an ELEMENT-point focus (offset = the + // A PROGRAMMATIC DOM selection built with `range.setEndAfter(spanElement)` — anchor in the text + // BEFORE a settled `\wj …\wj*` char span, focus set immediately after the span's whole DOM + // element — must not absorb the character AFTER the selection's own end: the closing curly quote + // `”` that immediately follows the span. Mouse-drawn selections never reproduce this shape, so + // this mirrors the DOM shape at the Lexical level with an ELEMENT-point focus (offset = the // CharNode's own index within its parent, plus one — the exact mapping `setEndAfter` produces), // not a text point landing inside either neighboring text node. diff --git a/packages/platform/src/editor/markerEdit/markerKind.utils.ts b/packages/platform/src/editor/markerEdit/markerKind.utils.ts index 03225f541..7286805c3 100644 --- a/packages/platform/src/editor/markerEdit/markerKind.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerKind.utils.ts @@ -25,8 +25,8 @@ import { isMilestoneHeuristicName, MarkerLookup, MarkerType, NoteNode } from "sh * * `tier2Rebuild.utils.ts`'s own-marker-prefix dedup needs this exact rule too — the SAME * stylesheet-first/unknown-as-paragraph classification `$buildParaFragment` already uses for the - * paragraph's own marker. A second, narrower `type === MarkerType.Paragraph` check there - * disagreed with it for any unknown/custom.sty marker. + * paragraph's own marker. A second, narrower `type === MarkerType.Paragraph` check there would + * disagree with it for any unknown/custom.sty marker. */ export function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { const clean = marker.replace(/^\+/, ""); diff --git a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx index 9c8d3dfd0..ba087f7f7 100644 --- a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx @@ -1,8 +1,8 @@ /** - * Multi-line marker-bearing paste semantics (live-verified, 2026-08-07) and the `\c`/`\id` strip. + * Multi-line marker-bearing paste semantics and the `\c`/`\id` strip. * A pasted line starting with its own paragraph-marker literal owns that marker instead of also * getting the host paragraph's cloned prefix; a marker-free line inherits the host's. `\c`/`\id` - * never survive paste normalization — pasting either used to be able to reach an unsaveable + * never survive paste normalization — pasting either would otherwise reach an unsaveable * editor state (a second chapter/book-id node the PDP rejects on save). Kept separate from * `whitespaceDisplay.plugin.utils.test.tsx` (the NBSP/claim-policy contract) and * `clipboardCopyFidelity.test.tsx` (copy-side byte fidelity) so this file stays focused on the @@ -357,12 +357,12 @@ describe("\\c/\\id strip on paste", () => { }); it("a pasted \\c never leaves the editor in the unsaveable state the live repro produced: exactly one chapter survives, no bare top-level string strands outside the paragraph", async () => { - // Live repro (2026-08-07): pasting a bare `\c 2` mid-chapter put a second chapter node in a - // real book/chapter/paragraph document, and every subsequent save failed with the PDP's - // "Multiple chapter markers present" — the error surfaced only in the renderer log, so disk - // and other editors silently stopped updating. Reproduced here in a realistic book+chapter+ - // paragraph document (not the bare single-paragraph fixture the pins above use) so the - // chapter-count assertion means something. + // A pasted bare `\c 2` mid-chapter must not put a second chapter node into a real + // book/chapter/paragraph document: the PDP rejects a save with a second chapter marker + // ("Multiple chapter markers present"), and that failure surfaces only in the renderer log — + // disk and other editors would silently stop reflecting further edits. Reproduced here in a + // realistic book+chapter+paragraph document (not the bare single-paragraph fixture the pins + // above use) so the chapter-count assertion means something. const { editor, text } = await bookChapterParaHost(); await pasteAndSettle(editor, () => text.select(7, 7), "\\c 5"); diff --git a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx index 4dfd7b00b..a32080b14 100644 --- a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx +++ b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx @@ -1243,11 +1243,11 @@ describe("multi-line plain-text paste inside note content", () => { it("strips \\c/\\id bytes from a multi-line paste — they never land in note content", async () => { // Single-line in-note pastes decline to the main external-paste handler // ($handlePasteForStandardView, whitespaceDisplay.plugin.utils.ts), which already strips - // `\c`/`\id`; this CRITICAL multi-line claim is a SEPARATE code path that did not share the - // strip until now. A `\c`/`\id` token landing in note content re-tokenizes through the same + // `\c`/`\id`; this CRITICAL multi-line claim is a SEPARATE code path with its own strip. A + // `\c`/`\id` token landing in note content re-tokenizes through the same // Tier 2 tokenizer a paragraph does, so it is just as reachable — and just as save-poisoning - // (a live-verified `\c` paste anywhere puts a second chapter node in the editor and fails - // every subsequent PDP save) — as one landing in body text. + // (a `\c` paste anywhere puts a second chapter node in the editor and fails every subsequent + // PDP save) — as one landing in body text. const { editor } = await renderStandardEditorWithUnclosedNote(); await pasteAt(editor, "first\n\\c 5\nlast", $selectFtEnd); @@ -1298,12 +1298,13 @@ describe("multi-line plain-text paste inside note content", () => { }); it("normalizes a marker-adjacent pasted NBSP positionally, not into `~` (the same corruption class the main paste path was fixed for)", async () => { - // Before this was wired to the shared positional rule, this claim's OWN blanket NBSP→`~` - // mapping turned the required separator after a marker's opener into data, corrupting a - // recognizable `\nd`…`\nd*` pair the same way a same-editor footnote paste once corrupted - // `\f`/`\fr`/`\ft` on the main external-paste path (whitespaceDisplay.plugin.utils.test.tsx's - // "2026-08-07 live-repro" pin). Multi-line so this CRITICAL in-note claim (not the HIGH - // external-paste handler) is the one doing the normalization. + // This claim runs its own NBSP normalization via the shared positional rule + // (`$normalizePastedNbsp`), not a blanket NBSP→`~` mapping: a blanket mapping would turn the + // required separator after a marker's opener into data, corrupting a recognizable + // `\nd`…`\nd*` pair — the same corruption class the main external-paste path guards against + // (see whitespaceDisplay.plugin.utils.test.tsx's NBSP-normalization pins). Multi-line so this + // CRITICAL in-note claim (not the HIGH external-paste handler) is the one doing the + // normalization. const { editor } = await renderStandardEditorWithUnclosedNote(); await pasteAt(editor, `\\nd${NBSP}light\\nd*\nsecond`, $selectFtEnd); diff --git a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx index 6595d185e..51fbaeb22 100644 --- a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx @@ -1,59 +1,56 @@ /** - * Optbreak (`//`) copy/paste coverage (filed 2026-08-13). Live symptom (TJ, most likely a - * pre-branch or parent-branch build): copying a selection containing an optbreak put `//` on the - * clipboard correctly, but pasting that same clipboard back did NOT insert the optbreak, while - * pasting external plaintext `//` DID. + * Optbreak (`//`) copy/paste coverage. Symptom under investigation: copying a selection containing + * an optbreak puts `//` on the clipboard correctly, but pasting that same clipboard back can fail + * to insert the optbreak, even though pasting external plaintext `//` always works. * * Three hypotheses, verified below rather than assumed: * - * - **H1** (the live symptom, on an OLD build): with the lexical flavor stripped by the async - * Ctrl+V clipboard read, Lexical's default paste prefers `text/html`; `//` renders in `text/html` - * as nothing at all (see "copy characterization" below — `UnknownNode.exportDOM` returns - * `{element: null}` unconditionally, so `@lexical/html`'s `$appendNodesToHTML` returns `false` - * before ever visiting the optbreak's display child, dropping it silently). This branch's + * - **H1** (a real native paste, whose async clipboard read strips the lexical flavor): with no + * lexical flavor to fall back on, Lexical's default paste prefers `text/html`; `//` renders in + * `text/html` as nothing at all (see "copy characterization" below — `UnknownNode.exportDOM` + * returns `{element: null}` unconditionally, so `@lexical/html`'s `$appendNodesToHTML` returns + * `false` before ever visiting the optbreak's display child, dropping it silently). * `$handlePasteForStandardView` (`whitespaceDisplay.plugin.utils.ts`) prefers `text/plain` - * whenever present, and this branch's own copy always populates `text/plain` — H1 predicts the - * symptom is already fixed here for the plain path. Pinned, not assumed, in "paste round trip" - * below. + * whenever present, and Standard view's own copy always populates `text/plain` — H1 predicts the + * symptom does not reach the plain path here. Pinned, not assumed, in "paste round trip" below. * - **H2**: `$normalizePastedNbsp`'s marker-token regexes (`whitespaceDisplay.plugin.utils.ts`) * only match `\`-shaped tokens — an NBSP adjacent to `//` would not be recognized as display - * whitespace and would fall to the blanket data-`~` rule instead. Characterized below: this - * branch's own copy walker (`$selectionToUsfmText`) inverts every TextNode's NBSP to a plain - * space unconditionally (the note-internal-separator special case aside), so `text/plain` never - * carries an NBSP next to `//` to begin with — H2 does not bite the round trip this task is - * scoped to, and doubly so: this branch's own `text/html` never carries `//` at all for an - * optbreak either (see the `text/html` pin below), so there is no `//` in P10's OWN html for a - * foreign NBSP to even land beside. A synthetic html-only foreign payload that DOES carry such - * an NBSP is characterized (not "fixed" — unreachable via this branch's own copy, reachable only - * via a genuinely foreign clipboard source) for completeness: the NBSP is genuine data from that + * whitespace and would fall to the blanket data-`~` rule instead. Characterized below: Standard + * view's own copy walker (`$selectionToUsfmText`) inverts every TextNode's NBSP to a plain space + * unconditionally (the note-internal-separator special case aside), so `text/plain` never + * carries an NBSP next to `//` to begin with — H2 does not bite the round trip this file covers, + * and doubly so: Standard view's own `text/html` never carries `//` at all for an optbreak + * either (see the `text/html` pin below), so there is no `//` in its OWN html for a foreign NBSP + * to even land beside. A synthetic html-only foreign payload that DOES carry such an NBSP is + * characterized (not "fixed" — unreachable via Standard view's own copy, reachable only via a + * genuinely foreign clipboard source) for completeness: the NBSP is genuine data from that * foreign source, and settles to the correct display form for real data-NBSP (`~`), just one * position off from where a marker-adjacent rule would have recognized it as a separator. - * - **H3** (confirmed root cause, TWO independent gaps): the same-namespace - * `application/x-lexical-editor` fast path (S2 in the clipboard semantics doc) reconstructs - * nodes via `@lexical/clipboard`'s JSON generator/parser, not DOM `importDOM`. That generator - * (`$appendNodesToJSON`, inside `@lexical/clipboard`'s `LexicalClipboard.dev.js`) computes - * `shouldExclude` from `currentNode.excludeFromCopy('html')` — the literal string `'html'`, - * hardcoded, for EVERY copy-out destination `$appendNodesToJSON` handles (`'clone'` is never - * passed by any Lexical-shipped code path in the installed version). `UnknownNode.excludeFromCopy` - * (`UnknownNode.ts`) returned `destination !== "clone"` — excluding itself unconditionally. When - * a node is excluded, `$appendNodesToJSON` does not drop it silently — it HOISTS the excluded - * node's own children into the parent's list in its place. For an optbreak, that stranded the - * `//` `ImmutableTypedTextNode` (a DecoratorNode) as a bare sibling with no owning `UnknownNode` - * wrapper: `$parseSerializedNode` on paste reconstructed a loose decorator, not a recognized - * optbreak — `$isUnknownNode` is false for it, and nothing re-tokenizes a decorator's text. - * `text/html` is UNAFFECTED by this whole mechanism: `$appendNodesToHTML` (`@lexical/html`) - * computes the same `excludeFromCopy('html')` value but returns early on `UnknownNode`'s - * unconditional `{element: null}` export BEFORE ever consulting it — so the fix below changes - * ONLY the lexical-JSON flavor. Fixed by narrowing `UnknownNode.excludeFromCopy` to leave a - * CHILD-BEARING "optbreak" out of the exclusion. A SECOND, independent gap surfaced while - * verifying that fix: a selection whose ending boundary resolves to an ELEMENT-type point ON the - * optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marked + * - **H3** (the root cause, TWO independent gaps): the same-namespace `application/x-lexical-editor` + * fast path (S2 in the clipboard semantics doc) reconstructs nodes via `@lexical/clipboard`'s + * JSON generator/parser, not DOM `importDOM`. That generator (`$appendNodesToJSON`, inside + * `@lexical/clipboard`'s `LexicalClipboard.dev.js`) computes `shouldExclude` from + * `currentNode.excludeFromCopy('html')` — the literal string `'html'`, hardcoded, for EVERY + * copy-out destination `$appendNodesToJSON` handles (`'clone'` is never passed by any + * Lexical-shipped code path in the installed version). An `UnknownNode` that excludes itself + * unconditionally from that check loses its content on exclusion: `$appendNodesToJSON` does not + * drop an excluded node silently — it HOISTS the excluded node's own children into the parent's + * list in its place. For an optbreak, that strands the `//` `ImmutableTypedTextNode` (a + * DecoratorNode) as a bare sibling with no owning `UnknownNode` wrapper: `$parseSerializedNode` + * on paste reconstructs a loose decorator, not a recognized optbreak — `$isUnknownNode` is false + * for it, and nothing re-tokenizes a decorator's text. `text/html` is UNAFFECTED by this whole + * mechanism: `$appendNodesToHTML` (`@lexical/html`) computes the same `excludeFromCopy('html')` + * value but returns early on `UnknownNode`'s unconditional `{element: null}` export BEFORE ever + * consulting it — so narrowing `UnknownNode.excludeFromCopy` to leave a CHILD-BEARING "optbreak" + * out of the exclusion (see `UnknownNode.ts`) changes ONLY the lexical-JSON flavor. The SECOND, + * independent gap: a selection whose ending boundary resolves to an ELEMENT-type point ON the + * optbreak at offset 0 (touching the wrapper without covering its own `//` child) still marks * the wrapper "selected" under Lexical's default `isSelected` (key-membership in * `selection.getNodes()`, blind to the node's own child count), producing a CHILDLESS * `{type:"optbreak"}` placeholder in the lexical-JSON copy even though the live node has a * child — disagreeing with `text/plain`, which correctly emits nothing for that same boundary. * `excludeFromCopy` cannot see this (it has no visibility into which children a given selection - * will include), so an `isSelected` override closes it directly, mirroring + * will include), so `UnknownNode`'s `isSelected` override closes it directly, mirroring * `$selectionToUsfmText`'s own `getNodes()`-based walk. Both rules key on the node's own child * count rather than on its kind, so every other `UnknownNode` construct (figure, sidebar, periph, * ref) is covered by the same predicate — their per-kind copy→paste round trips are pinned in @@ -235,15 +232,14 @@ describe("copy characterization: what the walker actually emits around an optbre }); it("carrier agreement at a childless boundary: a selection ending exactly at the optbreak's own start (touching the wrapper, not its `//` child) excludes it from BOTH carriers, not just text/plain", async () => { - // Before the `isSelected` override (`UnknownNode.ts`), this selection shape — a focus point - // resolving as an ELEMENT-type point ON the optbreak at offset 0 — still marked the WRAPPER - // "selected" under Lexical's default `isSelected` (key-membership in `selection.getNodes()`, - // blind to whether any of the wrapper's own children are actually covered), even though the - // `//` child itself was not selected. `$appendNodesToJSON` then serialized a CHILDLESS - // `{type:"unknown", tag:"optbreak", children:[]}` placeholder into the lexical-JSON payload — - // a carrier disagreement (`text/plain`, walking the same `getNodes()` list via - // `$selectionToUsfmText`, correctly emitted nothing for this boundary). Measured directly - // against this exact selection shape before writing the fix. + // Without `UnknownNode`'s `isSelected` override, this selection shape — a focus point + // resolving as an ELEMENT-type point ON the optbreak at offset 0 — would still mark the + // WRAPPER "selected" under Lexical's default `isSelected` (key-membership in + // `selection.getNodes()`, blind to whether any of the wrapper's own children are actually + // covered), even though the `//` child itself is not selected. `$appendNodesToJSON` would then + // serialize a CHILDLESS `{type:"unknown", tag:"optbreak", children:[]}` placeholder into the + // lexical-JSON payload — a carrier disagreement (`text/plain`, walking the same `getNodes()` + // list via `$selectionToUsfmText`, correctly emits nothing for this boundary). const { editor } = await renderUsjEditor(optbreakUsj()); let optbreakKey = ""; editor.getEditorState().read(() => { @@ -280,8 +276,8 @@ describe("copy characterization: what the walker actually emits around an optbre // split, usfmFragmentToUsj.ts, is a plain string `.split("//")`, unaffected by an adjacent // `~`) — asserted below by settling and checking the resulting USJ, not just the pre-settle // display text. This shape is reachable only via a foreign clipboard source supplying - // `text/html` with no `text/plain` at all — this branch's own copy never produces it (the two - // pins above). + // `text/html` with no `text/plain` at all — Standard view's own copy never produces it (the + // two pins above). let text: TextNode; const { editor } = await testEnvironment(() => { const para = $createParaNode("p"); @@ -327,8 +323,8 @@ describe("copy characterization: what the walker actually emits around an optbre }); describe("paste round trip: all three payload shapes carry a copied optbreak back", () => { - /** Harvests the genuine text/plain, text/html, and application/x-lexical-editor bytes this - * branch's own copy produces for `optbreakUsj()`'s paragraph CONTENT (marker excluded, see + /** Harvests the genuine text/plain, text/html, and application/x-lexical-editor bytes Standard + * view's own copy produces for `optbreakUsj()`'s paragraph CONTENT (marker excluded, see * `$selectParaContent`'s doc comment) — the same `COPY_COMMAND` path `$handleCopyForStandardView` * (`whitespaceDisplay.plugin.utils.ts`) runs for a real Ctrl+C. */ async function copyOptbreakContentPayload(): Promise<{ [key: string]: string }> { diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index 7953b29dc..d10f617fe 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -1084,8 +1084,9 @@ const LEADING_MARKER_LITERAL = /^\\\+?([\w-]+)(?:\*|[ \u00A0])/; * The embedded literal's marker-kind check uses `isParaKindMarker` (markerKind.utils.ts) — * stylesheet-first, UNKNOWN-AS-PARAGRAPH, the same classification `$buildParaFragment`'s own guard * below applies to the paragraph's own marker — rather than a bare `type === MarkerType.Paragraph` - * comparison, which rejected every unknown/custom.sty marker and left the stray-empty-paragraph - * bug reachable for any of them (e.g. a pasted `\zz one two`, unrecognized by the bundled sheet). + * comparison, which would reject every unknown/custom.sty marker and leave the + * stray-empty-paragraph bug reachable for any of them (e.g. a pasted `\zz one two`, unrecognized by + * the bundled sheet). * * Operates on the ALREADY-BUILT fragment (offsets into `out.text`/`out.spans`) rather than * pre-filtering which child nodes contribute, because the glyph's one-character trailing diff --git a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx index c7cddc502..b97884ee0 100644 --- a/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx @@ -1,8 +1,8 @@ /** * Copy→paste fidelity for every opaque construct the editor carries as an `UnknownNode` — figure, * sidebar, periph, ref — across the three clipboard payload shapes a real Ctrl+C/Ctrl+V produces, - * plus the table kinds that used to be `UnknownNode`s and are now their own `ImmutableTable*` - * nodes. + * plus the table kinds, which are their own dedicated `ImmutableTable*` nodes rather than + * `UnknownNode`. * * The shape under test is the one `optbreakClipboardFidelity.test.tsx` established: a payload * carrying `application/x-lexical-editor` takes Lexical's own same-namespace fast path, which diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index 56d55b811..90fe8581d 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -665,10 +665,9 @@ describe("clipboard normalization — null-event leg (ClipboardPlugin/ContextMen * appending a hidden placeholder element to the editor, pointing the DOM selection at it, and * running `document.execCommand("copy")` to provoke a real event to fill in. With nothing * selected, that filling step bails before it can suppress the browser's own copy, so the browser - * copies the placeholder and whatever the user had on the clipboard is gone. Live report - * (2026-08-26): trying to copy the marker text of a read-only construct put a single stray - * character on the clipboard; a plain collapsed caret anywhere in the view does the same, and so - * does cut. + * copies the placeholder and whatever the user had on the clipboard is gone. Without this guard, + * trying to copy the marker text of a read-only construct puts a single stray character on the + * clipboard; a plain collapsed caret anywhere in the view does the same, and so does cut. * * Two observables, because either alone can lie. `belowTheClaim` watches at a priority between this * handler's HIGH and rich-text's EDITOR: reached means the command was NOT claimed here, which is @@ -743,9 +742,9 @@ describe("copying an empty selection leaves the clipboard alone", () => { }); it("COPY_COMMAND(null) with the caret beside a read-only figure is claimed and writes nothing", async () => { - // The reported shape: a figure renders `contentEditable="false"` and its marker/attribute - // glyphs are display decorators that are not keyboard-selectable, so an attempt to select the - // marker text inside one leaves the caret in the prose beside it rather than in the figure. + // A figure renders `contentEditable="false"` and its marker/attribute glyphs are display + // decorators that are not keyboard-selectable, so an attempt to select the marker text inside + // one leaves the caret in the prose beside it rather than in the figure. const { editor } = await copyEnvironmentWithFigure(); await act(async () => editor.update(() => { @@ -802,9 +801,9 @@ describe("copying an empty selection leaves the clipboard alone", () => { * rebuilds it — and that rebuild re-derives its OWN structural separator byte regardless of * whether this handler inserted a plain space or left an NBSP, so asserting on raw post-paste * text content pins Tier 2's rebuild timing, not this handler's contract. Comparing USFM strings - * before and after a full round trip is what the live repro and this task actually care about: - * whether the marker was recognized and the data survived — not which byte an about-to-be- - * rebuilt separator held for one commit. `$selectionToUsfmText` (the copy leg) inverts every + * before and after a full round trip is what these pins care about: whether the marker was + * recognized and the data survived — not which byte an about-to-be-rebuilt separator held for one + * commit. `$selectionToUsfmText` (the copy leg) inverts every * STRUCTURAL NBSP back to a plain space regardless of which one Tier 2 chose, so the round trip * is meaningful however Tier 2 shapes the interim tree. */ @@ -887,10 +886,9 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); it("claims an external plain-text paste with no NBSP at all, inserting it unchanged", async () => { - // Previously this handler only claimed NBSP-bearing pastes; every OTHER external paste fell - // through to Lexical's own HTML/plain-text handling. It now claims every non-lexical paste — - // Standard view has no fidelity carrier but plain text, so an NBSP-free paste re-tokenizes - // the same way an NBSP-bearing one does. + // This handler claims every non-lexical paste, not just NBSP-bearing ones — Standard view has + // no fidelity carrier but plain text, so an NBSP-free paste must re-tokenize the same way an + // NBSP-bearing one does. let text: TextNode; const { editor } = await testEnvironment(() => { const para = $createParaNode("p"); @@ -1039,9 +1037,9 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); it("claims Word-style external HTML, inserting plain text with no formatting nodes", async () => { - // Previously this fell through to Lexical's own HTML import whenever text/html carried no - // NBSP, which would have created a bold-formatted TextNode from the ``. It now claims - // every non-lexical paste and never imports the html at all. + // This handler claims every non-lexical paste and never imports the html at all — a + // text/html-only fallback here would run Lexical's own HTML import instead and create a + // bold-formatted TextNode from the ``. let text: TextNode; const { editor } = await testEnvironment(() => { const para = $createParaNode("p"); @@ -1137,11 +1135,11 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); describe("positional NBSP normalization", () => { - // These replace the old blanket NBSP→`~` pins above (the ones with no marker in the pasted - // text stay unaffected — every existing NBSP-preservation pin above still holds byte-for-byte - // under the positional rule; there was simply no marker adjacency in them to normalize). + // The pins above hold byte-for-byte under the positional rule wherever the pasted text carries + // no marker adjacent to an NBSP (nothing there to normalize). These add the marker-adjacent + // cases the positional rule exists for. it("an NBSP immediately after a marker token is treated as a display artifact, not corrupted into `~` (the 2026-08-07 live-repro shape, fixed)", async () => { - // Live repro: a same-editor paste of a copied footnote turned every display-NBSP into a + // A same-editor paste of a copied footnote would otherwise turn every display-NBSP into a // literal `~`, corrupting `\f`/`\fr`/`\ft` into unknown-marker soup (see the round-trip // regression below for the full footnote). This is the minimal reproduction of one such // position — the required separator right after an opening glyph. Round-tripped through @@ -1331,10 +1329,10 @@ describe("paste normalization ($handlePasteForStandardView)", () => { describe("multi-line paste interplay (splitExpected arming)", () => { it("an NBSP-bearing multi-line paste both splits into prefixed paragraphs AND normalizes NBSPs positionally", async () => { - // Latent bug this closes: the old NBSP-only gate called `selection.insertText` on the - // WHOLE multi-line string with no paragraph-splitting logic at all, so a pasted "\n" - // landed as a literal character inside one text run instead of a paragraph break — an - // NBSP-bearing multi-line paste was never actually split. Dispatched via PASTE_COMMAND (not + // Without paragraph-splitting logic, calling `selection.insertText` on the WHOLE multi-line + // string would leave a pasted "\n" as a literal character inside one text run instead of a + // paragraph break, so an NBSP-bearing multi-line paste would never actually split. + // Dispatched via PASTE_COMMAND (not // a direct handler call) so MarkerEditPlugin's own registration arms // `context.splitExpected` before inserting; a direct call defaults that callback to a // no-op and the freshly split paragraphs would be merged back as "marker deleted". @@ -1480,7 +1478,7 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); describe("tilde-corruption regression (2026-08-07 live repro)", () => { - /** The exact live-repro footnote: `\f - \fr 1:1 \ft Caller test.\f*`. */ + /** The footnote shape this describe block pins: `\f - \fr 1:1 \ft Caller test.\f*`. */ function footnoteUsj(): Usj { return { type: "USJ", @@ -1537,7 +1535,7 @@ describe("paste normalization ($handlePasteForStandardView)", () => { await act(async () => sourceEditor.dispatchCommand(COPY_COMMAND, copyStub.event)); const sourceText = copyStub.getData("text/plain"); // Ground truth: no NBSP at all in a correct copy (matches - // clipboardCopyFidelity.test.tsx's pin for this exact live-repro footnote). + // clipboardCopyFidelity.test.tsx's pin for this exact footnote). expect(sourceText).toBe("\\f - \\fr 1:1 \\ft Caller test.\\f*"); let text: TextNode; diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index b3cd049e6..5caae6fad 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -228,8 +228,8 @@ export function $normalizePastedNbsp(text: string): string { /** A `\c`/`\id` marker token ANYWHERE in a line, capturing its payload up to — but not including — * the next marker or line end. Not anchored to the line's start: a chapter/book-id token can sit * mid-line (`x \c 5 y`, a paste landing mid-sentence), and an anchor there would silently miss it - * — the exact live-repro shape (see `$stripPastedChapterAndBookId`'s doc comment). Global so more - * than one occurrence on the same line is fully swept, not just the first. */ + * (see `$stripPastedChapterAndBookId`'s doc comment). Global so more than one occurrence on the + * same line is fully swept, not just the first. */ const CHAPTER_OR_BOOK_ID_TOKEN = /\\(?:c|id)(?![\w-])[^\n\\]*/g; /** @@ -237,11 +237,11 @@ const CHAPTER_OR_BOOK_ID_TOKEN = /\\(?:c|id)(?![\w-])[^\n\\]*/g; * next marker or newline) before insertion. Both create a document-structural node PT9 allows * only once per book (a `ChapterNode`/`BookNode`, materialized from the marker name alone — * `usj-editor.adaptor.ts` — same as a real load), and Standard view has no per-paste "am I the - * only one" check the way an initial document load does. Live repro (2026-08-07): pasting a bare - * `\c 2` mid-chapter created a second chapter node in the editor; every subsequent save then - * failed with the PDP's "Multiple chapter markers present" (the error surfaces only in the - * renderer log — disk and other editors silently stop updating). `\id` is the book-level twin of - * the same hazard and is stripped identically. A token need not be its own line — `x \c 5 y` + * only one" check the way an initial document load does. A pasted bare `\c 2` mid-chapter would + * otherwise create a second chapter node in the editor, and every subsequent save would then fail + * with the PDP's "Multiple chapter markers present" (an error that surfaces only in the renderer + * log — disk and other editors would silently stop updating). `\id` is the book-level twin of the + * same hazard and is stripped identically. A token need not be its own line — `x \c 5 y` * mid-paragraph reaches the same tokenizer branch (chapter/book-id tokens are recognized wherever * they occur, not just at a fragment's start) and left unstripped produces the identical poisoned * shape: a second chapter node PLUS the trailing bytes (`y`) stranding as a bare top-level USJ @@ -395,13 +395,12 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * the private `application/x-lexical-editor` flavor Lexical writes on copy is not one of them, so * the `DataTransfer` `pasteSelection` (`clipboard.utils.ts`) rebuilds from `navigator.clipboard. * read()` can never contain it. A real Ctrl+V — even a same-editor paste of the editor's own - * copy — therefore always rides `text/html`/`text/plain` like any external source. Previously - * this only claimed NBSP-bearing pastes and inserted the text with a BLANKET NBSP→`~` mapping; - * live repro (2026-08-07) showed that corrupts a same-editor paste of its own copy — every - * display-NBSP (the separator after `\f`/`\fr`/`\ft`) became a literal `~`, turning recognized - * markers into unknown-marker soup — and a browser-hop `\nd …\nd*` paste came back with an - * unmatched closer. `$normalizePastedNbsp` above replaces that blanket mapping with the - * positional rule. + * copy — therefore always rides `text/html`/`text/plain` like any external source. A BLANKET + * NBSP→`~` mapping would corrupt a same-editor paste of its own copy: every display-NBSP (the + * separator after `\f`/`\fr`/`\ft`) would become a literal `~`, turning recognized markers into + * unknown-marker soup, and a browser-hop `\nd …\nd*` paste would come back with an unmatched + * closer. `$normalizePastedNbsp` above uses a positional rule instead, so only genuine data-NBSPs + * become `~`. * * A MULTI-LINE payload is replayed line by line with an `INSERT_PARAGRAPH_COMMAND` dispatch * between lines, because no USFM line can carry a newline. Going through the COMMAND (rather than @@ -420,46 +419,33 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * never runs for them); or no text can be resolved. * * The same-namespace-flavor decline is SUSPENDED whenever the selection TOUCHES attribute-display - * text at either end ({@link $isSelectionInAttributeContext}). TJ's live repro (2026-08-11, filed - * against a pre-branch build): existing span `\nd asdf|who="hi"\nd*`, caret at the end of the - * `who="hi"` run, paste plain text `sid="things"` — the `who` attribute display and the closing - * `\nd*` glyph both vanished from the editor, the pasted text rendered outside the span, and the - * saved file diverged from the editor. `sid="things"` carries no NBSP, and that pre-branch build's - * `$handlePasteForStandardView` still had its OLD gate (documented above: "previously this only - * claimed NBSP-bearing pastes") — so the most plausible mechanism is that OLD gate declining an - * NBSP-free paste outright and falling through to Lexical's own default rich-paste node insertion, - * not the same-namespace-flavor path below (this branch's OWN copy already carries `text/plain` - * unconditionally, per the doc comment above, and the "private Lexical flavor is dead on Ctrl+V" - * fact recorded in the semantics doc's S3 still holds for the reconstructed-`DataTransfer` paste - * path this repro most likely used). What both mechanisms — the old NBSP gate and this file's own - * same-namespace-flavor decline — share is the SAME failure shape once either one declines: - * Lexical's default rich-paste node insertion has no notion that an attribute run's text must stay - * inside its one tagged TextNode, and (reproduced directly on THIS branch, see - * `attributeContextPasteFidelity.test.tsx`'s "root cause" describe) merges the run, the closing - * glyph, and even the FOLLOWING paragraph's own sibling text into one plain node: the attribute - * display and the closing marker both vanish, and the pasted bytes end up loose in body content. - * Confirmed regression classes on THIS branch, closed by this suspension: (1) a live native paste - * event that still carries a same-namespace `application/x-lexical-editor` flavor (S2's own - * documented case for when that CAN still reach a handler, unlike the reconstructed-`DataTransfer` - * path); (2) a multi-line plain-text payload, which the ordinary external-paste pipeline below - * would split into real paragraphs via `INSERT_PARAGRAPH_COMMAND` — inside an attribute run that - * is exactly as destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`) landing - * wholly INSIDE one attribute node, where the ordinary pipeline's `$stripPastedChapterAndBookId` - * would eat bytes out of an attribute VALUE that were never a chapter token to begin with — bytes - * merely TOUCHING a run get that strip, since they end up as re-tokenizing paragraph content - * ({@link $insertPastedTextIntoAttributeContext}); (4) a MIXED selection (one end inside the run, one - * end outside) combined with either (1) or (2) above — the selection touches attribute context, so - * it must not be allowed to reach either risky branch merely because its OTHER end sits outside. - * Attribute value bytes are never rich content — a user cannot "type formatting" into one either — - * so this handler must always claim a paste touching one and insert it as plain text - * ({@link $insertPastedTextIntoAttributeContext}), regardless of what other MIME flavors the + * text at either end ({@link $isSelectionInAttributeContext}). Lexical's default rich-paste node + * insertion has no notion that an attribute run's text must stay inside its one tagged TextNode, + * and (see `attributeContextPasteFidelity.test.tsx`'s "root cause" describe) merges the run, the + * closing glyph, and even the FOLLOWING paragraph's own sibling text into one plain node: the + * attribute display and the closing marker both vanish, and the pasted bytes end up loose in body + * content. Regression classes closed by this suspension: (1) a live native paste event that still + * carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for + * when that CAN still reach a handler, unlike the reconstructed-`DataTransfer` path); (2) a + * multi-line plain-text payload, which the ordinary external-paste pipeline below would split into + * real paragraphs via `INSERT_PARAGRAPH_COMMAND` — inside an attribute run that is exactly as + * destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`) landing wholly INSIDE + * one attribute node, where the ordinary pipeline's `$stripPastedChapterAndBookId` would eat bytes + * out of an attribute VALUE that were never a chapter token to begin with — bytes merely TOUCHING + * a run get that strip, since they end up as re-tokenizing paragraph content + * ({@link $insertPastedTextIntoAttributeContext}); (4) a MIXED selection (one end inside the run, + * one end outside) combined with either (1) or (2) above — the selection touches attribute + * context, so it must not be allowed to reach either risky branch merely because its OTHER end + * sits outside. Attribute value bytes are never rich content — a user cannot "type formatting" + * into one either — so this handler must always claim a paste touching one and insert it as plain + * text ({@link $insertPastedTextIntoAttributeContext}), regardless of what other MIME flavors the * clipboard also carries. Structure protection still takes precedence (checked first, below): an * attribute run inside a protected document defers to the same protection contract as everything - * else. One further pre-existing precedence is UNCHANGED by this suspension: the CRITICAL-priority - * in-note multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs BEFORE this handler - * and still wins for a multi-line payload whose selection touches EXPANDED note content — an - * attribute run that happens to sit inside an expanded note's content is reached by this handler - * (and this suspension) only when that in-note claim itself declines. + * else. One further precedence is unaffected by this suspension: the CRITICAL-priority in-note + * multi-line `PASTE_COMMAND` claim (`MarkerEditPlugin.tsx`) still runs BEFORE this handler and + * still wins for a multi-line payload whose selection touches EXPANDED note content — an attribute + * run that happens to sit inside an expanded note's content is reached by this handler (and this + * suspension) only when that in-note claim itself declines. * * Mutating: call inside `editor.update()`. It inserts text, removes the selected range, and * dispatches `INSERT_PARAGRAPH_COMMAND` against the editor it reads from `$getEditor()` — none of From 9f62a0f3783600cd6857f89af635c3a678b52527 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Tue, 15 Sep 2026 13:24:44 -0500 Subject: [PATCH 18/44] fix(clipboard): close two review gaps in the copy/paste path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `UnknownNode.isSelected` answered purely on child membership, which a `NodeSelection` never satisfies — it marks a node by its own key. A construct selected outright therefore reported unselected, and `$appendNodesToJSON` hoisted its (also unselected) children, copying nothing for it. Fall back to the base key-membership answer for a node selection while keeping the child-membership rule that makes the two clipboard carriers agree at a range boundary. `ContextMenuPlugin` is exported on its own but relied on `ClipboardPlugin` to register the empty-copy guard, so a host mounting it alone got the placeholder-overwrites-clipboard bug the guard exists to prevent. Register the guard from both plugins; a duplicate registration at the same priority is inert. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/usj/ContextMenuPlugin.test.tsx | 33 ++++++++++-- .../src/plugins/usj/ContextMenuPlugin.tsx | 17 +++++-- .../src/nodes/features/UnknownNode.test.ts | 50 +++++++++++++++++++ libs/shared/src/nodes/features/UnknownNode.ts | 7 +++ 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx index 49ec6c863..0c0093bb7 100644 --- a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx +++ b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.test.tsx @@ -7,9 +7,10 @@ import { $createParaNode } from "shared"; /** * The context menu's Cut/Copy dispatch the same commands the keyboard shortcuts do, so they are - * covered by the same empty-copy guard (`registerEmptyCopyGuard`, mounted by `ClipboardPlugin` — - * both plugins ship together in every editor that mounts either). This pins that the leg really - * does go through it, rather than dispatching around it. + * covered by the same empty-copy guard (`registerEmptyCopyGuard`). This pins that the leg really + * does go through it, rather than dispatching around it — both in the shape the shipped editors + * mount (alongside `ClipboardPlugin`, which registers the guard too) and with `ContextMenuPlugin` + * mounted alone, which a host consuming the plugin on its own is free to do. * * Note where `onSelect` runs: inside `editor.update()` (see the plugin's Enter handler). That is * why the guard has to live on the COMMAND and read the live selection — a check in front of the @@ -37,7 +38,9 @@ afterEach(async () => { Reflect.deleteProperty(document, "execCommand"); }); -async function contextMenuEnvironment(): Promise<{ editor: LexicalEditor; text: TextNode }> { +async function contextMenuEnvironment( + withClipboardPlugin = true, +): Promise<{ editor: LexicalEditor; text: TextNode }> { let text: TextNode | undefined; const { editor } = await baseTestEnvironment( () => { @@ -45,7 +48,7 @@ async function contextMenuEnvironment(): Promise<{ editor: LexicalEditor; text: $getRoot().append($createParaNode("p").append(text)); }, <> - + {withClipboardPlugin && } , ); @@ -107,3 +110,23 @@ describe("ContextMenuPlugin — Cut/Copy go through the empty-copy guard", () => expect(execCommand).toHaveBeenCalledWith("copy"); }); }); + +describe("ContextMenuPlugin mounted without ClipboardPlugin", () => { + it("Copy with a collapsed caret still leaves the clipboard untouched — the plugin carries its own guard", async () => { + const { editor, text } = await contextMenuEnvironment(false); + await act(async () => editor.update(() => text.select(3, 3))); + + await chooseContextMenuOption(editor, COPY_OPTION); + + expect(execCommand).not.toHaveBeenCalled(); + }); + + it("Copy with a selection still copies — the standalone guard does not over-claim", async () => { + const { editor, text } = await contextMenuEnvironment(false); + await act(async () => editor.update(() => text.select(0, text.getTextContentSize()))); + + await chooseContextMenuOption(editor, COPY_OPTION); + + expect(execCommand).toHaveBeenCalledWith("copy"); + }); +}); diff --git a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx index d7e77ea04..75adb0ade 100644 --- a/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx +++ b/libs/shared-react/src/plugins/usj/ContextMenuPlugin.tsx @@ -7,6 +7,7 @@ import { cutSelection, pasteSelection, pasteSelectionAsPlainText, + registerEmptyCopyGuard, } from "./clipboard.utils"; import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext"; import { @@ -139,10 +140,10 @@ export function ContextMenuPlugin({ const options = useMemo(() => { const builtIn = [ // Cut/Copy with nothing selected leave the clipboard alone rather than writing a placeholder - // over it — `ClipboardPlugin`'s guard claims the command (see `registerEmptyCopyGuard`), so - // this plugin needs no selection check of its own. They are not disabled in that case, - // because this option list is built once per editor rather than per menu opening, so its - // `isDisabled` flags cannot track the live selection. + // over it — `registerEmptyCopyGuard` (mounted below) claims the command, so no selection + // check is needed here. They are not disabled in that case, because this option list is + // built once per editor rather than per menu opening, so its `isDisabled` flags cannot track + // the live selection. new ContextMenuOption(`Cut`, { onSelect: () => { cutSelection(editor); @@ -179,6 +180,14 @@ export function ContextMenuPlugin({ setSelectedIndex(undefined); }, []); + // This plugin is exported on its own, so a host can mount it without `ClipboardPlugin` — and its + // Cut/Copy options would then hit the unguarded synthesized-copy path, overwriting whatever the + // clipboard already held with `@lexical/clipboard`'s hidden placeholder character. Registering + // the guard here keeps the plugin self-sufficient; a second registration alongside + // `ClipboardPlugin`'s is harmless, since both listeners sit at the same priority and the first + // one to claim an empty selection stops propagation before the other runs. + useEffect(() => registerEmptyCopyGuard(editor), [editor]); + // Register context menu event on editor root useEffect(() => { const handleContextMenu = (event: MouseEvent) => { diff --git a/libs/shared/src/nodes/features/UnknownNode.test.ts b/libs/shared/src/nodes/features/UnknownNode.test.ts index 25e5c93dd..ddc059ae3 100644 --- a/libs/shared/src/nodes/features/UnknownNode.test.ts +++ b/libs/shared/src/nodes/features/UnknownNode.test.ts @@ -3,6 +3,7 @@ import { $createParaNode, ParaNode } from "../usj/ParaNode.js"; import { createBasicTestEnvironment } from "../usj/test.utils.js"; import { $createUnknownNode, UnknownNode } from "./UnknownNode.js"; import { + $createNodeSelection, $createPoint, $createRangeSelection, $createTextNode, @@ -387,4 +388,53 @@ describe("UnknownNode", () => { }); }); }); + + // The override answers on CHILD membership, which is the right test for the range selections the + // clipboard pins exercise but blind to a `NodeSelection` — that marks a node by its OWN key and + // never puts the children in `getNodes()` at all. Paired with `excludeFromCopy` above, a false + // answer there would make `$appendNodesToJSON` hoist the (also unselected) children, so a + // construct selected outright would copy as nothing. + describe("isSelected()", () => { + function $figureInDocument() { + const figure = $createUnknownNode("figure", "fig"); + figure.append($createImmutableTypedTextNode("marker", "\\fig ")); + $getRoot().append($createParaNode("p").append(figure)); + return figure; + } + + it("answers true for a node a NodeSelection holds by its own key, whose children are NOT in the selection", () => { + const { editor } = createBasicTestEnvironment([ + UnknownNode, + ImmutableTypedTextNode, + ParaNode, + ]); + editor.update(() => { + const figure = $figureInDocument(); + const selection = $createNodeSelection(); + selection.add(figure.getKey()); + $setSelection(selection); + + expect(selection.getNodes().some((node) => node.is(figure.getFirstChild()))).toBe(false); + expect(figure.isSelected()).toBe(true); + }); + }); + + it("answers false for a NodeSelection holding some other node — the key test does not over-claim", () => { + const { editor } = createBasicTestEnvironment([ + UnknownNode, + ImmutableTypedTextNode, + ParaNode, + ]); + editor.update(() => { + const figure = $figureInDocument(); + const other = $createTextNode("elsewhere"); + $getRoot().append($createParaNode("p").append(other)); + const selection = $createNodeSelection(); + selection.add(other.getKey()); + $setSelection(selection); + + expect(figure.isSelected()).toBe(false); + }); + }); + }); }); diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index d8b282673..7c3e03f87 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -3,6 +3,7 @@ import { MarkerObject } from "@eten-tech-foundation/scripture-utilities"; import { $applyNodeReplacement, $getSelection, + $isNodeSelection, BaseSelection, DOMConversionMap, DOMConversionOutput, @@ -288,6 +289,12 @@ export class UnknownNode extends ElementNode { override isSelected(selection?: BaseSelection | null): boolean { const targetSelection = selection ?? $getSelection(); if (!targetSelection) return false; + // A `NodeSelection` marks a node selected by its OWN key, never by its children's, so the + // child-membership test below cannot see one and the base predicate has to answer first. + // Answering false there does not drop the construct quietly — `$appendNodesToJSON` HOISTS its + // (also unselected) children in its place — so a node the user selected outright would copy as + // nothing at all, the same convincing-lie hazard the range case above exists to prevent. + if ($isNodeSelection(targetSelection) && super.isSelected(targetSelection)) return true; const selectedNodes = targetSelection.getNodes(); return this.getChildren().some((child) => selectedNodes.some((node) => node.is(child))); } From 6bab467331ab595f9c1620b1d9ee9d84f4d447d0 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Tue, 15 Sep 2026 15:20:51 -0500 Subject: [PATCH 19/44] fix(clipboard): close two data-fidelity defects and the review's convention gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `\c`/`\id` strip ran to the next backslash or newline, so a paste of `x \c 5 y` landed as `x ` — content past the chapter number was destroyed with no feedback. The tokenizer reads only the next whitespace-delimited word as the number (getNextWord, usfmFragmentToUsj.ts), so removing more than that was never required to prevent the second-ChapterNode poisoning. Split into CHAPTER_TOKEN (marker + separator + one word, stopping at a backslash) and BOOK_ID_TOKEN, which keeps the wider rule because an `\id` line's whole remainder genuinely is payload. The pin that had encoded the loss now asserts the content survives. `Editor.tsx`'s structureProtectionMode pass-through had no coverage: Editor and MarkerEditPlugin both default the prop to "off", so deleting the line left all 4,234 tests green while silently starving StructureKeyboardPlugin's sanitizer on a protected document's paste. Threaded the option through mountStandardViewEditor and pinned it against a real Editor mount. Also from the review: - Reuse shared-react's $hasCopyableSelection in $handleCopyForStandardView instead of respelling its negation. The two layers must agree on what "nothing to copy" means, including the node-selection carve-out, and one shared predicate is what keeps them agreeing. - Derive the four new marker regexes from ENGINE_MARKER_NAME_BYTES. Two spellings had already diverged — `[a-z0-9-]i` excludes `_`, so `\my_marker` was a marker to the strip and not to the NBSP rules, and its separator NBSP survived into content as a literal `~`. - Collapse isParaKindMarker/isCharKindMarker onto one body. The module exists so the two tiers cannot disagree; two copies of the same five lines is how they would come to. - Drop the `$` sigil from normalizePastedNbsp and stripPastedChapterAndBookId. Both are pure string functions with no editor-context requirement to document, which CLAUDE.md requires of every exported `$`-prefixed one. - Strip spec-section IDs and provenance narration from comments, per CLAUDE.md's rule that code must stand on its own. - Move the clipboard semantics doc out of docs/superpowers, which .gitignore:67 ignores — it was tracked only by a force-add, so edits to the durable contract would never have shown in git status. Fix its dead cross-reference and the stale "not yet run" row for the e2e spec. - Pin the Lexical version $selectionToUsfmText mirrors its walk from. New pins were each verified by reverting the fix and confirming they go red; the suite is green at shared 697, shared-react 1674, platform-editor 1864, with typecheck, lint and format clean. Co-Authored-By: Claude Opus 5 (1M context) --- ...rd-semantics.md => clipboard-semantics.md} | 49 ++++---- libs/shared/src/nodes/features/UnknownNode.ts | 12 +- .../editor/markerEdit/MarkerEditPlugin.tsx | 12 +- .../attributeContextPasteFidelity.test.tsx | 10 +- .../src/editor/markerEdit/markerKind.utils.ts | 24 ++-- .../markerEdit/markerPasteFidelity.test.tsx | 9 +- .../editor/markerEdit/noteEnterFp.test.tsx | 6 +- .../optbreakClipboardFidelity.test.tsx | 4 +- .../editor/markerEdit/tier2Rebuild.utils.ts | 6 +- .../whitespaceDisplay.plugin.utils.test.tsx | 86 +++++++++++++- .../whitespaceDisplay.plugin.utils.ts | 110 +++++++++++------- .../src/editor/settledGetUsj.test-helpers.tsx | 18 ++- 12 files changed, 241 insertions(+), 105 deletions(-) rename docs/{superpowers/specs/2026-08-06-clipboard-semantics.md => clipboard-semantics.md} (67%) diff --git a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md b/docs/clipboard-semantics.md similarity index 67% rename from docs/superpowers/specs/2026-08-06-clipboard-semantics.md rename to docs/clipboard-semantics.md index a7286e8f1..60d318ed7 100644 --- a/docs/superpowers/specs/2026-08-06-clipboard-semantics.md +++ b/docs/clipboard-semantics.md @@ -12,7 +12,7 @@ This document specifies the copy/paste behavior for USFM content in Platform.Bib Already ~true because markers are real text nodes; this plan closes the exceptions: -- **Note callers:** `\f` + caller + content + `\f*` with the note's USJ `caller` value (`+`, `-`, or literal) — currently the caller is silently dropped (and rewrites to `+` on re-paste). This deliberately *exceeds* P9, where plain-text copy carries only the caller glyph and full notes survive only in CF_HTML. Rationale: in P10 there is no CF_HTML fidelity carrier for external apps, and the ticket requires "copy from Standard → paste into a plain-text editor: markers present and correctly placed". +- **Note callers:** `\f` + caller + content + `\f*` with the note's USJ `caller` value (`+`, `-`, or literal) — currently the caller is silently dropped (and rewrites to `+` on re-paste). This deliberately _exceeds_ P9, where plain-text copy carries only the caller glyph and full notes survive only in CF_HTML. Rationale: in P10 there is no CF_HTML fidelity carrier for external apps, and the ticket requires "copy from Standard → paste into a plain-text editor: markers present and correctly placed". - **Collapsed notes** contribute their full (visually hidden) `\f …\f*` bytes. "What you copy" > "what you see", for fidelity. This is documented, not changed. @@ -24,7 +24,7 @@ Already ~true because markers are real text nodes; this plan closes the exceptio - **Multi-block selections:** One `\n` between blocks; a selection starting mid-paragraph omits that paragraph's own `\p ` glyph (matches P9's rendered-text copy). Pinned, not changed. -- **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection *before* calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it *likelier*: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers, both of which decide INSIDE the update where the selection is authoritative — a check in front of the dispatch would read the last COMMITTED selection, which Lexical is a microtask behind, and would silently no-op a `copy()` made in the same tick as the selection it is copying. (1) `registerEmptyCopyGuard` (`clipboard.utils.ts`, shared-react), a `COPY_COMMAND`/`CUT_COMMAND` listener `ClipboardPlugin` registers at `COMMAND_PRIORITY_LOW` — above `@lexical/rich-text`'s EDITOR fallback, which is what synthesizes the event, and below every feature handler. One registration covers every leg (the keyboard shortcuts, the context menu's Cut/Copy, `EditorRef.copy()`/`.cut()`), since all of them dispatch the same command against the same editor. (2) `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into that fallback. A real clipboard event is still declined by layer 2 at a collapsed caret (the browser already writes nothing, and declining leaves it un-`preventDefault`ed). A NodeSelection is claimed by neither layer — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it. Such a selection — measured at the Lexical level as two ELEMENT points on the parent paragraph either side of the block — reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. **Unverified assumption:** that a real drag or click over a `contentEditable=false` block is what a browser resolves INTO that selection shape (on the reasoning that it will not place a selection endpoint inside one). jsdom cannot observe DOM-range → Lexical selection resolution — the same boundary S6 already records for the WI-2 quirk — so this needs a real-browser check before it is treated as fact. If a browser instead leaves the caret collapsed beside the block, the leak is still closed (nothing is written) but the user gets nothing rather than the figure, which would be a selection-affordance gap to file separately. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, including the node-selection carve-out, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view where no Standard-view handler is registered at all, and through `EditorRef.copy()`/`.cut()`), and shared-react's `ClipboardPlugin.test.tsx` (the shortcuts, and the same-tick select-then-copy case) and `ContextMenuPlugin.test.tsx` (the menu leg). +- **Copying nothing leaves the clipboard untouched, and a read-only construct is copied by selecting it from outside:** a copy triggered by anything other than a real browser clipboard event (the Ctrl+C/Ctrl+X key handling in `ClipboardPlugin`, the context menu's Cut/Copy items, `EditorRef.copy()`/`.cut()`) dispatches `COPY_COMMAND`/`CUT_COMMAND` with a `null` payload, and a null payload has to be turned into a real clipboard event before anything can be written: `@lexical/clipboard`'s `copyToClipboard` appends a hidden placeholder element to the editor root, points the DOM selection at it, and runs `document.execCommand("copy")` to provoke one. Its own handler for that synthesized event declines an empty Lexical selection _before_ calling `preventDefault`, so the browser's default copy then runs against the DOM selection it was handed — the placeholder — and the user's real clipboard contents are replaced by a character that was never in the document. Live report (2026-08-26): trying to copy the marker text of an uneditable construct (`\fig`) yielded one stray character. The construct only makes it _likelier_: `UnknownNode` renders `contentEditable="false"` and its marker/attribute glyphs are `ImmutableTypedTextNode` decorators with `isKeyboardSelectable() === false`, so a click or drag inside one leaves the caret in the prose beside it instead of selecting anything — but a plain collapsed caret anywhere reaches the same path, in every view mode, and cut does too. **Rule:** a copy with nothing selected writes nothing at all. Enforced at two layers, both of which decide INSIDE the update where the selection is authoritative — a check in front of the dispatch would read the last COMMITTED selection, which Lexical is a microtask behind, and would silently no-op a `copy()` made in the same tick as the selection it is copying. (1) `registerEmptyCopyGuard` (`clipboard.utils.ts`, shared-react), a `COPY_COMMAND`/`CUT_COMMAND` listener `ClipboardPlugin` registers at `COMMAND_PRIORITY_LOW` — above `@lexical/rich-text`'s EDITOR fallback, which is what synthesizes the event, and below every feature handler. One registration covers every leg (the keyboard shortcuts, the context menu's Cut/Copy, `EditorRef.copy()`/`.cut()`), since all of them dispatch the same command against the same editor. (2) `$handleCopyForStandardView` CLAIMS a null-payload dispatch it cannot build data for, rather than declining it into that fallback. A real clipboard event is still declined by layer 2 at a collapsed caret (the browser already writes nothing, and declining leaves it un-`preventDefault`ed). A NodeSelection is claimed by neither layer — it has real content, and Lexical's own path copies it correctly. **How a read-only construct IS copied:** by a selection that CONTAINS it. Such a selection — measured at the Lexical level as two ELEMENT points on the parent paragraph either side of the block — reaches every one of the block's display glyphs through the same walker as ordinary prose, so it copies the construct's real USFM bytes; no node-selection state over these constructs exists to recover, and none was invented. **Unverified assumption:** that a real drag or click over a `contentEditable=false` block is what a browser resolves INTO that selection shape (on the reasoning that it will not place a selection endpoint inside one). jsdom cannot observe DOM-range → Lexical selection resolution — the same boundary S6 already records for the WI-2 quirk — so this needs a real-browser check before it is treated as fact. If a browser instead leaves the caret collapsed beside the block, the leak is still closed (nothing is written) but the user gets nothing rather than the figure, which would be a selection-affordance gap to file separately. Pinned in `whitespaceDisplay.plugin.utils.test.tsx` (`"copying an empty selection leaves the clipboard alone"`, including the node-selection carve-out, plus the figure-alone copy), `CommandMenuPlugin.gate.test.tsx` (the same rule in a hidden-marker view where no Standard-view handler is registered at all, and through `EditorRef.copy()`/`.cut()`), and shared-react's `ClipboardPlugin.test.tsx` (the shortcuts, and the same-tick select-then-copy case) and `ContextMenuPlugin.test.tsx` (the menu leg). - **Optbreak (`\optbreak`, displayed as the literal token `//`) is a non-backslash marker token the copy walker must respect like any other content:** `$selectionToUsfmText` contributes an optbreak's `//` via its `ImmutableTypedTextNode` display child (the `$isDecoratorNode` branch), with no padding added on either side — the significant flanking spaces PT9 preserves byte-for-byte around an optbreak come entirely from the surrounding plain-text nodes, never from the walker itself, and since every TextNode's NBSP is inverted to a plain space unconditionally (the same rule as any other content), `text/plain` never carries an NBSP adjacent to `//`. Pinned in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). @@ -55,11 +55,11 @@ Any paste without a same-namespace Lexical payload is treated as a USFM text fra - **One extraction layer for every paste claim:** `getPastePayload` (`whitespaceDisplay.plugin.utils.ts`) is the single place a `PASTE_COMMAND` payload is read. It performs the jsdom-safe clipboard duck-check, decodes `text/html` through `htmlPasteText`, normalizes `\r\n` and bare `\r` to `\n` BEFORE any caller tests for a line break, and reports whether the clipboard carries this editor's own `application/x-lexical-editor` flavor. Four handlers race on `PASTE_COMMAND` — the in-note `\fp` claim at CRITICAL, this Standard-view claim and the character-stack line replay at HIGH, the paragraph-split arm at LOW — and they must agree byte-for-byte on what was pasted, so none of them reads the clipboard itself. What each claim DOES with `isInternal` still differs by design (the in-note claim covers internal pastes; the Standard-view claim declines them unless the selection touches an attribute run). -- **A multi-line payload is replayed line by line through `INSERT_PARAGRAPH_COMMAND`**, not through a bare `selection.insertParagraph()` per newline. Going through the command is what makes a paste landing inside a character-style stack close and reopen that stack at every line break, exactly as Enter does — a raw split tears the span, dropping its closing marker and leaving every line after the first outside the reopened style. The selection is removed up front rather than relying on the first `insertText` to replace it, so a payload whose first line is empty (a leading newline) cannot split a still-selected range. `MarkerEditContext.splitExpected` is armed before the first line goes in — the command's own handler arms it for each dispatch, but the first line is inserted before any dispatch runs — and `Tier2Context.pasteRebuildArmed` is armed unconditionally, single-line pastes included, since a single line can trigger the own-marker-prefix dedup just as easily. Foreign HTML formatting is dropped by design — Standard view is markers-as-text, exactly like P9's Standard view where the reformat pipeline re-tokenizes everything. This generalizes the existing NBSP-gated handler and kills both live-observed corruptions plus the latent doubled-glyph path (re-imported Standard-view HTML producing a `CharNode` via `data-marker` importDOM *and* literal `\nd` text, since MarkerNode has no importDOM). +- **A multi-line payload is replayed line by line through `INSERT_PARAGRAPH_COMMAND`**, not through a bare `selection.insertParagraph()` per newline. Going through the command is what makes a paste landing inside a character-style stack close and reopen that stack at every line break, exactly as Enter does — a raw split tears the span, dropping its closing marker and leaving every line after the first outside the reopened style. The selection is removed up front rather than relying on the first `insertText` to replace it, so a payload whose first line is empty (a leading newline) cannot split a still-selected range. `MarkerEditContext.splitExpected` is armed before the first line goes in — the command's own handler arms it for each dispatch, but the first line is inserted before any dispatch runs — and `Tier2Context.pasteRebuildArmed` is armed unconditionally, single-line pastes included, since a single line can trigger the own-marker-prefix dedup just as easily. Foreign HTML formatting is dropped by design — Standard view is markers-as-text, exactly like P9's Standard view where the reformat pipeline re-tokenizes everything. This generalizes the existing NBSP-gated handler and kills both live-observed corruptions plus the latent doubled-glyph path (re-imported Standard-view HTML producing a `CharNode` via `data-marker` importDOM _and_ literal `\nd` text, since MarkerNode has no importDOM). - **A paste's provenance outlives its own commit, keyed to the pended work rather than to the clock:** the own-marker-prefix dedup (`$withoutRedundantOwnPrefix`, `tier2Rebuild.utils.ts`) drops a host paragraph's now-redundant `\p ` glyph when the pasted line carries its own, and it is deliberately a PASTE-shape recognition — typing the same bytes keeps the engine's P9-parity split-with-empty behavior. `Tier2Context.pasteRebuildArmed` answers "is this the paste's own update", which is not enough on its own: a paragraph whose re-tokenization would EJECT content out of a milestone deliberately does not settle in the commit that produced it (`markerEditTier2Trigger.utils.ts` — ejection rearranges the line under a caret the user is still on), so its rebuild reaches a later, caret-departure settle with the flag long since reset, and the host was left with a stray empty paragraph ahead of the correct one. `Tier2Context.pastePendedKeys` closes that: a pend recorded while the paste's own update is armed also records its KEY, and the settle that finally routes that key to a rebuild consumes the provenance and passes it down. Scoping it by key is what keeps it out of typed input: a key is only ever added inside a paste's own update, it is consumed by the first rebuild it reaches, and the plugin's per-commit update listener prunes any key that has stopped pending. Answering "what if the user types between the paste and the departure" precisely, since the two cases differ (both measured): typing into a DIFFERENT node pends that node's own key, which carries no provenance, so its rebuild takes the ordinary typed split-with-empty path; typing into the still-pending node ITSELF adds no key at all — the paste's key is already pended and is still what the departure settles — so the paste path runs and the dedup applies to the line those typed bytes are now part of. The second is the intended reading of provenance (the line is still the pasted line), not an escape from it. -- **Positional NBSP normalization (replaces the blanket NBSP→`~`), three-part rule:** (1) a leading NBSP — string-start or right after a newline — is a structural separator with nothing in front of it to match against and becomes a space; (2) an NBSP immediately FOLLOWING a marker token (`\marker`, nested `\+marker`, either's closer, or a milestone's anonymous `\*` self-closer) is the required opener/closer separator and becomes a space; (3) an NBSP immediately PRECEDING a marker token is a structural spacer with no source counterpart and is DROPPED entirely (neither spaced nor kept as data) — `createNote` (`usj-editor.adaptor.ts`) appends a spacer after EVERY note child, not just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller; a browser-hop/html-derived copy of a collapsed note therefore carries structural NBSPs on BOTH sides of its interior markers, not only after the opener. Every remaining NBSP is user data → `~`. This is P9's `PostprocessUsfm` model. Live repro that mandates it (2026-08-07): pasting P10's own copied footnote back produced `\f~ \fr~1:1 ~ \ft~Caller test.~ \f*` (every display-NBSP became `~`, breaking marker recognition); a browser-hop paste of `\nd …\nd*` produced `\nd~light … \nd*` with an unmatched pair. **Rule precedence when both position rules match the same NBSP:** a closed-children note shape (`\xo*` immediately followed by NBSP immediately followed by `\xt`) sits after a closing marker AND before the next opening marker at once; the after-marker rule wins and the NBSP becomes a space rather than being dropped, because `$normalizePastedNbsp`'s `.replace` chain applies `AFTER_MARKER_NBSP` before `BEFORE_MARKER_NBSP`, consuming the NBSP into a space before the before-marker pass ever sees it. The tie-break is correct, not incidental: an after-closer NBSP in ordinary body text (`\nd*and`) is a real content space that dropping would silently lose, and the case is reachable only via a foreign NBSP-preserving carrier — P10's own `text/plain` copy (S1) never emits an NBSP in this position to begin with. +- **Positional NBSP normalization (replaces the blanket NBSP→`~`), three-part rule:** (1) a leading NBSP — string-start or right after a newline — is a structural separator with nothing in front of it to match against and becomes a space; (2) an NBSP immediately FOLLOWING a marker token (`\marker`, nested `\+marker`, either's closer, or a milestone's anonymous `\*` self-closer) is the required opener/closer separator and becomes a space; (3) an NBSP immediately PRECEDING a marker token is a structural spacer with no source counterpart and is DROPPED entirely (neither spaced nor kept as data) — `createNote` (`usj-editor.adaptor.ts`) appends a spacer after EVERY note child, not just the first, so one sits directly before `\ft`/`\f*` and every other child after the caller; a browser-hop/html-derived copy of a collapsed note therefore carries structural NBSPs on BOTH sides of its interior markers, not only after the opener. Every remaining NBSP is user data → `~`. This is P9's `PostprocessUsfm` model. Live repro that mandates it (2026-08-07): pasting P10's own copied footnote back produced `\f~ \fr~1:1 ~ \ft~Caller test.~ \f*` (every display-NBSP became `~`, breaking marker recognition); a browser-hop paste of `\nd …\nd*` produced `\nd~light … \nd*` with an unmatched pair. **Rule precedence when both position rules match the same NBSP:** a closed-children note shape (`\xo*` immediately followed by NBSP immediately followed by `\xt`) sits after a closing marker AND before the next opening marker at once; the after-marker rule wins and the NBSP becomes a space rather than being dropped, because `normalizePastedNbsp`'s `.replace` chain applies `AFTER_MARKER_NBSP` before `BEFORE_MARKER_NBSP`, consuming the NBSP into a space before the before-marker pass ever sees it. The tie-break is correct, not incidental: an after-closer NBSP in ordinary body text (`\nd*and`) is a real content space that dropping would silently lose, and the case is reachable only via a foreign NBSP-preserving carrier — P10's own `text/plain` copy (S1) never emits an NBSP in this position to begin with. - **Optbreak (`//`) is a non-backslash marker token the positional NBSP rule does not recognize (characterized, not a live gap):** `AFTER_MARKER_NBSP`/`BEFORE_MARKER_NBSP` (`whitespaceDisplay.plugin.utils.ts`) only match `\`-shaped tokens, so an NBSP adjacent to `//` is not treated as display whitespace and falls to the blanket `~` rule like any other interior NBSP. This is UNREACHABLE via P10's own copy, and DOUBLY so: (1) S1 above establishes `text/plain` never carries an NBSP next to `//` to begin with, and the plain payload is always preferred when present; (2) even if `text/plain` were somehow absent, P10's own `text/html` never carries `//` at ALL for an optbreak (see the "Known Accepted Asymmetries" entry below), so there is no `//` for a foreign-sourced NBSP to even be adjacent to in P10's own html either. Reachable only via a genuinely foreign clipboard source supplying its OWN `text/html` (never P10's) with no `text/plain` at all: a synthetic `

before // after

`-shaped payload settles to `before~// after` — the optbreak token itself still recognizes correctly (the tokenizer's `//` split is a plain string `.split("//")`, unaffected by an adjacent `~`), and the NBSP itself is genuine data FROM that foreign source (not a byte P10 invented) — the `~` it becomes is the correct display form for real data-NBSP under the "every remaining NBSP is user data → `~`" rule above, just landing one position earlier than a marker-adjacent rule would recognize. Characterized (not extended/fixed — no live repro reaches it) in `optbreakClipboardFidelity.test.tsx`'s "copy characterization" describe (2026-08-13). @@ -218,7 +218,7 @@ The following items are recorded as deferred and not addressed in this plan: 5. **Custom `text/usfm` MIME type / "Copy as USFM" command:** A dedicated `text/usfm` MIME type or separate "Copy as USFM" command was considered and rejected. Once `text/plain` IS USFM (S1), a separate command adds no value; the existing single-copy-format approach is sufficient. -6. **Popover expanded-note `isStandardView` gating issue:** An issue with note-expansion behavior in popovers has an open tracking document at `docs/superpowers/specs/2026-07-07-standard-view-followups.md`. This is not addressed in the current plan. +6. **Popover expanded-note `isStandardView` gating issue:** An issue with note-expansion behavior in popovers is tracked separately and is not addressed in the current plan. 7. **Formatted-view copy includes CSS-hidden collapsed-note content as plain prose:** hidden-marker views (S5) hide a collapsed note's content — `\ft` text and the rest — from view with CSS; the content stays in the DOM/editor state rather than being excluded from it. A prose copy in these views is not scoped to only the visibly-rendered text, so the hidden collapsed-note body is included in the copied prose alongside the surrounding paragraph's own text. This is a pre-existing consequence of how hidden-marker views render (CSS-hide, not exclude) rather than something introduced by this plan, and is out of scope here. @@ -228,27 +228,26 @@ The following items are recorded as deferred and not addressed in this plan: 10. **A selection ending exactly at a TEXT-LED construct's own start carries it in the lexical flavor but not in `text/plain`:** `UnknownNode.isSelected` counts a construct selected when one of its own children is in `selection.getNodes()`. For every construct whose display bytes lead (figure, sidebar, periph, optbreak) that boundary is an element point that reaches no child, and the carriers agree. A `ref` has no display bytes of its own, so its first child is a real `TextNode` and Lexical normalizes the same boundary into a TEXT point at that child's offset 0 — the child is in `getNodes()` while contributing zero characters, so `text/plain` correctly emits none of it while the `application/x-lexical-editor` flavor carries the whole `ref` (its `loc` attribute and child text included). The narrower "covers a child's content" predicate closes the disagreement and was rejected on measurement: excluding the wrapper makes `$appendNodesToJSON` HOIST its children in its place, and `createUnknown` stamps `mode:"token"` on every text child so `$sliceSelectedTextNodeContent` never trims the zero-width one — the copy then carries `Genesis 1:1` with the `ref` node and its `loc` silently gone, trading a visible superset for an invisible structural loss. Closing this properly needs a way for a node to exclude itself AND its children from the JSON generator; `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a `children` array, so there is no such lever without a custom node class for construct content. The chosen behavior is pinned (not merely characterized) in `unknownClipboardFidelity.test.tsx`'s `"a construct's own start boundary"` describe, on content bytes, so the lossy variant cannot come back unnoticed. - ## Test Mapping (S1–S7) -| Semantic | Pinning Test | -|----------|--------------| -| **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"`, `"attributes a construct carries in bytes the display had to reconstruct"` (a collapsed note's `\cat` run; a spanning table cell's span suffix) describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | -| **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"` and `"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"` (external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction); `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | -| **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | -| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (a confirmed regression class, not the literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | -| **S4** Paste-as-plain-text (Ctrl+Shift+V) — equivalence to external paste | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"paste-as-plain-text equivalence (S4): no literal mode, plain always wins"` describe | -| **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | -| **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | -| **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | -| **S3 (peripheral divisions)** `\periph` re-tokenizes back into a `periph` division — its title, its attribute list, and the blocks it contains | `libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts` — `"usfmFragmentToUsjContent — peripheral divisions (\\periph)"` describe: the marker line split into `alt` plus attributes, the line-per-marker spelling reading identically, following blocks taken into the division, one division ending at the next, a chapter-free contentless division, a nested sidebar, the unparseable-attribute degrade to a literal paragraph, and note context opening no division at all | -| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus AND the repo's real-world fixture | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped. Its `"real-world multi-construct fixture (2sa)"` describe sweeps `libs/test-data/src/data/2sa.usj.ts` (143 top-level items, every supported construct in the shapes a real project produced) to full USJ equality, with the one inherent verse-`sid` loss asserted by name on both sides. The whole file runs in about a second | -| **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | -| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 10), asserting content BYTES rather than tags | -| **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | -| **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | -| **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | -| **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | -| **E2E** Real-browser clipboard round trip (Standard view, written, not yet run in this task) | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | +| Semantic | Pinning Test | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **S1** Copy (Standard view) — `text/plain` is valid USFM | `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"note caller fidelity"`, `"phantom-space live-repro pins (2026-08-07)"`, `"multi-paragraph selections"`, `"AttributeRunNode traversal"`, `"attributes a construct carries in bytes the display had to reconstruct"` (a collapsed note's `\cat` run; a spanning table cell's span suffix) describes; `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"clipboard normalization"` and `"copy across an UnknownNode (figure)"` describes (display-NBSP→space, `~` data-NBSP preserved, full-USFM figure byte display) | +| **S2** Paste (Standard view, internal) — Lexical node tree reconstruction | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"declines internal pastes (a same-namespace \`application/x-lexical-editor\` payload is present)"`and`"keeps current behavior (declines) when an \`application/x-lexical-editor\` payload is present, even if \`text/html\` carries NBSP"`(external-paste handler steps aside so the Lexical fast path owns the node-tree reconstruction);`packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` equivalence pins exercise the resulting tokenized tree shape end-to-end | +| **S3** Paste (Standard view, external) — plain text as fidelity carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"paste normalization ($handlePasteForStandardView)"` describe, including its `"positional NBSP normalization"`, `"multi-line paste interplay (splitExpected arming)"`, `"structure protection"`, and `"tilde-corruption regression (2026-08-07 live repro)"` sub-describes; `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"multi-line marker-bearing paste semantics (live-verified 2026-08-07)"` and `"\\c/\\id strip on paste"` describes | +| **S3 (attribute-context paste)** Paste inside a char/milestone/verse attribute display run — paste ≡ typing | `packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx` — `"typed characterization (baseline)"`, `"paste ≡ typed (TJ's repro shape)"` (undo pinned), `"root cause: a native paste event carrying a same-namespace application/x-lexical-editor flavor must not corrupt the run"` (a confirmed regression class, not the literal pre-branch mechanism — see the S3 subsection above), `"leading-space payload"`, `"replace-selection paste inside the attribute value"` (undo pinned), `"multi-line payload collapses to a single space, per newline"` (undo pinned), `"marker-bearing payload"`, `"CUT of a selection inside the attribute value"`, `"mixed selection"` (now claims the attribute path — both the flavor and multi-line corruption shapes pinned), `"mixed selection: the pasted bytes are BODY content, not attribute-value bytes"` (the `\\c` strip and the positional NBSP rule applying to a touching selection from either end, the wholly-inside boundary that keeps value-byte semantics, and a plain payload still settling identically to the same character typed over the same range), `"milestone attribute run paste"` (including the same touching-selection rule on a second run kind), and `"verse \\va run paste"` describes | +| **S4** Paste-as-plain-text (Ctrl+Shift+V) — equivalence to external paste | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"paste-as-plain-text equivalence (S4): no literal mode, plain always wins"` describe | +| **S5** Hidden-marker views — prose copy, paste gate preservation | `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"CommandMenuPlugin editable-mode gate"` and `"MarkerEditPlugin's Standard-view clipboard handlers do not leak into Formatted view"` describes | +| **S6** Cut = copy + `removeText()` — WI-2 regression pin | `packages/platform/src/editor/markerEdit/markerEditDeletion.utils.test.tsx` — `"selection-delete at a settled char-span boundary (WI-2 filed)"` (does not reproduce at the Lexical level; pinned as regression armor); `packages/platform/src/editor/markerEdit/clipboardCopyFidelity.test.tsx` — `"cut = copy + removeText"` describe pins the byte-fidelity half of the semantic (cut's clipboard bytes match copy's, and the source is removed) | +| **S7** Undo — paste and rebuilds as single step | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — the `undoAndSettle`-based pins (multi-line marker-bearing paste, `\c`/`\id` strip, no-op `\c` paste) each asserting one `UNDO_COMMAND` dispatch restores the exact pre-paste USJ; `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx` — the two one-step undo pins covering the pend-then-departure-settle shape on both carriers | +| **S3 (peripheral divisions)** `\periph` re-tokenizes back into a `periph` division — its title, its attribute list, and the blocks it contains | `libs/shared/src/converters/usfm/usfmFragmentToUsj.test.ts` — `"usfmFragmentToUsjContent — peripheral divisions (\\periph)"` describe: the marker line split into `alt` plus attributes, the line-per-marker spelling reading identically, following blocks taken into the division, one division ending at the next, a chapter-free contentless division, a nested sidebar, the unparseable-attribute degrade to a literal paragraph, and note context opening no division at all | +| **Corpus sweep** Copy→paste round trip across the shared USJ fixture corpus AND the repo's real-world fixture | `packages/platform/src/editor/markerEdit/clipboardCorpusRoundTrip.test.tsx` — `"corpus copy/paste round trip (Standard view)"` describe; three known-lossy fixtures kept as `it.skip` (see "Known Lossy Constructs" above); the corpus's own `"optional line break (optbreak)"`, `"sidebar (esb)"`, `"figure (USFM 3 attributes)"` and `"table with header and cells"` fixtures are swept clean, not skipped. Its `"real-world multi-construct fixture (2sa)"` describe sweeps `libs/test-data/src/data/2sa.usj.ts` (143 top-level items, every supported construct in the shapes a real project produced) to full USJ equality, with the one inherent verse-`sid` loss asserted by name on both sides. The whole file runs in about a second | +| **Optbreak (`//`) clipboard fidelity** Copy characterization (`text/plain`/`text/html`), paste round trip across all three real-world payload shapes (plain-only, plain+html, plain+html+lexical), undo (one step for both the lexical-flavor path and a plain-payload paste that pends until departure — S7), cut | `packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx`; the S2 fix itself lives in `libs/shared/src/nodes/features/UnknownNode.ts` (`excludeFromCopy`) | +| **Per-kind opaque constructs** Copy→paste round trip for figure/sidebar/periph/ref (and the `ImmutableTable*` kinds that used to be `UnknownNode`s) across all three payload shapes, plus the shared `isSelected` boundary rule on more than one kind | `packages/platform/src/editor/markerEdit/unknownClipboardFidelity.test.tsx` — one describe per kind. Figure, sidebar, periph and table round-trip on all three shapes; `ref` round-trips on the lexical flavor and has its plain-carrier LOSS asserted by name (USFM has no bytes at all for a `ref` wrapper). The periph describe also pins WHY the copy stays on one line, by pasting the line-per-marker spelling and asserting the division comes back empty with its paragraph beside it. Mounts the full `Editor`, not `MarkerEditPlugin` alone. Its `"a construct's own start boundary"` describe pins the `isSelected` rule for a decorator-led kind (`figure`, excluded from both carriers) and a text-led one (`ref`, kept whole in the lexical flavor — Deferred item 10), asserting content BYTES rather than tags | +| **S3 (deferred paste rebuild)** A pasted line whose Tier-2 rebuild PENDS keeps the paste-scoped own-marker-prefix dedup across the caret-departure settle | `packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx` — `"own-marker-prefix dedup survives a DEFERRED settle"` describe: a paste whose bytes eject content out of a milestone (so the rebuild waits for departure) still lets the pasted marker own the paragraph, with the pre-departure literal asserted so the pin cannot pass through the immediate-rebuild path; its sibling pins that the SAME bytes arriving without a paste behind them still split with an empty predecessor. The unit-level typed/paste split lives in `tier2Rebuild.utils.test.tsx`'s `"own-marker-prefix dedup: paste-only scoping"` describe | +| **S1 (`text/html` flavor)** Collapse-aware display-NBSP inversion for the rich-text carrier | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"text/html flavor — collapse-aware display-NBSP inversion"` describe (interior single inverted, run of 2+ and fragment edges kept, `~` untouched in both flavors, cross-span run judgment, and both flavors decoding to the same document text), including its `"invertDisplayNbspInHtml mechanics"` unit pins | +| **S1/S6 (empty copy)** A copy or cut with nothing selected writes nothing; a read-only construct is copied by a selection containing it | `packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx` — `"copying an empty selection leaves the clipboard alone"` describe (collapsed-caret copy, cut, and the caret-beside-a-figure shape from the live report), plus `"copies the figure's own bytes when the selection covers the figure alone"`; `packages/platform/src/editor/CommandMenuPlugin.gate.test.tsx` — `"copying an empty selection leaves the clipboard alone in a hidden-marker view"` (the rule holds where no Standard-view handler is registered); `libs/shared-react/src/plugins/usj/ClipboardPlugin.test.tsx` — the Ctrl+C/Ctrl+X key handling itself, with the paste keys pinned unaffected | +| **S3 (opaque construct materialized by a paste)** A pasted `\fig …\fig*` keeps its caption INSIDE the figure — the construct's own content is not ejected as paragraph prose | `packages/platform/src/editor/markerEdit/figurePasteFidelity.test.tsx` — mid-paragraph and end-of-paragraph USJ pins plus a display-byte-order pin whose oracle is a real LOAD of the same document; the fix itself lives in `libs/shared-react/src/plugins/usj/TextSpacingPlugin.tsx` (`$textNodeInUnknownTransform` ejects only into a wrapper that predates the update), with its boundary pinned in that plugin's own suite. These pins mount the full `Editor`: the ejection lives in a sibling plugin, so a `MarkerEditPlugin`-only harness reports the same paste clean | +| **E2E** Real-browser clipboard round trip (Standard view) through the real OS clipboard | `e2e-tests/tests/isolated/scripture-editor/clipboard-usfm-round-trip.spec.ts` (paranext-core repo) | --- diff --git a/libs/shared/src/nodes/features/UnknownNode.ts b/libs/shared/src/nodes/features/UnknownNode.ts index 7c3e03f87..f99b16dbc 100644 --- a/libs/shared/src/nodes/features/UnknownNode.ts +++ b/libs/shared/src/nodes/features/UnknownNode.ts @@ -213,9 +213,8 @@ export class UnknownNode extends ElementNode { // computes this same `excludeFromCopy('html')` value, but `exportDOM()` above always returns // `{element: null}` and `$appendNodesToHTML` returns early on a null `element` BEFORE ever // consulting the exclusion value it just computed, so `text/html` output is byte-identical for - // every kind whatever this predicate answers (confirmed by reading `LexicalHtml.dev.js`). - // `'clone'` is never passed by any Lexical-shipped code path in the installed version (confirmed - // by reading `LexicalClipboard.dev.js`), so an unconditional `destination !== "clone"` would + // every kind whatever this predicate answers. `'clone'` is never passed by any Lexical-shipped + // code path in the installed version, so an unconditional `destination !== "clone"` would // exclude every `UnknownNode` from the lexical-JSON flavor outright. // // Excluding a node does not drop it silently: `$appendNodesToJSON` HOISTS the excluded node's own @@ -282,10 +281,9 @@ export class UnknownNode extends ElementNode { // the copy ends up carrying the construct's CHARACTERS with the wrapper and its attributes // silently gone. That is the convincing-lie hazard this whole pair exists to prevent. Membership // keeps the construct whole, so the lexical flavor is a SUPERSET of `text/plain` at that one - // boundary rather than a structural loss. Recorded as a residual (the clipboard semantics doc's - // "Deferred / Out of Scope" list) rather than papered over; closing it needs a lever Lexical does - // not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` to return a - // `children` array, so a node cannot say "drop me AND my children". + // boundary rather than a structural loss. That residual stands deliberately: closing it needs a + // lever Lexical does not offer — `exportNodeToJSON` requires every ElementNode's `exportJSON()` + // to return a `children` array, so a node cannot say "drop me AND my children". override isSelected(selection?: BaseSelection | null): boolean { const targetSelection = selection ?? $getSelection(); if (!targetSelection) return false; diff --git a/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx b/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx index 2f617c6ff..91fbeb684 100644 --- a/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx +++ b/packages/platform/src/editor/markerEdit/MarkerEditPlugin.tsx @@ -30,8 +30,8 @@ import { $displayWhitespaceTransform, $handleCopyForStandardView, $handlePasteForStandardView, - $normalizePastedNbsp, - $stripPastedChapterAndBookId, + normalizePastedNbsp, + stripPastedChapterAndBookId, getPastePayload, } from "./whitespaceDisplay.plugin.utils"; import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext"; @@ -341,7 +341,7 @@ function registerPasteNormalization( // ($handlePasteForStandardView, whitespaceDisplay.plugin.utils.ts) at HIGH is fine // because this claim runs its own normalization below — it does not depend on that // handler running first — but it reuses that handler's exported positional rule - // (`$normalizePastedNbsp`) rather than a divergent mapping of its own, so a display-NBSP + // (`normalizePastedNbsp`) rather than a divergent mapping of its own, so a display-NBSP // here is never corrupted into data any differently than it would be outside a note. // // The claim covers editor-internal rich pastes (application/x-lexical-editor) too: @@ -363,19 +363,19 @@ function registerPasteNormalization( const pastedText = payload.text; if (pastedText.includes("\n")) { // Standard view: every pasted NBSP is normalized POSITIONALLY here, via the same - // `$normalizePastedNbsp` the Standard-view external-paste handler uses + // `normalizePastedNbsp` the Standard-view external-paste handler uses // (whitespaceDisplay.plugin.utils.ts) — a display-NBSP (the separator after // `\fr`/`\ft`, a note's inter-child spacer) settles to a space or is dropped exactly // as it would outside a note, and only genuine data survives as `~`. Inserted raw an // NBSP is indistinguishable from a display-NBSP (a plain space in a run), so // serialization would corrupt it into a plain space if left unmapped. A pasted // literal `~` is already the display form and passes through unchanged. `\c`/`\id` - // bytes are dropped first, via the same `$stripPastedChapterAndBookId` the external + // bytes are dropped first, via the same `stripPastedChapterAndBookId` the external // handler uses — note content re-tokenizes literal text through the SAME Tier 2 // tokenizer a paragraph does, so a pasted `\c`/`\id` landing here is just as reachable // (and just as save-poisoning) as one landing in body text. const noteText = isStandardView - ? $normalizePastedNbsp($stripPastedChapterAndBookId(pastedText)) + ? normalizePastedNbsp(stripPastedChapterAndBookId(pastedText)) : pastedText; const lines = noteText.split("\n"); let outcome = $handlePasteLinesInNote(lines, context.getMarker); diff --git a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx index b159436a3..e03d4c493 100644 --- a/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/attributeContextPasteFidelity.test.tsx @@ -540,7 +540,7 @@ describe("marker-bearing payload: literal value text, no strip — and what the editor.getEditorState().read(() => { // Literal bytes intact in the run — the chapter/book-id strip - // ($stripPastedChapterAndBookId) must never run against attribute-context text. + // (stripPastedChapterAndBookId) must never run against attribute-context text. expect($charAttributeRun($firstChar()).getTextContent()).toBe('|who="hi"\\c 5'); }); const usj = usjOf(editor); @@ -555,8 +555,8 @@ describe("marker-bearing payload: literal value text, no strip — and what the // re-tokenize": the attribute tag survives the insertion, but the caret-departure settle // re-tokenizes the whole paragraph and reads `\c 5` as the chapter marker it spells. Recorded // here rather than left invisible behind an unsettled assertion. The residual — marker bytes - // reaching a value at all — is the TYPED `\c` hole (Deferred item 2 in the semantics doc), - // which paste inherits by design; closing it for paste alone would eat bytes out of an + // reaching a value at all — is the TYPED `\c` hole, which paste reaches through a different + // door rather than introducing; closing it for paste alone would eat bytes out of an // attribute value that were never a chapter token, and would break the equivalence below. const pasted = await testEnvironmentWithCharSync($charFixture); await act(async () => pasted.editor.update($selectRunEnd)); @@ -856,8 +856,8 @@ describe("mixed selection: the pasted bytes are BODY content, not attribute-valu // // Settled deliberately, and the settled shape asserted concretely: the attribute tag survives // the insertion but not the caret-departure re-tokenization, so `\c 5` DOES become a chapter - // marker here — the typed `\c` hole (Deferred item 2), inherited by design rather than - // introduced. An assertion that stopped before the settle would read "safe" over a corrupt + // marker here — the typed `\c` hole, reached through a different door rather than + // introduced by paste. An assertion that stopped before the settle would read "safe" over a corrupt // document, which is the shape this suite exists to refuse. function $selectValueInterior(): void { const run = $charAttributeRun($firstChar()); diff --git a/packages/platform/src/editor/markerEdit/markerKind.utils.ts b/packages/platform/src/editor/markerEdit/markerKind.utils.ts index 7286805c3..f16902b42 100644 --- a/packages/platform/src/editor/markerEdit/markerKind.utils.ts +++ b/packages/platform/src/editor/markerEdit/markerKind.utils.ts @@ -29,20 +29,28 @@ import { isMilestoneHeuristicName, MarkerLookup, MarkerType, NoteNode } from "sh * disagree with it for any unknown/custom.sty marker. */ export function isParaKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { - const clean = marker.replace(/^\+/, ""); - if (clean === "v" || clean === "c") return false; - const kind = getMarkerFn(clean)?.type; - if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Paragraph; - if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; - return true; + return isKindMarker(marker, getMarkerFn, MarkerType.Paragraph); } /** Same-positional-kind rule for char openers (see {@link isParaKindMarker}). */ export function isCharKindMarker(marker: string, getMarkerFn: MarkerLookup): boolean { + return isKindMarker(marker, getMarkerFn, MarkerType.Character); +} + +/** + * The one body both public rules run, differing only in the kind they compare the stylesheet's + * answer against. One function rather than two parallel ones because the whole reason this module + * exists is that the two tiers must never disagree about a marker's position — and two copies of + * the same five lines is exactly how they would come to. + * + * The unknown fallback answers TRUE for BOTH kinds deliberately: a marker the effective sheet does + * not know stays wherever it was typed, so neither kind rules it out. + */ +function isKindMarker(marker: string, getMarkerFn: MarkerLookup, kind: MarkerType): boolean { const clean = marker.replace(/^\+/, ""); if (clean === "v" || clean === "c") return false; - const kind = getMarkerFn(clean)?.type; - if (kind !== undefined && kind !== MarkerType.Unknown) return kind === MarkerType.Character; + const sheetKind = getMarkerFn(clean)?.type; + if (sheetKind !== undefined && sheetKind !== MarkerType.Unknown) return sheetKind === kind; if (NoteNode.isValidMarker(clean) || isMilestoneHeuristicName(clean)) return false; return true; } diff --git a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx index ba087f7f7..eb3a2eb98 100644 --- a/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/markerPasteFidelity.test.tsx @@ -400,7 +400,12 @@ describe("\\c/\\id strip on paste", () => { // earlier pins use — the chapter-count and paragraph-content assertions already cover what // matters (the pasted "\c 5" left no trace, structural or textual, inside the paragraph). expect(topLevelBareStrings(usj)).toEqual([]); - expect((usj.content[2] as MarkerObject).content).toEqual(["before x after"]); + // The strip takes the marker and its chapter NUMBER, and stops there: the trailing "y" is + // ordinary content the user pasted and survives. ("yafter" only because this fixture splits + // "before after" at offset 7 and the payload lands between the halves.) Assert the whole + // paragraph, not just that "y" appears somewhere, so a strip that widened again — back to + // eating to the newline — fails here rather than passing on the untouched "before ". + expect((usj.content[2] as MarkerObject).content).toEqual(["before x yafter"]); }); it('paste "text \\id GEN more" mid-paragraph: still exactly one book id, one chapter, no stranded top-level string', async () => { @@ -538,7 +543,7 @@ describe("own-marker-prefix dedup survives a DEFERRED settle", () => { }); describe("paste-as-plain-text equivalence (S4): no literal mode, plain always wins", () => { - // S4 (docs/superpowers/specs/2026-08-06-clipboard-semantics.md): Ctrl+Shift+V / "paste as plain + // Paste-as-plain-text (docs/clipboard-semantics.md): Ctrl+Shift+V / "paste as plain // text" narrows the clipboard payload down to `text/plain` only, but `$handlePasteForStandardView` // reads `text/plain` unconditionally whenever it is present — the `text/html` leg only comes into // play when `text/plain` is ABSENT (`htmlPasteText(html)` fallback). So a full (plain+html) paste diff --git a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx index a32080b14..30b0b86e7 100644 --- a/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx +++ b/packages/platform/src/editor/markerEdit/noteEnterFp.test.tsx @@ -1261,7 +1261,7 @@ describe("multi-line plain-text paste inside note content", () => { it("normalizes a pasted data-NBSP with no marker adjacency to `~` (round-trips to NBSP in USJ)", async () => { // This claim outranks the standard-view paste normalization at HIGH, so it runs its own NBSP - // normalization — the SAME positional rule that path uses (`$normalizePastedNbsp`, + // normalization — the SAME positional rule that path uses (`normalizePastedNbsp`, // whitespaceDisplay.plugin.utils.ts), not a divergent mapping of its own. This payload has no // marker literal anywhere near its NBSP, so it lands as `~` (genuine data) under the // positional rule exactly as it would under a blanket one — the positional-vs-blanket @@ -1299,7 +1299,7 @@ describe("multi-line plain-text paste inside note content", () => { it("normalizes a marker-adjacent pasted NBSP positionally, not into `~` (the same corruption class the main paste path was fixed for)", async () => { // This claim runs its own NBSP normalization via the shared positional rule - // (`$normalizePastedNbsp`), not a blanket NBSP→`~` mapping: a blanket mapping would turn the + // (`normalizePastedNbsp`), not a blanket NBSP→`~` mapping: a blanket mapping would turn the // required separator after a marker's opener into data, corrupting a recognizable // `\nd`…`\nd*` pair — the same corruption class the main external-paste path guards against // (see whitespaceDisplay.plugin.utils.test.tsx's NBSP-normalization pins). Multi-line so this @@ -1333,7 +1333,7 @@ describe("multi-line plain-text paste inside note content", () => { }); it("normalizes a data-NBSP at the start of a note's SECOND \\fp line to a plain space, not `~` — the same leading-NBSP-per-line pass the main paste path uses", async () => { - // `$normalizePastedNbsp`'s leading-NBSP pass is `gm`-flagged: `^` matches right after every + // `normalizePastedNbsp`'s leading-NBSP pass is `gm`-flagged: `^` matches right after every // `\n`, not just at the very start of the whole paste (whitespaceDisplay.plugin.utils.test.tsx // pins the identical outcome for the main external-paste path). A leading NBSP reads as a // structural separator with nothing in front of it to match against, the same as a partial diff --git a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx index 51fbaeb22..d76958d3c 100644 --- a/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx +++ b/packages/platform/src/editor/markerEdit/optbreakClipboardFidelity.test.tsx @@ -13,7 +13,7 @@ * `$handlePasteForStandardView` (`whitespaceDisplay.plugin.utils.ts`) prefers `text/plain` * whenever present, and Standard view's own copy always populates `text/plain` — H1 predicts the * symptom does not reach the plain path here. Pinned, not assumed, in "paste round trip" below. - * - **H2**: `$normalizePastedNbsp`'s marker-token regexes (`whitespaceDisplay.plugin.utils.ts`) + * - **H2**: `normalizePastedNbsp`'s marker-token regexes (`whitespaceDisplay.plugin.utils.ts`) * only match `\`-shaped tokens — an NBSP adjacent to `//` would not be recognized as display * whitespace and would fall to the blanket data-`~` rule instead. Characterized below: Standard * view's own copy walker (`$selectionToUsfmText`) inverts every TextNode's NBSP to a plain space @@ -267,7 +267,7 @@ describe("copy characterization: what the walker actually emits around an optbre }); it("characterization only (not fixed — unreachable via this branch's own copy): a synthetic html-only foreign payload with an NBSP directly before `//` still settles to a real optbreak node, with the foreign source's own NBSP kept as data `~` rather than folded into a display space", async () => { - // $normalizePastedNbsp's marker-token regexes only match `\`-shaped tokens (AFTER_MARKER_NBSP / + // normalizePastedNbsp's marker-token regexes only match `\`-shaped tokens (AFTER_MARKER_NBSP / // BEFORE_MARKER_NBSP, whitespaceDisplay.plugin.utils.ts) — `//` has no backslash, so neither // pass recognizes an NBSP next to it, and the final blanket `.replaceAll(NBSP, "~")` converts // it to a literal data tilde — the correct display form for genuine data-NBSP, just one diff --git a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts index d10f617fe..5149a2b81 100644 --- a/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts +++ b/packages/platform/src/editor/markerEdit/tier2Rebuild.utils.ts @@ -12,7 +12,7 @@ import usjEditorAdaptor from "../adaptors/usj-editor.adaptor"; import { isParaKindMarker } from "./markerKind.utils"; -import { UNTERMINATED_MARKER_TAIL } from "./markerName.pattern"; +import { ENGINE_MARKER_NAME_BYTES, UNTERMINATED_MARKER_TAIL } from "./markerName.pattern"; import { $serializeExpandedNoteContent, ATOMIC_SENTINEL, @@ -1064,7 +1064,9 @@ function $appendNodesFragment( /** A paragraph-kind marker literal at the very start of a text run — the same terminated-marker * shape `TERMINATED_MARKER_IN_TEXT_REGEX` (markerEditTier2Trigger.utils.ts) recognizes anywhere * in a run, anchored here to the run's first character. */ -const LEADING_MARKER_LITERAL = /^\\\+?([\w-]+)(?:\*|[ \u00A0])/; +const LEADING_MARKER_LITERAL = new RegExp( + String.raw`^\\\+?([${ENGINE_MARKER_NAME_BYTES}]+)(?:\*|[ \u00A0])`, +); /** * "Own marker wins": when `para` already carries its own visible marker prefix (a real glyph, not diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx index 90fe8581d..291820fd8 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.test.tsx @@ -1,4 +1,5 @@ import { MarkerEditPlugin } from "./MarkerEditPlugin"; +import { mountStandardViewEditor } from "../settledGetUsj.test-helpers"; import { copyEvent, execCommandSpy, @@ -14,6 +15,8 @@ import { $handleCopyForStandardView, $handlePasteForStandardView, invertDisplayNbspInHtml, + normalizePastedNbsp, + stripPastedChapterAndBookId, } from "./whitespaceDisplay.plugin.utils"; import { displayTextToUsj } from "./whitespaceDisplay.utils"; import { act } from "@testing-library/react"; @@ -1135,6 +1138,52 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); describe("positional NBSP normalization", () => { + // Direct pins on the pure string passes. The round-trip tests below exercise the same rules + // through a real editor, but a rebuild re-derives its own separators, which hides exactly the + // marker-RECOGNITION question these two ask. + describe("marker recognition (the byte class the token regexes are built from)", () => { + it("recognizes a marker name the ENGINE class allows but canonical USFM does not", () => { + // The engine's name class is `\w` plus `-` — deliberately wider than canonical USFM, so a + // wrong-case or underscored name is still a marker-shaped run the tokenizer will read as a + // marker (markerName.pattern.ts). The positional NBSP rules have to agree with it: a name + // this normalization fails to recognize keeps its separator NBSP, which then survives into + // content as a literal `~` right where the tokenizer expects a space. + expect(normalizePastedNbsp(`\\my_marker${NBSP}light`)).toBe("\\my_marker light"); + expect(normalizePastedNbsp(`\\ND${NBSP}light`)).toBe("\\ND light"); + expect(normalizePastedNbsp(`light${NBSP}\\my_marker*`)).toBe("light\\my_marker*"); + }); + + it("leaves an NBSP that is not adjacent to any marker as data", () => { + expect(normalizePastedNbsp(`a${NBSP}b`)).toBe("a~b"); + }); + }); + + describe("chapter/book-id strip takes the token and nothing else", () => { + it("keeps content that follows the chapter number", () => { + // `\c` takes exactly one whitespace-delimited word as its number (`getNextWord`, + // usfmFragmentToUsj.ts); everything after that is the user's content, and a strip that ate + // to the newline would drop it silently. + expect(stripPastedChapterAndBookId("x \\c 5 y")).toBe("x y"); + expect(stripPastedChapterAndBookId("x \\c 5")).toBe("x "); + }); + + it("stops at the next marker rather than swallowing it", () => { + expect(stripPastedChapterAndBookId("\\c 5\\v 1 In the beginning")).toBe( + "\\v 1 In the beginning", + ); + expect(stripPastedChapterAndBookId("\\c\\nd x")).toBe("\\nd x"); + }); + + it("takes an `\\id` line's whole remainder — book code plus description are all payload", () => { + expect(stripPastedChapterAndBookId("text \\id GEN more")).toBe("text "); + }); + + it("leaves a name that merely starts with `c`/`id` alone", () => { + expect(stripPastedChapterAndBookId(`\\cls${NBSP}x`)).toBe(`\\cls${NBSP}x`); + expect(stripPastedChapterAndBookId("\\ide UTF-8")).toBe("\\ide UTF-8"); + }); + }); + // The pins above hold byte-for-byte under the positional rule wherever the pasted text carries // no marker adjacent to an NBSP (nothing there to normalize). These add the marker-adjacent // cases the positional rule exists for. @@ -1368,7 +1417,7 @@ describe("paste normalization ($handlePasteForStandardView)", () => { }); it("a leading NBSP on a SECOND line (right after an internal `\\n`, not string-start) also normalizes to a space, under the `gm`-flagged leading-NBSP pass", async () => { - // `$normalizePastedNbsp`'s leading-NBSP pass uses the `gm` flags, so `^` matches after + // `normalizePastedNbsp`'s leading-NBSP pass uses the `gm` flags, so `^` matches after // every `\n`, not just at the very start of the whole paste — a later paragraph of a // multi-line paste can itself start mid-span (a partial selection spanning a paragraph // boundary) and reads as the same structural separator the string-start case does. @@ -1475,6 +1524,41 @@ describe("paste normalization ($handlePasteForStandardView)", () => { expect(content).toContain("pasted"); }); }); + + it("the real `Editor` hands its `structureProtectionMode` down, so a protected paste declines there too", async () => { + // The pin above passes the mode straight to `MarkerEditPlugin`, which proves the handler's + // rule but not the wiring that reaches it. Both `Editor` and the plugin default the prop to + // `"off"`, so a dropped pass-through is invisible to every other test in the suite: the + // plugin would simply never be told, and claim the paste that the sanitizer must govern. + const { lexical } = await mountStandardViewEditor( + { + type: "USJ", + version: "3.1", + content: [{ type: "para", marker: "p", content: ["hello world"] }], + }, + { structureProtectionMode: "protected" }, + ); + + const verseHtml = + '

' + + '2pasted

'; + + await act(async () => + lexical.update(() => { + const paragraph = $getRoot().getChildren().filter($isParaNode)[0]; + paragraph.selectEnd(); + lexical.dispatchCommand(PASTE_COMMAND, pasteEvent({ "text/html": verseHtml }).event); + }), + ); + + lexical.getEditorState().read(() => { + // Same tell as the plugin-level pin: a claimed paste leaks the verse span's visible "2" + // as literal text, while the sanitizer strips the verse node and leaves only "pasted". + const content = $getRoot().getTextContent(); + expect(content).not.toContain("2pasted"); + expect(content).toContain("pasted"); + }); + }); }); describe("tilde-corruption regression (2026-08-07 live repro)", () => { diff --git a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts index 5caae6fad..f848bcf52 100644 --- a/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts +++ b/packages/platform/src/editor/markerEdit/whitespaceDisplay.plugin.utils.ts @@ -47,7 +47,12 @@ import { NBSP, textTypeState, } from "shared"; -import { $isImmutableNoteCallerNode, $opaqueBlockAncestor } from "shared-react"; +import { + $hasCopyableSelection, + $isImmutableNoteCallerNode, + $opaqueBlockAncestor, +} from "shared-react"; +import { ENGINE_MARKER_NAME_BYTES } from "./markerName.pattern"; /** Spaces in runs display as NBSP so they are visible while typing. */ export function $displayWhitespaceTransform(node: TextNode): void { @@ -171,7 +176,7 @@ export function getPastePayload( // TRUE of every P10 copy, and an NBSP-presence test would route P10's own round trip through // html, whose decoded text drops a collapsed note's caller entirely (it rides as a `data-caller` // attribute, never as text). A lost note caller on the editor's own copy is a worse, far likelier - // loss than a foreign clipboard's data-NBSP. Recorded in the semantics doc's deferred list. + // loss than a foreign clipboard's data-NBSP. return { text: plainText || htmlText, isInternal: !!clipboardData.getData("application/x-lexical-editor"), @@ -182,9 +187,15 @@ export function getPastePayload( * A marker token this handler recognizes for positional NBSP normalization: a plain or * nested-char marker (`\nd`, `\+nd`), either one's closer (`\nd*`, `\+nd*`), or a milestone's * anonymous self-closer (`\*`). + * + * Built from the engine's liberal name class, not the canonical one: this is RECOGNIZING a + * marker-shaped byte run in arbitrary pasted text, which is exactly the job that class exists for, + * and a name the tokenizer will read as a marker has to be read as one here too — otherwise its + * separator NBSP survives into content as a data `~`. */ -const AFTER_MARKER_NBSP = /(\\(?:\+?[a-z0-9-]+\*?|\*))\u00A0/gi; -const BEFORE_MARKER_NBSP = /\u00A0(?=\\(?:\+?[a-z0-9-]+\*?|\*))/gi; +const MARKER_TOKEN = String.raw`\\(?:\+?[${ENGINE_MARKER_NAME_BYTES}]+\*?|\*)`; +const AFTER_MARKER_NBSP = new RegExp(String.raw`(${MARKER_TOKEN})\u00A0`, "g"); +const BEFORE_MARKER_NBSP = new RegExp(String.raw`\u00A0(?=${MARKER_TOKEN})`, "g"); /** * Positional NBSP normalization for an external paste's resolved text. Standard view has no @@ -217,7 +228,7 @@ const BEFORE_MARKER_NBSP = /\u00A0(?=\\(?:\+?[a-z0-9-]+\*?|\*))/gi; * and is preserved as `~`, the same display form typed data-NBSP takes, so serialization * round-trips it to a real NBSP instead of silently collapsing it to a plain space or dropping it. */ -export function $normalizePastedNbsp(text: string): string { +export function normalizePastedNbsp(text: string): string { return text .replace(/^\u00A0/gm, " ") .replace(AFTER_MARKER_NBSP, "$1 ") @@ -225,12 +236,23 @@ export function $normalizePastedNbsp(text: string): string { .replaceAll(NBSP, "~"); } -/** A `\c`/`\id` marker token ANYWHERE in a line, capturing its payload up to — but not including — - * the next marker or line end. Not anchored to the line's start: a chapter/book-id token can sit - * mid-line (`x \c 5 y`, a paste landing mid-sentence), and an anchor there would silently miss it - * (see `$stripPastedChapterAndBookId`'s doc comment). Global so more than one occurrence on the - * same line is fully swept, not just the first. */ -const CHAPTER_OR_BOOK_ID_TOKEN = /\\(?:c|id)(?![\w-])[^\n\\]*/g; +/** A `\c` chapter token ANYWHERE in a line: the marker, its separator, and the ONE + * whitespace-delimited word the tokenizer reads as the chapter number (`getNextWord`, + * usfmFragmentToUsj.ts) — and nothing more, because everything past that word is ordinary content + * the paste must keep. Stops at a backslash too, so `\c\nd x` strips the bare `\c` and leaves + * `\nd x` whole. Not anchored to the line's start: a chapter token can sit mid-line (`x \c 5 y`, + * a paste landing mid-sentence), and an anchor there would silently miss it. Global so more than + * one occurrence on the same line is fully swept, not just the first. */ +const CHAPTER_TOKEN = new RegExp( + String.raw`\\c(?![${ENGINE_MARKER_NAME_BYTES}])[ \u00A0]*[^\s\\]*`, + "g", +); + +/** A `\id` book-id token ANYWHERE in a line, capturing its payload up to — but not including — + * the next marker or line end. Wider than {@link CHAPTER_TOKEN} on purpose: an `\id` line's whole + * remainder IS the token's payload (book code plus an optional free-text description), so there is + * no trailing content to preserve the way there is after a chapter number. */ +const BOOK_ID_TOKEN = new RegExp(String.raw`\\id(?![${ENGINE_MARKER_NAME_BYTES}])[^\n\\]*`, "g"); /** * Drops every pasted `\c`/`\id` token and its payload (the chapter number / book code, up to the @@ -248,28 +270,30 @@ const CHAPTER_OR_BOOK_ID_TOKEN = /\\(?:c|id)(?![\w-])[^\n\\]*/g; * string outside any paragraph, since a chapter token closes the enclosing paragraph the same way * it does on a real load. * + * Only the token's OWN bytes go: `\c` plus the one word the tokenizer reads as the chapter + * number. Content that merely follows it on the same line is the user's and survives — `x \c 5 y` + * strips to `x y`, not to `x `. Removing more would trade one silent corruption for another, and + * the poisoning this exists to prevent is fully undone once the marker and its number are gone. + * A token sharing a line with a LATER marker — `\c 5\v 1 In the beginning` — likewise only loses + * its own bytes: both token regexes stop at the next `\`. + * * Splits on lines and strips per line (not one global pass over the whole text) so a token that * consumes an ENTIRE line can cleanly take that line's own newline with it too (no stray empty - * paragraph left behind). A token sharing a line with a LATER marker — `\c 5\v 1 In the - * beginning` — only loses its own bytes: the token regex stops at the next `\`, leaving `\v 1 In - * the beginning` to paste normally. But `[^\n\\]*` has no such stop when nothing marker-shaped - * follows on the line: the mid-line `x \c 5 y` shape above loses the token's trailing payload TOO, - * all the way to the newline — `x \c 5 y` strips down to `x ` alone, the trailing `y` dropped - * along with the marker (pinned in `markerPasteFidelity.test.tsx`). A line that already carried no - * other content becomes empty after stripping and is dropped from the output entirely, rather than - * surviving as a blank paragraph; a line that was ALREADY blank in the source paste (nothing to do - * with `\c`/`\id`) is left alone. + * paragraph left behind). A line that already carried no other content becomes empty after + * stripping and is dropped from the output entirely, rather than surviving as a blank paragraph; + * a line that was ALREADY blank in the source paste (nothing to do with `\c`/`\id`) is left + * alone. * * Exported for the in-note CRITICAL multi-line paste claim (`MarkerEditPlugin.tsx`), which shares - * this strip the same way it shares `$normalizePastedNbsp` — a `\c`/`\id` token pasted into note + * this strip the same way it shares `normalizePastedNbsp` — a `\c`/`\id` token pasted into note * content is just as reachable (the note-content Tier 2 rebuild tokenizes literal text the same * way a paragraph rebuild does) and just as harmful there. */ -export function $stripPastedChapterAndBookId(text: string): string { +export function stripPastedChapterAndBookId(text: string): string { return text .split("\n") .map((line) => { - const stripped = line.replace(CHAPTER_OR_BOOK_ID_TOKEN, ""); + const stripped = line.replace(CHAPTER_TOKEN, "").replace(BOOK_ID_TOKEN, ""); return stripped === "" && line !== "" ? undefined : stripped; }) .filter((line): line is string => line !== undefined) @@ -327,20 +351,19 @@ function $isSelectionInAttributeContext(selection: RangeSelection): boolean { * only one whose value-byte carve-outs ({@link $insertPastedTextIntoAttributeContext}) are earned. * What earns them is paste ≡ TYPING: the bytes land where the same keystrokes would land, and this * editor's whole attribute-context contract is that a paste in a value behaves as typing there - * does. It is NOT that the bytes never re-tokenize — measured, that is false: the `"attribute"` tag - * survives the insertion, but the caret-departure settle re-tokenizes the paragraph, so a pasted - * `\c 5` in a value DOES become a chapter marker. It becomes one identically when TYPED, which is - * why this is the typed-`\c` hole the semantics doc defers (Deferred item 2), inherited rather than - * introduced. Extending the strip to cover it would eat bytes out of an attribute value that were + * does. The bytes DO still re-tokenize: the `"attribute"` tag survives the insertion, but the + * caret-departure settle re-tokenizes the paragraph, so a pasted `\c 5` in a value becomes a + * chapter marker. It becomes one identically when TYPED, so this is the typed-`\c` hole, reached + * through a different door — not something paste introduces. Extending the strip to cover it would eat bytes out of an attribute value that were * never a chapter token — the very regression the carve-out exists to prevent — and would break the * equivalence. Both halves are pinned settled, on concrete USJ, in * `attributeContextPasteFidelity.test.tsx`. * * A selection that merely TOUCHES attribute context does not land in a value at all. Removing the * range takes the bytes out of the run, and when the range covers a char span's closing glyph it - * deletes the closer too, so what arrives is ordinary paragraph content — measured both ways round, - * a range starting in body text and reaching INTO the run, and a range starting inside the run and - * reaching PAST the closer. Those bytes get body content's rules, which is a divergence from typing + * deletes the closer too, so what arrives is ordinary paragraph content. That holds both ways + * round: a range starting in body text and reaching INTO the run, and a range starting inside the + * run and reaching PAST the closer. Those bytes get body content's rules, which is a divergence from typing * in exactly the two places body paste always diverges from it. */ function $isSelectionWithinOneAttributeNode(selection: RangeSelection): boolean { @@ -381,7 +404,7 @@ function $isSelectionWithinOneAttributeNode(selection: RangeSelection): boolean */ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: string): void { const valueBytes = $isSelectionWithinOneAttributeNode(selection); - const resolved = valueBytes ? text : $normalizePastedNbsp($stripPastedChapterAndBookId(text)); + const resolved = valueBytes ? text : normalizePastedNbsp(stripPastedChapterAndBookId(text)); selection.insertText(resolved.replace(/\n/g, " ")); } @@ -399,7 +422,7 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * NBSP→`~` mapping would corrupt a same-editor paste of its own copy: every display-NBSP (the * separator after `\f`/`\fr`/`\ft`) would become a literal `~`, turning recognized markers into * unknown-marker soup, and a browser-hop `\nd …\nd*` paste would come back with an unmatched - * closer. `$normalizePastedNbsp` above uses a positional rule instead, so only genuine data-NBSPs + * closer. `normalizePastedNbsp` above uses a positional rule instead, so only genuine data-NBSPs * become `~`. * * A MULTI-LINE payload is replayed line by line with an `INSERT_PARAGRAPH_COMMAND` dispatch @@ -424,13 +447,13 @@ function $insertPastedTextIntoAttributeContext(selection: RangeSelection, text: * and (see `attributeContextPasteFidelity.test.tsx`'s "root cause" describe) merges the run, the * closing glyph, and even the FOLLOWING paragraph's own sibling text into one plain node: the * attribute display and the closing marker both vanish, and the pasted bytes end up loose in body - * content. Regression classes closed by this suspension: (1) a live native paste event that still - * carries a same-namespace `application/x-lexical-editor` flavor (S2's own documented case for - * when that CAN still reach a handler, unlike the reconstructed-`DataTransfer` path); (2) a + * content. The suspension covers: (1) a live native paste event that still carries a + * same-namespace `application/x-lexical-editor` flavor, which unlike the + * reconstructed-`DataTransfer` path can still reach a handler; (2) a * multi-line plain-text payload, which the ordinary external-paste pipeline below would split into * real paragraphs via `INSERT_PARAGRAPH_COMMAND` — inside an attribute run that is exactly as * destructive as the rich-paste shape; (3) a marker-bearing payload (`\c 5`) landing wholly INSIDE - * one attribute node, where the ordinary pipeline's `$stripPastedChapterAndBookId` would eat bytes + * one attribute node, where the ordinary pipeline's `stripPastedChapterAndBookId` would eat bytes * out of an attribute VALUE that were never a chapter token to begin with — bytes merely TOUCHING * a run get that strip, since they end up as re-tokenizing paragraph content * ({@link $insertPastedTextIntoAttributeContext}); (4) a MIXED selection (one end inside the run, @@ -481,7 +504,7 @@ export function $handlePasteForStandardView( // all) can just as easily trigger the immediate own-marker-prefix dedup rebuild (`\p one` pasted // right after an existing `\p` host's prefix) as a multi-line one can. armPasteRebuildDedup(); - const normalized = $normalizePastedNbsp($stripPastedChapterAndBookId(text)); + const normalized = normalizePastedNbsp(stripPastedChapterAndBookId(text)); const lines = normalized.split("\n"); if (lines.length < 2) { selection.insertText(normalized); @@ -546,7 +569,7 @@ function $isNoteInternalDisplaySeparator(node: TextNode): boolean { * `isLast` is what keeps it inside the selection. These bytes stand for the region AFTER the * separator, so a selection that STOPS there covers none of them: emitting anyway put * `\cat People\cat*` on the clipboard for a range the user never selected, with no `\f` opener in - * front of the run that followed (measured: `\f -\cat People\cat*`). The sibling caller rule + * front of the run that followed (`\f -\cat People\cat*`). The sibling caller rule * guards its own boundary the same way. */ function $collapsedNoteCategoryBytes(node: TextNode, isLast: boolean): string { @@ -594,7 +617,9 @@ function $startsBlockLine(node: LexicalNode): boolean { /** * Source-faithful USFM text of `selection` — the `text/plain` leg of Standard-view copy/cut. Walks - * `selection.getNodes()` the same way `RangeSelection.getTextContent()` does (single `\n` between + * `selection.getNodes()` the same way `RangeSelection.getTextContent()` does — mirrored from + * lexical 0.43.0, so a Lexical upgrade should re-read that walk and confirm it still matches + * before trusting this one — (single `\n` between * non-inline block boundaries, anchor/focus offsets respected on the boundary text nodes, * `DecoratorNode`s contributing their own text; an inline element like `AttributeRunNode` or * `NoteNode` contributes nothing itself, its children being walked as their own list entries), with @@ -762,7 +787,10 @@ export function $handleCopyForStandardView( const selection = $getSelection(); if (!$isRangeSelection(selection) || selection.isCollapsed()) { const isNullPayloadDispatch = !event || !("clipboardData" in event); - return isNullPayloadDispatch && (!selection || selection.isCollapsed()); + // The same predicate shared-react's guard uses, not a second spelling of it: the two layers + // have to agree on what "nothing to copy" means — including that a node selection has real + // content and is therefore declined here — and one shared function is what keeps them agreeing. + return isNullPayloadDispatch && !$hasCopyableSelection(); } const data = $getStandardViewClipboardData(editor); if (!data) return false; diff --git a/packages/platform/src/editor/settledGetUsj.test-helpers.tsx b/packages/platform/src/editor/settledGetUsj.test-helpers.tsx index d9056ddb5..4b62fa12a 100644 --- a/packages/platform/src/editor/settledGetUsj.test-helpers.tsx +++ b/packages/platform/src/editor/settledGetUsj.test-helpers.tsx @@ -26,7 +26,13 @@ import { NoteNode, TypedMarkNode, } from "shared"; -import { getViewOptions, STANDARD_VIEW_MODE, usjReactNodes, ViewOptions } from "shared-react"; +import { + getViewOptions, + STANDARD_VIEW_MODE, + StructureProtectionMode, + usjReactNodes, + ViewOptions, +} from "shared-react"; import { expect } from "vitest"; // Reaching inside only for tests. // eslint-disable-next-line @nx/enforce-module-boundaries @@ -82,6 +88,12 @@ type OnUsjChange = EditorProps["onUsjChange"]; interface MountOptions { onUsjChange?: OnUsjChange; scrRef?: SerializedVerseRef; + /** + * `EditorOptions.structureProtectionMode` — the host option the real `Editor` must hand down to + * `MarkerEditPlugin` (and to `StructureKeyboardPlugin`) for a protected document's paste to + * reach the sanitizer. + */ + structureProtectionMode?: StructureProtectionMode; } export function requireStandardViewOptions(): ViewOptions { @@ -119,7 +131,7 @@ export const spanUsj: Usj = { async function mountEditor( usj: Usj, view: ViewOptions, - { onUsjChange, scrRef }: MountOptions = {}, + { onUsjChange, scrRef, structureProtectionMode }: MountOptions = {}, ): Promise<{ ref: RefObject; lexical: LexicalEditor }> { const ref = createRef(); const lexicalRef = createRef(); @@ -130,7 +142,7 @@ async function mountEditor( ref={ref} defaultUsj={usj} scrRef={scrRef} - options={{ view }} + options={{ view, structureProtectionMode }} onUsjChange={onUsjChange} > {capture} From 3158b40498882242b4ff0d64c2689a509c34f01a Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Tue, 15 Sep 2026 15:51:42 -0500 Subject: [PATCH 20/44] chore: rebuild the committed platform dist and note the copy/cut change This repository commits each published package's rolled-up build output, which the eten-tech-foundation repository does not, so the dist inherited by rebasing onto this main predates every commit on this branch. Rebuilt with `npm run rebuild-committed-output`. Also record the `EditorRef.copy()`/`cut()` behavior change in the changelog. The signatures are unchanged, so a consumer upgrading gets no compile-time signal that an empty-selection copy no longer writes to the clipboard. Co-Authored-By: Claude Opus 5 (1M context) --- packages/platform/CHANGELOG.md | 8 ++++++++ packages/platform/dist/index.d.ts | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/platform/CHANGELOG.md b/packages/platform/CHANGELOG.md index d7e1ea799..5fd94e419 100644 --- a/packages/platform/CHANGELOG.md +++ b/packages/platform/CHANGELOG.md @@ -32,6 +32,14 @@ refused. The public surface grew substantially; nothing was removed. ### Changed +- **`EditorRef.copy()` and `EditorRef.cut()` with nothing selected now leave the clipboard alone.** + Previously either one, called at a collapsed caret or with no selection, still wrote to the system + clipboard — it put a lone `#` there, because `@lexical/clipboard` synthesizes a copy event by + appending a hidden placeholder element and declines to fill it in before suppressing the browser's + own copy. A host calling `copy()` speculatively therefore destroyed whatever the user had on the + clipboard. The signatures are unchanged, so this arrives with no compile-time signal: a host that + worked around the old behavior (clearing the clipboard first, or reading it back and treating `#` + as empty) should drop that workaround. - `EditorRef.insertMarker` returns `string | undefined` (was `void`) — the created node's key. - `NoteCallerOnClick` takes a 7th parameter, `getNoteIndex: () => number | undefined`. - **Marker menu descriptions no longer carry the `(basic)` token.** `usfm.sty` marks commonly-used diff --git a/packages/platform/dist/index.d.ts b/packages/platform/dist/index.d.ts index 56dd407ca..f73b7303d 100644 --- a/packages/platform/dist/index.d.ts +++ b/packages/platform/dist/index.d.ts @@ -349,12 +349,18 @@ export declare interface EditorRef { /** Redo the last undone action. */ redo(): void; /** - * Cut the selected text. + * Cut the selected text. With nothing selected it does nothing and the clipboard keeps whatever + * it already held — see {@link EditorRef.copy}. * @throws Will throw an error if the editor is in readonly mode or uses the block verse layout * (`ViewOptions.verseLayout: "block"`), which is read-only by construction. */ cut(): void; - /** Copy the selected text. */ + /** + * Copy the selected text. With nothing selected — no selection, or a collapsed caret — it does + * nothing and the clipboard keeps whatever it already held, rather than receiving a placeholder + * character the editor never contained. A selection made programmatically immediately before this + * call still copies: the guard reads the live selection, not the last committed one. + */ copy(): void; /** * Paste text at the current cursor position. From 4b802047ddb45f2829910f40ead4e79d3628d930 Mon Sep 17 00:00:00 2001 From: TJ Couch Date: Wed, 16 Sep 2026 19:00:55 -0500 Subject: [PATCH 21/44] fix(selection): snap a selection landing inside a decorator to its boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Standard view a read-only construct's display bytes — a figure's `\fig ` opener, its `|src="…"` attribute run, its `\fig*` closer — are ImmutableTypedTextNode decorators whose glyph text is an ordinary DOM text child of the decorator's own element. Lexical 0.43's $internalResolveSelectionPoint resolves a DOM point inside a decorator's element to null (the walk up from the text reaches the decorator, which is not a TextNode), $internalResolveSelectionPoints then nulls the WHOLE selection, and on commit updateDOMSelection calls removeAllRanges() because the previous selection was inside the editor. Measured effects: a mouse drag begun on the `\fig ` glyph left the editor with no selection at all, so Ctrl+C reached only the empty-copy guard and the clipboard was never written; a drag from the caption into the attribute run stopped dead at the caption's end. DecoratorBoundarySelectionPlugin (shared-react, mounted unconditionally) handles SELECTION_CHANGE_COMMAND at CRITICAL — Lexical dispatches it even for a selection it resolved to null, while the DOM selection is still intact — and moves each offending end to the decorator's boundary in its parent, the one form Lexical does resolve. A range grows outward so the glyph is wholly in or wholly out; a caret takes the nearer side. The snapped DOM points are turned into a Lexical selection in the same update via $createRangeSelectionFromDom, because moving the DOM selection alone would still leave a null pending selection for the reconciler to clear. It only acts when $getSelection() is null (a resolved selection cannot have an end inside a decorator, and reading the DOM over one would clobber a programmatically placed caret), and it steps around the note caller: TrailingNoteCaretGuardPlugin recognizes a landing on a collapsed note's caller precisely by the selection being unresolved, and supplying one there disabled that repair and left the caret in the note's hidden body (measured, pinned). Pins: six platform tests against a real `\fig` document (five were red before the plugin), two shared-react guard-rail tests. Co-Authored-By: Claude Opus 5 (1M context) --- .../DecoratorBoundarySelectionPlugin.test.tsx | 150 ++++++++++ .../usj/DecoratorBoundarySelectionPlugin.tsx | 202 ++++++++++++++ libs/shared-react/src/plugins/usj/index.ts | 1 + packages/platform/src/editor/Editor.tsx | 4 + .../decoratorBoundarySelection.test.tsx | 259 ++++++++++++++++++ 5 files changed, 616 insertions(+) create mode 100644 libs/shared-react/src/plugins/usj/DecoratorBoundarySelectionPlugin.test.tsx create mode 100644 libs/shared-react/src/plugins/usj/DecoratorBoundarySelectionPlugin.tsx create mode 100644 packages/platform/src/editor/markerEdit/decoratorBoundarySelection.test.tsx diff --git a/libs/shared-react/src/plugins/usj/DecoratorBoundarySelectionPlugin.test.tsx b/libs/shared-react/src/plugins/usj/DecoratorBoundarySelectionPlugin.test.tsx new file mode 100644 index 000000000..b2a766bcb --- /dev/null +++ b/libs/shared-react/src/plugins/usj/DecoratorBoundarySelectionPlugin.test.tsx @@ -0,0 +1,150 @@ +/** + * The two edges of {@link DecoratorBoundarySelectionPlugin}'s scope, which the figure pins in + * `decoratorBoundarySelection.test.tsx` (platform) cannot reach: the DOM point form Lexical resolves + * by a route of its own, and the one decorator whose landings belong to another rule. + */ + +import { $createImmutableNoteCallerNode } from "../../nodes/usj"; +import { DecoratorBoundarySelectionPlugin } from "./DecoratorBoundarySelectionPlugin"; +import { baseTestEnvironment } from "./react-test.utils"; +import { TrailingNoteCaretGuardPlugin } from "./TrailingNoteCaretGuardPlugin"; +import { act } from "@testing-library/react"; +import { + $createTextNode, + $getRoot, + $getSelection, + $isRangeSelection, + $isTextNode, + LexicalEditor, +} from "lexical"; +import { + $createCharNode, + $createImmutableTypedTextNode, + $createMarkerNode, + $createMarkerTrailingSeparator, + $createNoteNode, + $createParaNode, + $createUnknownNode, + $isParaNode, + CURSOR_PLACEHOLDER_CHAR, + ParaNode, +} from "shared"; +import { ReactNode } from "react"; + +/** Flush the macrotask jsdom fires its native `selectionchange` on, plus the microtasks the caret + * guards defer their work by. */ +async function flushSelectionChange(): Promise { + await act(async () => { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + }); +} + +/** Point the DOM selection at one position and let the native `selectionchange` carry it into + * Lexical, the way a click does. */ +async function putDomCaret(node: Node, offset: number): Promise { + await flushSelectionChange(); + const domSelection = document.getSelection(); + if (!domSelection) throw new Error("no DOM selection"); + await act(async () => { + domSelection.setBaseAndExtent(node, offset, node, offset); + }); + await flushSelectionChange(); +} + +/** The document's one paragraph. */ +function readParagraph(editor: LexicalEditor): ParaNode { + return editor.getEditorState().read(() => { + const para = $getRoot().getChildren().find($isParaNode); + if (!para) throw new Error("no paragraph in the tree"); + return para; + }); +} + +describe("a DOM point that IS a decorator's own element", () => { + /** `\p before \fig cap\fig*` — a read-only construct whose opening and closing USFM bytes are + * `ImmutableTypedTextNode` decorators, the shape Standard view renders. */ + async function figureEnvironment(): Promise<{ editor: LexicalEditor }> { + return baseTestEnvironment( + () => { + const figure = $createUnknownNode("figure", "fig").append( + $createImmutableTypedTextNode("marker", "\\fig "), + $createTextNode("cap"), + $createImmutableTypedTextNode("marker", "\\fig*"), + ); + $getRoot().append($createParaNode("p").append($createTextNode("before "), figure)); + }, + , + ); + } + + it("is left to Lexical, whose own answer for that form the caret guards depend on", async () => { + // The plugin's job is the form Lexical REFUSES to resolve — a point strictly INSIDE a + // decorator's element. A point that IS the element resolves by a different route: Lexical makes + // an element point beside the decorator out of it, and then deliberately nulls a selection whose + // BOTH ends came from decorator DOM, which is the arrival `TrailingNoteCaretGuardPlugin` reads. + // Answering it here would take that arrival away from the rules that can already see it. + const { editor } = await figureEnvironment(); + const openerDom = editor.getRootElement()?.querySelector('[data-text-type="marker"]'); + if (!openerDom) throw new Error("the opener glyph did not render"); + + await putDomCaret(openerDom, 0); + + editor.getEditorState().read(() => { + expect($getSelection()).toBeNull(); + }); + }); +}); + +describe("a landing on a collapsed note's caller", () => { + /** `\p before |note|` — a collapsed note nothing renders past, the shape + * `TrailingNoteCaretGuardPlugin` hosts a caret for. */ + async function noteEnvironment(children: ReactNode): Promise<{ editor: LexicalEditor }> { + return baseTestEnvironment(() => { + const note = $createNoteNode("f", "+").append( + $createMarkerNode("f", "opening"), + $createImmutableNoteCallerNode("+", "note preview"), + $createMarkerTrailingSeparator(), + $createCharNode("ft").append( + $createMarkerNode("ft", "opening"), + $createTextNode("note body"), + ), + $createMarkerNode("f", "closing"), + ); + $getRoot().append($createParaNode("p").append($createTextNode("before "), note)); + }, children); + } + + /** The caller's rendered `