From 28ef0076ad805d27f3a4cfbee6fbfe8dc0a380e9 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 5 Oct 2026 08:59:36 +0800 Subject: [PATCH 01/14] chore: OTA update v32 for Build 23 --- lib/version.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/version.ts b/lib/version.ts index 86c4bbe..fabaac3 100644 --- a/lib/version.ts +++ b/lib/version.ts @@ -4,8 +4,8 @@ export const APP_VERSION = { version: "1.0.0", // Marketing version for app stores buildNumber: 23, // Native build number (increments only for native builds) - otaVersion: 31, // OTA update version (increments for JS updates) - timestamp: "2026-10-04T08:40:35Z", // Last update timestamp + otaVersion: 32, // OTA update version (increments for JS updates) + timestamp: "2026-10-05T00:59:34Z", // Last update timestamp channel: "testflight", // Release channel }; From 44485b49dc23e6e5b772acc9a9881728a608db96 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 5 Oct 2026 15:06:20 +0800 Subject: [PATCH 02/14] Smart Review takes every flagged card in the window, with an All option and counts Each Start rebuilds the session's cards as new, so every flagged card in the window is in it; before, cards studied in an earlier session hid until due and the header counted only those leftovers. The picker shows how many cards each window holds, and the session counter counts every card in it. Co-Authored-By: Claude Opus 5.5 --- app/(tabs)/lists/study.tsx | 9 +- components/SmartReviewModal.tsx | 52 ++++++++---- lib/smart-review.test.ts | 80 ++++++++++-------- lib/smart-review.ts | 142 ++++++++++---------------------- stores/settings.ts | 2 +- 5 files changed, 133 insertions(+), 152 deletions(-) diff --git a/app/(tabs)/lists/study.tsx b/app/(tabs)/lists/study.tsx index 3ce1098..e9d342a 100644 --- a/app/(tabs)/lists/study.tsx +++ b/app/(tabs)/lists/study.tsx @@ -1651,14 +1651,17 @@ function StudyScreen() { // Store in pools — refillDeque will pull from these duePoolRef.current = dueItems; newPoolRef.current = newItems; - dueIdsRef.current = new Set(dueItems.map((item) => item.srsCard!.id)); + // A flagged-card review is built fresh as all-new cards, so its + // progress counts every card rather than only the due ones. + const countedItems = isEphemeralListId(listId) ? [...dueItems, ...newItems] : dueItems; + dueIdsRef.current = new Set(countedItems.map((item) => item.srsCard!.id)); // Seed the deque: refill from empty state pulls first ~10 const seeded = refillDeque([], 0); setCards(seeded); setCursor(0); setOriginalCardCount(seeded.length); - setDueAtStart(dueRows.length); + setDueAtStart(countedItems.length); completedSrsIdsRef.current.clear(); setSessionPhase("studying"); } else { @@ -1908,7 +1911,7 @@ function StudyScreen() { shouldReQueue = true; reQueueItem = { ...item, srsCard: rated.card }; } else if (dueIdsRef.current.has(card.id)) { - // Only count due cards toward progress (not new cards) + // Only cards counted at session start advance progress completedSrsIdsRef.current.add(card.id); } } else if (card) { diff --git a/components/SmartReviewModal.tsx b/components/SmartReviewModal.tsx index 06c1425..a5afa5b 100644 --- a/components/SmartReviewModal.tsx +++ b/components/SmartReviewModal.tsx @@ -5,11 +5,23 @@ import { Text } from "@/components/ui/text"; import { Button } from "@/components/ui/button"; import { useUserDb } from "@/db/user-provider"; import { dayResetHourAtom, smartReviewDaysAtom } from "@/stores/settings"; -import { countMarkedInWindow, getOrCreateSmartList } from "@/lib/smart-review"; +import { + countMarkedInWindow, + getOrCreateSmartList, + type SmartReviewWindow, +} from "@/lib/smart-review"; import { useRouter } from "expo-router"; import type { WordList } from "@/db/types"; -const DAY_OPTIONS = [3, 7, 14, 30] as const; +const WINDOW_OPTIONS: SmartReviewWindow[] = [3, 7, 14, 30, null]; + +function windowLabel(days: SmartReviewWindow): string { + return days == null ? "All" : `${days}d`; +} + +function windowPhrase(days: SmartReviewWindow): string { + return days == null ? "in total" : `in the last ${days} days`; +} interface SmartReviewModalProps { visible: boolean; @@ -22,24 +34,27 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo const userDb = useUserDb(); const [days, setDays] = useAtom(smartReviewDaysAtom); const [resetHour] = useAtom(dayResetHourAtom); - const [count, setCount] = useState(null); + const [counts, setCounts] = useState | null>(null); const [starting, setStarting] = useState(false); useEffect(() => { if (!visible || !userDb || !sourceList) return; let cancelled = false; - setCount(null); - countMarkedInWindow(userDb, sourceList.id, days, resetHour) - .then((n) => { - if (!cancelled) setCount(n); + setCounts(null); + Promise.all(WINDOW_OPTIONS.map((d) => countMarkedInWindow(userDb, sourceList.id, d, resetHour))) + .then((ns) => { + if (!cancelled) setCounts(new Map(WINDOW_OPTIONS.map((d, i) => [d, ns[i]]))); }) - .catch(() => { - if (!cancelled) setCount(0); + .catch((err) => { + console.error("[SmartReview] count failed:", err); + if (!cancelled) setCounts(new Map()); }); return () => { cancelled = true; }; - }, [visible, userDb, sourceList?.id, days, resetHour]); + }, [visible, userDb, sourceList?.id, resetHour]); + + const count = counts ? (counts.get(days) ?? 0) : null; async function handleStart() { if (!userDb || !sourceList || starting) return; @@ -71,15 +86,15 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo > Smart Review - Review cards you flagged on this list, prioritized by how often and how recently you - flagged or failed them. + Review every card you flagged on this list in the window, most-flagged and most-failed + first. Progress here stays separate from the list. Lookback window - {DAY_OPTIONS.map((d) => ( + {WINDOW_OPTIONS.map((d) => ( setDays(d)} className={`flex-1 items-center rounded-lg border py-2 ${ days === d ? "border-primary bg-primary/10" : "border-border" @@ -90,7 +105,10 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo days === d ? "text-primary" : "text-muted-foreground" }`} > - {d}d + {windowLabel(d)} + + + {counts ? (counts.get(d) ?? 0) : "–"} ))} @@ -104,11 +122,11 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo ) : count === 0 ? ( - No flagged cards in the last {days} days. + No flagged cards {windowPhrase(days)}. ) : ( - {count} flagged {count === 1 ? "card" : "cards"} in the last {days} days. + {count} flagged {count === 1 ? "card" : "cards"} {windowPhrase(days)}. )} diff --git a/lib/smart-review.test.ts b/lib/smart-review.test.ts index 2631f91..eacd4bd 100644 --- a/lib/smart-review.test.ts +++ b/lib/smart-review.test.ts @@ -163,6 +163,12 @@ describe("countMarkedInWindow", () => { expect(await countMarkedInWindow(db, "src-1", 60, 3)).toBe(1); }); + test("a null window counts every flag", async () => { + await insertMark(db, "m-old", 100, "src-1", daysAgoIso(80)); + await insertMark(db, "m-new", 200, "src-1", new Date().toISOString()); + expect(await countMarkedInWindow(db, "src-1", null, 3)).toBe(2); + }); + test("filters by sourceListId", async () => { const now = new Date().toISOString(); await insertMark(db, "m1", 100, "src-1", now); @@ -366,67 +372,73 @@ describe("getOrCreateSmartList", () => { expect(cards).toHaveLength(2); }); - test("additive refresh — new marks added, existing cards preserved", async () => { + test("rebuild resets studied cards to new, so every flagged card is in the session", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); - const now = new Date().toISOString(); - await insertMark(db, "m1", 100, "src-1", now); - + await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); await getOrCreateSmartList(db, source, 7, 3); - await insertMark(db, "m2", 200, "src-1", now); + await db.runAsync( + `UPDATE srs_cards SET simple_stage = 1, simple_n = 99999, simple_interval = 5 + WHERE list_id = ? AND entry_id = 100`, + ["_smart_src-1"], + ); + await getOrCreateSmartList(db, source, 7, 3); - const entries = await db.getAllAsync<{ entry_id: number }>( - `SELECT entry_id FROM list_entries WHERE list_id = ? ORDER BY entry_id`, + const cards = await db.getAllAsync<{ entry_id: number; simple_stage: number | null }>( + `SELECT entry_id, simple_stage FROM srs_cards WHERE list_id = ? ORDER BY entry_id`, ["_smart_src-1"], ); - expect(entries.map((e) => e.entry_id)).toEqual([100, 200]); + expect(cards).toEqual([ + { entry_id: 100, simple_stage: null }, + { entry_id: 200, simple_stage: null }, + ]); }); - test("cards past 'new' (simple_stage NOT NULL) keep their created_at on refresh", async () => { + test("entries that leave the window are dropped on rebuild", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); await getOrCreateSmartList(db, source, 7, 3); - // Simulate the user studying the card past the new pool - const studied = "2099-12-31T00:00:00.000Z"; - await db.runAsync( - `UPDATE srs_cards SET simple_stage = 1, simple_n = 5, simple_interval = 5, - created_at = ? WHERE list_id = ? AND entry_id = ?`, - [studied, "_smart_src-1", 100], - ); - - // Refresh + await db.runAsync(`UPDATE review_marks SET marked_at = ? WHERE id = 'm1'`, [daysAgoIso(30)]); await getOrCreateSmartList(db, source, 7, 3); - const row = await db.getFirstAsync<{ created_at: string; simple_stage: number | null }>( - `SELECT created_at, simple_stage FROM srs_cards WHERE list_id = ? AND entry_id = 100`, + const entries = await db.getAllAsync<{ entry_id: number }>( + `SELECT entry_id FROM list_entries WHERE list_id = ?`, ["_smart_src-1"], ); - expect(row?.simple_stage).toBe(1); - expect(row?.created_at).toBe(studied); + expect(entries.map((e) => e.entry_id)).toEqual([200]); }); - test("entries no longer in window are NOT removed (additive)", async () => { + test("an 'all' window takes every flag, however old", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); - await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); - await getOrCreateSmartList(db, source, 7, 3); + await insertMark(db, "m1", 100, "src-1", daysAgoIso(80)); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); - // Backdate the mark so it leaves the window - await db.runAsync( - `UPDATE review_marks SET marked_at = datetime('now', '-30 days') WHERE id = 'm1'`, - ); + await getOrCreateSmartList(db, source, null, 3); + + const cards = await db.getAllAsync(`SELECT * FROM srs_cards WHERE list_id = ?`, [ + "_smart_src-1", + ]); + expect(cards).toHaveLength(2); + }); + test("leaves the source list's own cards untouched", async () => { + const source = makeSourceList(); + await insertSourceListRow(db, source); + await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertSrsCard(db, "src-card-100", 100, "src-1"); + + await getOrCreateSmartList(db, source, 7, 3); await getOrCreateSmartList(db, source, 7, 3); - const entries = await db.getAllAsync<{ entry_id: number }>( - `SELECT entry_id FROM list_entries WHERE list_id = ?`, - ["_smart_src-1"], - ); - expect(entries.map((e) => e.entry_id)).toEqual([100]); + const own = await db.getAllAsync(`SELECT * FROM srs_cards WHERE list_id = 'src-1'`); + expect(own).toHaveLength(1); }); test("bumps updated_at and clears deleted_at", async () => { diff --git a/lib/smart-review.ts b/lib/smart-review.ts index f8b7f8c..8d13f31 100644 --- a/lib/smart-review.ts +++ b/lib/smart-review.ts @@ -36,7 +36,12 @@ export function isEphemeralListId(id: string | null | undefined): boolean { return isSmartListId(id) || isMarkedListId(id); } -function windowStartIso(days: number, resetHour: number): string { +/** Lookback window in logical days; `null` means every retained flag. */ +export type SmartReviewWindow = number | null; + +function windowStartIso(days: SmartReviewWindow, resetHour: number): string { + // Sorts before every ISO timestamp, so `marked_at >= ?` keeps every row. + if (days == null) return ""; const today = getLogicalToday(resetHour); const start = new Date(`${today}T00:00:00Z`); // Inclusive window: "7 days" means today + the previous 6 logical days. @@ -65,7 +70,7 @@ export interface SmartCandidate { export async function countMarkedInWindow( userDb: WrappedUserDb, sourceListId: string, - days: number, + days: SmartReviewWindow, resetHour: number, ): Promise { const start = windowStartIso(days, resetHour); @@ -92,7 +97,7 @@ export async function countMarkedInWindow( export async function rankSmartCandidates( userDb: WrappedUserDb, sourceListId: string, - days: number, + days: SmartReviewWindow, resetHour: number, nowMs: number = Date.now(), ): Promise { @@ -165,27 +170,21 @@ function priorityTimestamp(rank: number): string { } /** - * Idempotent create/refresh of a per-source smart-review list. + * Create or rebuild the per-source smart-review list. * - * Behavior: * - Creates `_smart_${sourceListId}` if missing, inheriting visual settings * (front/back faces, audio, animations) from the source list but always * using `simple_srs` mode so the pass/fail UX is available. - * - Computes priority ranking via {@link rankSmartCandidates}, then upserts - * list_entries and srs_cards so the new-card pool surfaces highest-priority - * cards first. `list_entries.added_at` and `srs_cards.created_at` get - * synthesised timestamps ordered by rank. - * - Cards already past the "new" stage (simple_stage IS NOT NULL) are not - * re-ranked — their SRS state and queue position is preserved. - * - Cards present in the smart list but no longer in the current window remain; - * they just don't get re-ranked. (Additive refresh.) + * - Every call rebuilds the cards from scratch: each card flagged in the window + * enters as new, ordered by {@link rankSmartCandidates}. The source list's own + * cards are never touched, and these rows never sync. * * Returns the smart list id. */ export async function getOrCreateSmartList( userDb: WrappedUserDb, sourceList: WordList, - days: number, + days: SmartReviewWindow, resetHour: number, ): Promise { const smartId = smartListIdFor(sourceList.id); @@ -229,96 +228,45 @@ export async function getOrCreateSmartList( ]); } - const ranked = await rankSmartCandidates(userDb, sourceList.id, days, resetHour); - if (ranked.length === 0) return smartId; - - const existingEntries = await userDb.getAllAsync<{ - entry_id: number; - kanji_literal: string | null; - }>( - `SELECT entry_id, kanji_literal FROM list_entries - WHERE list_id = ? AND deleted_at IS NULL`, - [smartId], - ); - const existingEntryKeys = new Set( - existingEntries.map((r) => `${r.entry_id}|${r.kanji_literal ?? ""}`), - ); + await userDb.runAsync(`DELETE FROM srs_cards WHERE list_id = ?`, [smartId]); + await userDb.runAsync(`DELETE FROM list_entries WHERE list_id = ?`, [smartId]); - const existingCards = await userDb.getAllAsync<{ - id: string; - entry_id: number; - kanji_literal: string | null; - simple_stage: number | null; - }>( - `SELECT id, entry_id, kanji_literal, simple_stage - FROM srs_cards WHERE list_id = ? AND deleted_at IS NULL`, - [smartId], - ); - const cardByKey = new Map( - existingCards.map((c) => [`${c.entry_id}|${c.kanji_literal ?? ""}`, c]), - ); + const ranked = await rankSmartCandidates(userDb, sourceList.id, days, resetHour); for (let rank = 0; rank < ranked.length; rank++) { const cand = ranked[rank]; - const key = `${cand.entryId}|${cand.kanjiLiteral ?? ""}`; + // created_at carries priority: the new-card pool reads ORDER BY created_at. const ts = priorityTimestamp(rank); - - if (existingEntryKeys.has(key)) { - await userDb.runAsync( - `UPDATE list_entries SET added_at = ?, position = ?, updated_at = ?, deleted_at = NULL - WHERE list_id = ? AND entry_id = ? - AND ((kanji_literal IS NULL AND ? IS NULL) OR kanji_literal = ?)`, - [ts, rank, now, smartId, cand.entryId, cand.kanjiLiteral, cand.kanjiLiteral], - ); - } else { - await userDb.runAsync( - `INSERT OR IGNORE INTO list_entries (id, list_id, entry_id, kanji_literal, added_at, position, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - [ - makeId(smartId, cand.entryId, cand.kanjiLiteral), - smartId, - cand.entryId, - cand.kanjiLiteral, - ts, - rank, - now, - ], - ); - } - - const card = cardByKey.get(key); - if (!card) { - // New card — insert as "new" (simple_stage NULL). created_at carries - // priority so the new-card pool's ORDER BY created_at ASC surfaces - // highest priority first. - await userDb.runAsync( - `INSERT OR IGNORE INTO srs_cards ( - id, entry_id, kanji_literal, list_id, - due, stability, difficulty, elapsed_days, scheduled_days, - reps, lapses, state, front_mode, back_mode, - created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 'kanji', 'english', ?, ?)`, - [ - makeId(`${smartId}-srs`, cand.entryId, cand.kanjiLiteral), - cand.entryId, - cand.kanjiLiteral, - smartId, - ts, - ts, - now, - ], - ); - } else if (card.simple_stage == null) { - // Existing but still in "new" pool — update created_at to new rank so - // priority reflects the latest signal. - await userDb.runAsync(`UPDATE srs_cards SET created_at = ?, updated_at = ? WHERE id = ?`, [ + await userDb.runAsync( + `INSERT OR IGNORE INTO list_entries (id, list_id, entry_id, kanji_literal, added_at, position, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [ + makeId(smartId, cand.entryId, cand.kanjiLiteral), + smartId, + cand.entryId, + cand.kanjiLiteral, + ts, + rank, + now, + ], + ); + await userDb.runAsync( + `INSERT OR IGNORE INTO srs_cards ( + id, entry_id, kanji_literal, list_id, + due, stability, difficulty, elapsed_days, scheduled_days, + reps, lapses, state, front_mode, back_mode, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 'kanji', 'english', ?, ?)`, + [ + makeId(`${smartId}-srs`, cand.entryId, cand.kanjiLiteral), + cand.entryId, + cand.kanjiLiteral, + smartId, + ts, ts, now, - card.id, - ]); - } - // Past-new cards (simple_stage NOT NULL): leave alone. Their queue - // position is now driven by simple_n / due time. + ], + ); } return smartId; diff --git a/stores/settings.ts b/stores/settings.ts index 967a8c3..a77e1cd 100644 --- a/stores/settings.ts +++ b/stores/settings.ts @@ -68,7 +68,7 @@ export const defaultSettings = Object.freeze({ flashcardSwipeAnimation: true as boolean, flashcardButtonAnimation: true as boolean, dayResetHour: 3 as number, - smartReviewDays: 7 as number, + smartReviewDays: 7 as number | null, }); export type AppSettings = typeof defaultSettings; From 6ac87c718d16a70337fd2746c0b55cfcd4f3df15 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 5 Oct 2026 15:11:12 +0800 Subject: [PATCH 03/14] Fix two study-screen regressions from the re-queue and back-counter changes "N back" read one high when the frontier card was flipped but unanswered. And with re-queued copies now carrying fresh reps/lapses, the confusion check read its cooldown from a stale in-memory row and could fire on every later fail; it now reads the cooldown from the DB. Co-Authored-By: Claude Opus 5.5 --- app/(tabs)/lists/study.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app/(tabs)/lists/study.tsx b/app/(tabs)/lists/study.tsx index e9d342a..008e325 100644 --- a/app/(tabs)/lists/study.tsx +++ b/app/(tabs)/lists/study.tsx @@ -2298,9 +2298,13 @@ function StudyScreen() { if (list?.confusionDetection === false) return; if (sessionMode === "add_order") return; - // Check cooldown: skip if we checked this card recently - if (card.lastConfusionCheck) { - const lastCheck = new Date(card.lastConfusionCheck).getTime(); + // Check cooldown against the DB: a re-queued copy's row predates this session's checks. + const checked = await userDb.getFirstAsync<{ last_confusion_check: string | null }>( + "SELECT last_confusion_check FROM srs_cards WHERE id = ?", + [card.id], + ); + if (checked?.last_confusion_check) { + const lastCheck = new Date(checked.last_confusion_check).getTime(); const cooldownMs = CONFUSION_COOLDOWN_HOURS * 60 * 60 * 1000; if (Date.now() - lastCheck < cooldownMs) return; } @@ -2889,7 +2893,8 @@ function StudyScreen() { > {isBrowsingHistory ? (() => { - const frontier = cards.findIndex((c) => c.status === "pending"); + // A flipped-but-unanswered card is the frontier too, not only a pending one. + const frontier = cards.findIndex((c) => c.status !== "rated"); const stepsBack = (frontier === -1 ? cards.length : frontier) - cursor; return `\u2190 ${stepsBack} back`; })() From 630843a3f759b4b92ae9fc91957803ad8df117fc Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 14:19:16 +0800 Subject: [PATCH 04/14] =?UTF-8?q?Teach=20the=20reader=20the=20negative=20?= =?UTF-8?q?=EF=BD=9E=E3=81=AC,=20=EF=BD=9E=E3=81=AA=E3=81=8F=20and=20?= =?UTF-8?q?=EF=BD=9E=E3=81=AA=E3=81=8F=E3=81=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 畳まぬ was printed じょう and answered たたみ; 誘わなくなった was printed いざな, a name out of JMnedict, and answered nothing. Neither form had a rule, so the chain stopped at a fragment and the fallbacks took over. Every rule fires on the surface as written: ～ぬ is also a verb ending, so without that guard the te-form rule んで→ぬ hands these rules a word they strip again and たくさんで comes back as 託す. The three bare ichidan rules also carry what the character before the ending has to be, because お小遣いがなくて otherwise reaches がる. 33 of 3,772 corpus taps change, 21 of them the point; 44 of 67,299 surfaces gain furigana and none lose or change one; the gate goes from 128 disagreeing pairs to 126. Costs read and recorded in docs/reader-lookup-decisions.md. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 51 +++++++++++- .../src/deinflect.negative-nu.test.ts | 77 ++++++++++++++++++ packages/japanese-reader/src/deinflect.ts | 81 +++++++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 packages/japanese-reader/src/deinflect.negative-nu.test.ts diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 96c58b3..3002b07 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -861,7 +861,56 @@ take a past. That was reading **any** entry with an `exp` sense as classless, and 棒 is `n,exp` — so ぼって was painted across the corpus as a te-form of the noun 棒. `exp` now has to be the only class recorded. -### Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way +#### Negative ～ぬ, ～なく and ～なくて + +Reported from a novel: 広げた風呂敷を**畳まぬ**ようなもの printed じょう over 畳 +and answered the tap with the mat たたみ, and 散歩に**誘わなく**なった printed +いざな — a girl's name out of JMnedict — and answered nothing at all. Neither +form was in the rule table. + +- **～ぬ** is the ～ない of literary prose, parallel to the ～ず rules above and + spelled out the same way per godan row, plus ichidan, する and くる. +- **～なく / ～なくて** is that negative's adverbial and its te-form, the half of + 誘わなくなった that is a verb. The adjective rule く→い already reaches 誘わない + and stops there: nothing is spelled that way, and it types the result ADJ so + no verb rule can follow it. + +Three guards, each from a measured failure: + +- **`typeIn: RAW` on every one of them.** ～ぬ is also a verb ending — the + te-form rule んで→ぬ makes たくさんで a ぬ-verb — so without it a second pass + strips the ending the first one produced: たくさんで answered 託す, んです + answered 酢, and 〜れたんだ answered 劣. All four of those vanish with RAW. +- **`stemEnd: V1_STEM` on the three bare ichidan rules**, a new field holding + what the character before the ending has to be. お小遣い**が**なくて is a + particle and ない, but が passes a length floor, so the bare rule reached がる + and answered a particle with a verb (4 taps). Only an え/い-row kana or a + kanji can be an ichidan stem; が is あ-row. +- **`minStem: 1` on the row rules, and none on せぬ / こぬ / しなく / こなく**, + which are whole words with no stem at all. The test caught that: せぬ reached + せる and す but never する. + +Measured. **Tap sweep: 33 of 3,772 corpus taps change.** 21 are the point — +知らぬ→知る (was 糠, 等, 知/ち), 知れぬ→知れる, 切れぬ→切れる, 消えぬ→消える, +使えぬ→使う, 出揃わぬ→出揃う, 出なく→出る. **Furigana sweep: 44 of 67,299 +surfaces gain a reading, 0 lose one, 0 read differently** — 似ぬ, 心得ぬ, +思えぬ, 分らなく, 聞かなくて, 鳴らなくて and the rest. Gate 98.0%, 128 +disagreeing pairs → **126**. + +**Accepted regressions**, all read in that diff: + +- 何にもせぬ (5 taps). おやじは何にもせぬ男で: the span now swallows せぬ and + shows 何にも, because せぬ→する→the suru-noun rule lands back on 何にも. The + entry shown is still the right one and two of those taps were junk before + (燃やす for せ); the ideal split, 何にも + する, needs the suru-noun rule to + refuse a negative it did not undo itself. +- はせぬ → 蓮/はす (3 taps). そんな依怙贔負はせぬ男だ is は + せぬ. Junk for junk: + it answered 蓮 in its imperative before. +- なくなった → ナウ (3 taps). 水が出なくなった now answers 出なく on the first + three characters, which is right, and the ナウ is what tapping the leftover + なった already gave everywhere else. + +## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no 敷き込む, and きこんで (four characters, squared: 16) beats 敷き + こんで diff --git a/packages/japanese-reader/src/deinflect.negative-nu.test.ts b/packages/japanese-reader/src/deinflect.negative-nu.test.ts new file mode 100644 index 0000000..1c4ff1b --- /dev/null +++ b/packages/japanese-reader/src/deinflect.negative-nu.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { deinflect } from "./deinflect"; + +function bases(word: string): string[] { + return deinflect(word).map((c) => c.word); +} + +// ～ぬ is the ～ない of literary prose and the table only had ～ず. 広げた風呂敷を +// 畳まぬ therefore resolved nothing: the tap answered the mat 畳/たたみ and the +// furigana wrote じょう, the counter, over a verb. +describe("negative ～ぬ", () => { + it("recovers godan dictionary forms", () => { + expect(bases("畳まぬ")).toContain("畳む"); + expect(bases("知らぬ")).toContain("知る"); + expect(bases("言わぬ")).toContain("言う"); + expect(bases("書かぬ")).toContain("書く"); + expect(bases("急がぬ")).toContain("急ぐ"); + expect(bases("出さぬ")).toContain("出す"); + expect(bases("待たぬ")).toContain("待つ"); + expect(bases("死なぬ")).toContain("死ぬ"); + expect(bases("呼ばぬ")).toContain("呼ぶ"); + }); + + it("recovers ichidan, suru and kuru dictionary forms", () => { + expect(bases("見ぬ")).toContain("見る"); + expect(bases("消えぬ")).toContain("消える"); + expect(bases("せぬ")).toContain("する"); + expect(bases("来ぬ")).toContain("来る"); + }); + + it("labels the reason", () => { + expect(deinflect("畳まぬ").find((c) => c.word === "畳む")?.reasons).toEqual(["negative"]); + }); + + // 死ぬ is itself a ぬ-verb, so the te-form rule んで→ぬ hands these rules a + // word that already ends in ぬ. Without RAW a second pass strips the ending + // the first one produced and たくさんで comes back as 託す. + it("fires on the text as written, never on another rule's output", () => { + expect(bases("たくさんで")).not.toContain("託す"); + // 死る is the ichidan rule reading 死 as a stem. No dictionary holds it, so + // it costs a lookup and nothing else; 死ぬ keeps answering as itself. + expect(bases("死ぬ")).toEqual(["死ぬ", "死る"]); + }); +}); + +// 誘わなくなった had no path at all: the adjective rule く→い reaches 誘わない, +// which nothing spells, and types it ADJ so no verb rule can follow. +describe("negative adverbial ～なく and ～なくて", () => { + it("recovers the verb the negative was built on", () => { + expect(bases("誘わなく")).toContain("誘う"); + expect(bases("出なく")).toContain("出る"); + expect(bases("聞かなく")).toContain("聞く"); + expect(bases("心配しなく")).toContain("心配する"); + expect(bases("来なく")).toContain("来る"); + }); + + it("handles the te-form of the negative", () => { + expect(bases("誘わなくて")).toContain("誘う"); + expect(bases("鳴らなくて")).toContain("鳴る"); + expect(bases("食べなくて")).toContain("食べる"); + }); + + it("labels the reason", () => { + expect(deinflect("誘わなく").find((c) => c.word === "誘う")?.reasons).toEqual(["negative"]); + expect(deinflect("誘わなくて").find((c) => c.word === "誘う")?.reasons).toEqual([ + "negative te-form", + ]); + }); + + // お小遣い**が**なくて is a particle and ない, not a verb: が is あ-row and + // cannot be an ichidan stem, so the bare rule must not reach がる. + it("refuses a stem that cannot be an ichidan one", () => { + expect(bases("お小遣いがなくて")).not.toContain("お小遣いがる"); + expect(bases("がなく")).not.toContain("がる"); + expect(bases("時間がなくて")).not.toContain("時間がる"); + }); +}); diff --git a/packages/japanese-reader/src/deinflect.ts b/packages/japanese-reader/src/deinflect.ts index 073877c..4c60df1 100644 --- a/packages/japanese-reader/src/deinflect.ts +++ b/packages/japanese-reader/src/deinflect.ts @@ -42,6 +42,10 @@ const NOT_MASU_STEM = ANY & ~V5; */ const RAW = 0x80; +/** An ichidan stem's last character: an え/い-row kana, or any kanji. */ +const V1_STEM = + /[えけげせぜてでねへべぺめれいきぎしじちぢにひびぴみり\u3005\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/u; + // Passive, causative, and potential forms are themselves ichidan verbs. // When they conjugate (past, negative, etc.), the V1 deinflection strips // the conjugation and outputs V1. The next step must still recognize the @@ -58,6 +62,12 @@ interface DeinflectRule { reason: string; /** Minimum stem length left after stripping `from`. Guards 1-char inputs. */ minStem?: number; + /** + * What the character before the ending has to be. Only an え/い-row kana or a + * kanji can be an ichidan stem, so without it お小遣い**が**なくて deinflects + * to がる and answers a particle with a verb. + */ + stemEnd?: RegExp; } // ─── Deinflection rules ─── @@ -355,6 +365,76 @@ const RULES: DeinflectRule[] = [ { from: "せず", to: "する", typeIn: SURU, typeOut: SURU, reason: "negative" }, { from: "こず", to: "くる", typeIn: KURU, typeOut: KURU, reason: "negative" }, + // ── Negative ～ぬ: the ～ない of literary prose, and ～なく / ～なくて ── + // Every rule here fires on the surface as written. ～ぬ is also a verb ending + // — the te-form rule んで→ぬ makes たくさんで a ぬ-verb — so without RAW a + // second pass strips the ending it just produced and 託す answers たくさんで. + { from: "わぬ", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "かぬ", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "がぬ", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "さぬ", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "たぬ", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "なぬ", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "ばぬ", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "まぬ", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "らぬ", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { + from: "ぬ", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative", + minStem: 1, + stemEnd: V1_STEM, + }, + { from: "せぬ", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative" }, + { from: "こぬ", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative" }, + + // ～なく is the negative's adverbial, the half of 誘わなくなった that is the + // verb. The ADJ rule く→い already reaches 誘わない and stops: nothing is + // spelled that way, and its output is typed ADJ so no verb rule follows it. + { from: "わなく", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "かなく", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "がなく", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "さなく", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "たなく", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "ななく", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "ばなく", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "まなく", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "らなく", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { + from: "なく", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative", + minStem: 1, + stemEnd: V1_STEM, + }, + { from: "しなく", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative" }, + { from: "こなく", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative" }, + + { from: "わなくて", to: "う", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "かなくて", to: "く", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "がなくて", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "さなくて", to: "す", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "たなくて", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "ななくて", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "ばなくて", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "まなくて", to: "む", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { from: "らなくて", to: "る", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, + { + from: "なくて", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative te-form", + minStem: 1, + stemEnd: V1_STEM, + }, + { from: "しなくて", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative te-form" }, + { from: "こなくて", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative te-form" }, + // ── Generic te-iru forms (works across verb types after te-form resolution) ── { from: "ている", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru" }, { from: "てる", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru (casual)" }, @@ -628,6 +708,7 @@ export function deinflect(word: string): DeinflectCandidate[] { const stemLen = current.word.length - rule.from.length; if (stemLen + rule.to.length <= 0) continue; if (rule.minStem != null && stemLen < rule.minStem) continue; + if (rule.stemEnd && !rule.stemEnd.test(current.word[stemLen - 1] ?? "")) continue; if (!(current.typeMask & rule.typeIn)) continue; const base = current.word.slice(0, stemLen) + rule.to; From 72b6b094272161ff2791087e266e3f567d8403da Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 14:22:54 +0800 Subject: [PATCH 05/14] Stop a name's trailing kana from reading the kanji before it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 花のつぼみ printed あや over 花 and 水の流れ printed にず over 水. JMnedict holds 花の and 水の as names, neither ever observed, and the furigana path takes a name outright when no word covers the surface — so two characters beat one and the page got a reading only the whole name has, over a noun and a particle. A name's trailing kana is part of its spelling, not okurigana, and the ruby had already cut it off. A surface ending in kana is no longer considered for a name reading. 15 of 67,299 corpus surfaces change, all of them junk name readings going away. Of the 7,596 name spellings that end in kana, 4,410 print the same ruby from the kanji run alone — 帝国ホテル still reads 帝国 — and the rest are prewar given-name spellings the reader will not meet. 五十嵐, 杏子, 丸木 and 西條 are unmoved. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 31 ++++++++++ .../src/furigana.name-trailing-kana.test.ts | 60 +++++++++++++++++++ packages/japanese-reader/src/furigana.ts | 7 ++- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 packages/japanese-reader/src/furigana.name-trailing-kana.test.ts diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 3002b07..23df0d3 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -910,6 +910,37 @@ disagreeing pairs → **126**. three characters, which is right, and the ナウ is what tapping the leftover なった already gave everywhere else. +### A name spelling that ends in kana is not evidence about the kanji before it + +花のつぼみのような printed **あや** over 花, and 水の流れ printed **にず** over +水. JMnedict holds 花の (Ayano) and 水の (Nizuno); the furigana path takes a +name outright when no word covers the surface, the renderer takes the longest +surface, and two characters beat one. Both spellings have no observed +frequency at all — nobody has been seen reading 水の as にずの. + +The ruby that came out is the tell: `stripOkurigana` had already cut the の +off, so the page got a reading over 花 alone that only the whole name has. A +name's trailing kana is part of its spelling, not okurigana, so a surface +ending in kana is now never considered for a name reading +(`shouldConsiderNameFuriganaSurface`). + +Measured over the corpus: **15 of 67,299 surfaces change, all of them losses +of a name reading, every one junk** — 上の, 仲の, 坂の, 木の, 東の, 松の, 水の, +浜の, 理の, 育の, 高の, 魚の (a kanji and the particle), plus 夢か, 新し and +帝国ホテル. Nothing gains or changes a reading. + +The whole population, not just the corpus: of the **7,596** name spellings in +JMnedict that end in kana, 7,247 stop resolving as a surface. **4,410 of those +print exactly the same ruby anyway**, because the kanji run before the kana +carries the name on its own — 帝国ホテル still reads 帝国/ていこく, and +龍ヶ嶽トンネル still reads 龍ヶ嶽. The remaining 2,837 are spellings with the +kanji buried inside kana (い乃り, み津ゑ, しら坂トンネル): prewar given-name +orthography and signposted place names, which the reader will not meet, and +where the old ruby covered a leading fragment. + +五十嵐, 杏子, 丸木, 後味 and 西條 are all unmoved — nothing written in kanji +is touched. + ## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no diff --git a/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts new file mode 100644 index 0000000..32eb0a3 --- /dev/null +++ b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts @@ -0,0 +1,60 @@ +/** + * A name spelling that ends in kana is not evidence about the kanji before it. + * + * 花のつぼみ printed あや over 花 and 水の流れ printed にず over 水: JMnedict + * holds 花の (Ayano) and 水の (Nizuno), the surface was two characters so it + * beat the one-character word, and the ruby that came out covered the kanji + * alone — a reading only the whole name has, over a noun and a particle. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { resolveFuriganaBatch } from "./furigana"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function entry(surface: string): Promise<{ reading?: string; isName?: boolean }> { + const map = await resolveFuriganaBatch([surface], wrap(dict!), wrap(ext!)); + return map[surface] ?? {}; +} + +describe.skipIf(!hasBothDbs)("a name spelled with trailing kana", () => { + it("is not read as a name", async () => { + expect((await entry("花の")).isName).toBeFalsy(); + expect((await entry("水の")).isName).toBeFalsy(); + expect((await entry("上の")).isName).toBeFalsy(); + expect((await entry("木の")).isName).toBeFalsy(); + }); + + it("leaves the kanji run itself to the word it is", async () => { + expect((await entry("花")).reading).toBe("はな"); + expect((await entry("水")).reading).toBe("みず"); + }); + + // The kanji half of a place name still carries it, which is what the page + // shows either way: the ruby never covered the katakana. + it("keeps the reading on the kanji the ruby actually covered", async () => { + expect((await entry("帝国")).reading).toBe("ていこく"); + }); + + it("leaves a name written in kanji alone", async () => { + expect((await entry("西條")).isName).toBe(true); + expect((await entry("杏子")).reading).toBe("きょうこ"); + expect((await entry("五十嵐")).reading).toBe("いがらし"); + }); +}); diff --git a/packages/japanese-reader/src/furigana.ts b/packages/japanese-reader/src/furigana.ts index 150f600..19d7b37 100644 --- a/packages/japanese-reader/src/furigana.ts +++ b/packages/japanese-reader/src/furigana.ts @@ -587,7 +587,12 @@ export function nameReadingDominance( function shouldConsiderNameFuriganaSurface(surface: string): boolean { // 四十三 and 五十八 are given names in JMnedict and numbers everywhere else. // A run written only in numerals is a number. - return hasKanjiText(surface) && !isKanjiNumeralRun(surface); + if (!hasKanjiText(surface) || isKanjiNumeralRun(surface)) return false; + // A name's trailing kana is part of its spelling, not okurigana, so the ruby + // would cover the kanji alone and claim a reading only the longer name has: + // 花の is the surname Ayano, and in prose it is 花 and the particle. + const chars = [...surface]; + return !isKana(chars[chars.length - 1]); } function resolveLexicalSuffixJlpt( From b1c85398d7b4c94167948f50a252d82f4333ad51 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 14:29:25 +0800 Subject: [PATCH 06/14] Let a counted two-kanji name outrank a one-kanji word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tapping 西 in 西條さんがそばに来て answered 西/せい, "Spain", while the furigana over the same two characters read さいじょう. A name may only override a shorter word from three characters up, and the note beside that floor already named the cost: two-kanji surnames. The floor exists for spans straddling a word boundary — 田先 out of 山田先生, 中電 out of 食事中電話 — and name_freq separates those from real surnames: 西條 has been seen 24 times, 田先 and 中電 never. Two kanji with any observed frequency now override. 12 of 11,296 auto-mode corpus taps change, every one a name in the book that used to answer as a single kanji: 茂作, 箱根, 小倉, 古賀, 浅井, 堀田. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 23 ++++++ .../src/auto-name.override.test.ts | 73 +++++++++++++++++++ packages/japanese-reader/src/auto-name.ts | 19 +++++ packages/japanese-reader/src/lookup.ts | 4 +- 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 packages/japanese-reader/src/auto-name.override.test.ts diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 23df0d3..ca7c783 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -941,6 +941,29 @@ where the old ruby covered a leading fragment. 五十嵐, 杏子, 丸木, 後味 and 西條 are all unmoved — nothing written in kanji is touched. +### A two-kanji name the counts have seen may override a one-kanji word + +演じた**西條**さんがそばに来て: the furigana read さいじょう and the tap, on the +same two characters, answered 西/せい — "Spain". Tapping the second character +gave the name; tapping the first gave the word. + +`AUTO_NAME_OVERRIDE_MIN_LENGTH` is 3, and the note beside it says what that +costs: "two-kanji surnames (渋沢, 山田), which still resolve to a single kanji". +The floor is there for two-kanji spans straddling a word boundary — 田先 out of +山田先生, 中電 out of 食事中電話 — and `names.name_freq` now separates those +cleanly: 西條 is さいじょう in **24** sightings, 山田 やまだ in 935, 渋沢 しぶさわ +in 34, while 田先 and 中電 have never been observed as names at all. A +two-kanji name with any observed frequency may now override +(`nameMayOverrideShorterWord`). + +Measured over **11,296** auto-mode corpus taps: **12 change, all of them +names in the book that a tap used to answer with a single kanji** — 茂作 (was +作), 箱根 (箱, 根), 小倉 (小, 倉), 古賀 (古, 賀), 浅井 (井, three times) and +堀田 (堀, 田). Nothing else moves, in either direction. + +The floor itself is untouched above two characters, and an uncounted two-kanji +spelling is still refused — which is the whole of what keeps 田先 out. + ## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no diff --git a/packages/japanese-reader/src/auto-name.override.test.ts b/packages/japanese-reader/src/auto-name.override.test.ts new file mode 100644 index 0000000..d301e9f --- /dev/null +++ b/packages/japanese-reader/src/auto-name.override.test.ts @@ -0,0 +1,73 @@ +/** + * A two-kanji name the counts have seen may override a one-kanji word. + * + * 西條さんがそばに来て: tapping 西 answered 西/せい, "Spain", while the furigana + * over the same two characters read さいじょう. The override floor is three + * characters, and it is there for spans straddling a word boundary — 田先 out + * of 山田先生 — which the frequency column now separates: 西條 has been seen 24 + * times, 田先 and 中電 never. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { nameMayOverrideShorterWord } from "./auto-name"; +import { autoLookupWithOffset } from "./lookup"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function tap(text: string, at: number) { + return (await autoLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; +} + +describe("the override floor", () => { + it("lets a counted two-kanji name through", () => { + expect(nameMayOverrideShorterWord(2, 24)).toBe(true); + expect(nameMayOverrideShorterWord(2, 1)).toBe(true); + }); + + it("keeps an uncounted one out", () => { + expect(nameMayOverrideShorterWord(2, null)).toBe(false); + expect(nameMayOverrideShorterWord(2, 0)).toBe(false); + }); + + it("leaves three characters where they were", () => { + expect(nameMayOverrideShorterWord(3, null)).toBe(true); + }); +}); + +describe.skipIf(!hasBothDbs)("a two-kanji surname under the finger", () => { + it("answers the name from either character", async () => { + for (const at of [12, 13]) { + const hit = await tap("ずいぶん多くの役を演じた西條さんがそばに来て、こんな", at); + expect(hit?.matchedText).toBe("西條"); + expect(hit?.nameMatches?.[0]?.kana).toBe("さいじょう"); + } + }); + + // The corpus's own names, each previously answered as a single-kanji word. + it("answers the names the corpus holds", async () => { + expect((await tap("の角をつれて、茂作の人参畠をあらした", 8))?.matchedText).toBe("茂作"); + expect((await tap("と云うと「箱根のさきですか", 6))?.matchedText).toBe("箱根"); + expect((await tap("同じ数学の教師に堀田というのが居た", 8))?.matchedText).toBe("堀田"); + }); + + it("still refuses a span straddling a word boundary", async () => { + expect((await tap("山田先生のお宅へ行った", 1))?.matchedText).not.toBe("田先"); + expect((await tap("食事中電話が鳴った", 2))?.matchedText).not.toBe("中電"); + }); +}); diff --git a/packages/japanese-reader/src/auto-name.ts b/packages/japanese-reader/src/auto-name.ts index 1c0f939..5ead431 100644 --- a/packages/japanese-reader/src/auto-name.ts +++ b/packages/japanese-reader/src/auto-name.ts @@ -20,6 +20,25 @@ export const AUTO_NAME_DUAL_CONFIDENCE = 45; */ export const AUTO_NAME_OVERRIDE_MIN_LENGTH = 3; +/** + * What that floor costs is two-kanji surnames, and the counts can pay it back: + * 西條 is さいじょう in 24 sightings, while 田先 and 中電 — the straddles the + * floor exists for — have never been observed as names at all. + */ +export const AUTO_NAME_OVERRIDE_COUNTED_LENGTH = 2; +export const AUTO_NAME_OVERRIDE_MIN_FREQ = 1; + +export function nameMayOverrideShorterWord( + matchedLength: number, + topFreq: number | null | undefined, +): boolean { + if (matchedLength >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; + return ( + matchedLength >= AUTO_NAME_OVERRIDE_COUNTED_LENGTH && + (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_MIN_FREQ + ); +} + export interface AutoNameWordCandidate { matchedText: string; exactSurface: boolean; diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index b5dd84f..57ee20b 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -4,9 +4,9 @@ import { AUTO_DUAL_MIN_MATCH_LENGTH, AUTO_NAME_DUAL_CONFIDENCE, AUTO_NAME_ONLY_CONFIDENCE, - AUTO_NAME_OVERRIDE_MIN_LENGTH, AUTO_NAME_ONLY_WITH_EXACT_WORD_CONFIDENCE, computeAutoNameConfidence, + nameMayOverrideShorterWord, shouldShowBothAutoResults, type AutoNameNameCandidate, type AutoNameWordCandidate, @@ -693,7 +693,7 @@ function chooseAutoLookupVariants( nameIsLonger && (nameConfidence >= nameOnlyConfidence || (topNameExactSurfaceMatch(bestName) && - bestName.matchedText.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH)) + nameMayOverrideShorterWord(bestName.matchedText.length, bestName.nameMatches?.[0]?.freq))) ) { return taggedNames; } From 11eeebc4f8fb93659097896b51ae0ffdfce359c1 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 14:49:52 +0800 Subject: [PATCH 07/14] Let a suru-verb noun close its own kana, and hold names to the same starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 赴任したためだった answered したため — 認む, in its imperative — because 赴任した could not prove its tail: the chain runs through 赴任する, which no dictionary spells, to 赴任, which every dictionary does, and that step was counted as a chain through an auxiliary. Closing those starts exposed the name walk, which had no such guard and answered 注文したから with the surname たから; auto mode now computes the starts once and hands them to both walks. And it exposed the override itself, which accepted a match on a name's READING — ためだった holds 為田's — so it now asks for the name's own spelling, kanji or a kana name spelled out, which keeps ひろし and たけし. Word mode: 15 of 11,296 corpus taps change, every one a span that was cutting into a suru-verb. Auto mode: 489 change, 472 of them a kana run answered with a JMnedict name and now answered with a word. Gate 126 disagreeing pairs to 123. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 39 ++++++ .../src/lookup.span-starts.test.ts | 69 ++++++++++ packages/japanese-reader/src/lookup.ts | 124 +++++++++++++----- 3 files changed, 200 insertions(+), 32 deletions(-) create mode 100644 packages/japanese-reader/src/lookup.span-starts.test.ts diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index ca7c783..06a9c17 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -964,6 +964,45 @@ names in the book that a tap used to answer with a single kanji** — 茂作 (wa The floor itself is untouched above two characters, and an uncounted two-kanji spelling is still refused — which is the whole of what keeps 田先 out. +### A suru-verb noun proves its own kana tail, and the name walk refuses the same starts + +海外へ赴任した**ため**だった answered したため — 認む, "to write down", in its +imperative — out of a span starting inside 赴任した. + +"A span may not start inside a word" proves a position closed by looking the +kanji run and its kana up together and finding an **inflecting** word. 赴任した +deinflects to 赴任する, which no dictionary spells, and then to 赴任, which every +dictionary does — and that second step was counted as a chain through an +auxiliary and dropped. Stripping する off a suru-verb noun is not a second word, +so it no longer counts towards that bound, and an entry reached that way proves +the tail when it is tagged `vs`. + +**Closing those starts exposed the other half.** The name walk had no such +guard, so with the word span refused, 注文したから came back as the surname +たから. Auto mode now computes the starts once and hands them to both walks +(`okuriganaStartsForTap`). + +**And then the name walk's own hole.** A name may override a shorter word +outright when the surface is "an exact name match", and that accepted a match +on the name's **reading**: ためだった holds the reading of 為田, したから holds +多可良's. The override now asks that the surface be how the name is _written_ — +its kanji form, or the whole spelling for a name written in kana, so ひろし and +たけし still resolve (`nameSpellingIsSurface`). + +Measured over 11,296 corpus taps. **Word mode: 15 change, all of them a span +that was cutting into a suru-verb** — 注文したから→から, 勉強してやろう→やろう, +合点したもの→もの, 紹介してやる→やる, 心配しなくって→って, 珍重してく→くれた. +**Auto mode: 489 change — 472 of them a kana run that was being answered with a +JMnedict name and is now answered with a word**: たから→から (24 taps), +いから→から (14), たのだ→のだ, ものか→もの, ぬから→から, なるの→なる, plus a +tail that reaches real words — いなが→云い, きなが→泣き, きでな→好き, +みもな→望み, しもなか→少しも, うがい→使う. Two are lateral (いなひと→ひとは, +junk either way). Gate 126 disagreeing pairs → **123**. + +**What this does not fix:** めだった still answers 目立つ. ため is a noun, so +its め is not okurigana and nothing proves the position closed; separating that +from a real word boundary needs segmentation, not a rule. + ## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no diff --git a/packages/japanese-reader/src/lookup.span-starts.test.ts b/packages/japanese-reader/src/lookup.span-starts.test.ts new file mode 100644 index 0000000..8bffef0 --- /dev/null +++ b/packages/japanese-reader/src/lookup.span-starts.test.ts @@ -0,0 +1,69 @@ +/** + * Where a span may begin, and what a name has to be to win one. + * + * 赴任したためだった answered したため — the archaic 認む, "to write down" — + * because 赴任した could not prove its own kana tail: the chain runs through + * 赴任する, which no dictionary spells, to 赴任, which every dictionary does, + * and that second step was counted as a second word. + * + * Closing those starts then exposed the other half: the name walk had no such + * guard, so 注文したから came back as the surname たから — a kana run that is a + * name's READING, not a name's spelling. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { autoLookupWithOffset, smartLookupWithOffset } from "./lookup"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +const word = async (text: string, at: number) => + (await smartLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; +const auto = async (text: string, at: number) => + (await autoLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; + +describe.skipIf(!hasBothDbs)("a suru-verb noun proves its own kana tail", () => { + it("closes the し and た of 赴任した", async () => { + expect((await word("海外へ赴任したためだった。", 7))?.matchedText).toBe("ため"); + expect((await auto("海外へ赴任したためだった。", 7))?.matchedText).toBe("ため"); + }); + + it("stops a span cutting into 注文した, 勉強して, 合点した", async () => { + expect((await word("切ってみろと注文したから、何だ指ぐらい", 10))?.matchedText).toBe("から"); + expect((await word("学資にして勉強してやろう。六百円を", 9))?.matchedText).toBe("やろう"); + expect((await word("相違ないと合点したものらしい。", 9))?.matchedText).toBe("もの"); + }); + + it("leaves a noun's trailing particle alone", async () => { + // 注意に is not 注意する in any form, so に stays open. + expect((await word("注意にも程がある", 3))?.matchedText).not.toBe("注意に"); + }); +}); + +describe.skipIf(!hasBothDbs)("what a name has to be to outrank a word", () => { + it("refuses a kana run that is only a name's reading", async () => { + const hit = await auto("切ってみろと注文したから、何だ指ぐらい", 10); + expect(hit?.lookupKind).not.toBe("name"); + expect(hit?.matchedText).toBe("から"); + }); + + it("keeps a name that is spelled the way the page spells it", async () => { + expect((await auto("ひろしに聞いた", 0))?.nameMatches?.[0]?.kana).toBe("ひろし"); + expect((await auto("演じた西條さんがそばに来て", 3))?.matchedText).toBe("西條"); + }); +}); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index 57ee20b..a9cb1ac 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -14,6 +14,7 @@ import { import type { ReaderSqlDb } from "./backend"; import { CONTRACTION_REASONS, + SURU_NOUN_REASON, deinflect, generateSubstrings, kanaSpellings, @@ -345,6 +346,19 @@ function topWordExactSurfaceMatch(result: LookupResult): boolean { ); } +/** + * The surface is how a name is WRITTEN, not merely how one is read. ためだった + * holds the reading of 為田 and したから holds 多可良's, and a kana run that + * spells out a name — ひろし, たけし — is a spelling in its own right. + */ +function nameSpellingIsSurface(result: LookupResult): boolean { + return ( + result.nameMatches?.some((name) => + name.kanji ? name.kanji === result.matchedText : name.kana === result.matchedText, + ) ?? false + ); +} + function topNameExactSurfaceMatch(result: LookupResult): boolean { return ( result.nameMatches?.some( @@ -549,6 +563,44 @@ async function lookupNegativeScopeParticleAtTap( * same span could win from one character and lose from the next — which is the * disagreement `yarn check:tap-consistency` measures. */ +/** + * One tap, one lookup per word. The walk asks for the same word from many + * overlapping substrings, and the kana spellings multiply that again. + */ +function makeEntryCache(dictDb: ReaderSqlDb) { + const entryCache = new Map(); + const lookupOnce = async (word: string): Promise => { + const cached = entryCache.get(word); + if (cached) return cached; + const found = await lookupExactJapanese(dictDb, word); + entryCache.set(word, found); + return found; + }; + /** Fill the cache for a whole length's candidates in one round trip. */ + const prefetch = async (words: string[]): Promise => { + const missing = [...new Set(words)].filter((word) => !entryCache.has(word)); + if (missing.length === 0) return; + const found = await lookupExactJapaneseMany(dictDb, missing); + for (const word of missing) entryCache.set(word, found.get(word) ?? []); + }; + return { entryCache, lookupOnce, prefetch }; +} + +/** + * The positions a span may not begin at, for a tap the caller has not already + * resolved. Auto mode runs the word walk and the name walk side by side and + * both have to refuse the same starts, so it computes this once and hands it + * to each. + */ +export async function okuriganaStartsForTap( + text: string, + tapOffset: number, + dictDb: ReaderSqlDb, +): Promise> { + const { lookupOnce, prefetch } = makeEntryCache(dictDb); + return findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce); +} + /** * Positions where a candidate would begin inside a word rather than at one. * @@ -566,12 +618,15 @@ async function findOkuriganaStarts( lookupOnce: (word: string) => Promise, ): Promise> { const earliest = Math.max(1, tapOffset - 14 - MAX_OKURIGANA); - const tails: { first: number; ends: { end: number; words: string[] }[] }[] = []; + const tails: { + first: number; + ends: { end: number; words: { word: string; viaSuru: boolean }[] }[]; + }[] = []; for (let first = earliest; first <= tapOffset; first++) { if (!isKanaChar(text[first]) || !isKanjiChar(text[first - 1])) continue; let head = first - 1; while (head > 0 && isKanjiChar(text[head - 1])) head--; - const ends: { end: number; words: string[] }[] = []; + const ends: { end: number; words: { word: string; viaSuru: boolean }[] }[] = []; for (let end = first; end < text.length && end < first + MAX_OKURIGANA; end++) { if (!isKanaChar(text[end])) break; ends.push({ @@ -579,24 +634,36 @@ async function findOkuriganaStarts( // One inflection step is the word conjugating itself. Two is a chain // through an auxiliary, which is a second word: 倒して is 倒す in its // te-form, but 倒してや reaches 倒す as well and would swallow やった. + // Stripping する off a suru-verb noun is not a second word: 赴任した + // reaches 赴任する, which no dictionary spells, and then 赴任, which + // every dictionary does. That step does not count towards the bound. words: deinflect(text.slice(head, end + 1)) - .filter((candidate) => candidate.reasons.length <= 1) - .map((candidate) => candidate.word), + .filter( + (candidate) => + candidate.reasons.filter((reason) => reason !== SURU_NOUN_REASON).length <= 1, + ) + .map((candidate) => ({ + word: candidate.word, + viaSuru: candidate.reasons.includes(SURU_NOUN_REASON), + })), }); } if (ends.length > 0) tails.push({ first, ends }); } if (tails.length === 0) return new Set(); - await prefetch(tails.flatMap((tail) => tail.ends.flatMap((option) => option.words))); + await prefetch( + tails.flatMap((tail) => tail.ends.flatMap((option) => option.words.map((w) => w.word))), + ); const starts = new Set(); for (const { first, ends } of tails) { // Longest first: the word decides how far its own tail runs, and taking the // shortest would leave the rest of the tail open to a span that cuts in. for (const { end, words } of [...ends].reverse()) { let inflects = false; - for (const word of words) { - if ((await lookupOnce(word)).some(entryInflects)) { + for (const { word, viaSuru } of words) { + const entries = await lookupOnce(word); + if (entries.some((entry) => entryInflects(entry) || (viaSuru && entryTakesSuru(entry)))) { inflects = true; break; } @@ -612,6 +679,11 @@ async function findOkuriganaStarts( /** How far past a kanji a single word's kana tail is allowed to reach. */ const MAX_OKURIGANA = 4; +/** A noun that takes する: its し, した, して are the verb, not a particle. */ +function entryTakesSuru(entry: ReaderDictEntry): boolean { + return entry.senses.some((sense) => sense.partOfSpeech?.some((pos) => pos.startsWith("vs"))); +} + /** Only an inflecting word has okurigana; a noun's trailing kana is a particle. */ function entryInflects(entry: ReaderDictEntry): boolean { return entry.senses.some((sense) => @@ -692,7 +764,7 @@ function chooseAutoLookupVariants( if ( nameIsLonger && (nameConfidence >= nameOnlyConfidence || - (topNameExactSurfaceMatch(bestName) && + (nameSpellingIsSurface(bestName) && nameMayOverrideShorterWord(bestName.matchedText.length, bestName.nameMatches?.[0]?.freq))) ) { return taggedNames; @@ -1126,32 +1198,13 @@ export async function smartLookupWithOffset( tapOffset: number, dictDb: ReaderSqlDb, extDb?: ReaderSqlDb | null, + precomputedOkuriganaStarts?: ReadonlySet, ): Promise { const negativeScopeParticle = await lookupNegativeScopeParticleAtTap(text, tapOffset, dictDb); if (negativeScopeParticle) return [asWordLookupResult(negativeScopeParticle)]; const kanjiCache = new Map(); - // The walk asks for the same word from many overlapping substrings, and the - // kana spellings below multiply that again. One tap, one lookup per word. - const entryCache = new Map(); - const lookupOnce = async (word: string): Promise => { - const cached = entryCache.get(word); - if (cached) return cached; - const found = await lookupExactJapanese(dictDb, word); - entryCache.set(word, found); - return found; - }; - /** - * Fill the cache for a whole length's candidates in one round trip. Most - * candidate spellings do not exist, and asking one at a time spent the bulk - * of a tap's queries proving that. - */ - const prefetch = async (words: string[]): Promise => { - const missing = [...new Set(words)].filter((word) => !entryCache.has(word)); - if (missing.length === 0) return; - const found = await lookupExactJapaneseMany(dictDb, missing); - for (const word of missing) entryCache.set(word, found.get(word) ?? []); - }; + const { entryCache, lookupOnce, prefetch } = makeEntryCache(dictDb); /** * Spellings of `substr` with its one kanji written as a reading instead. @@ -1181,7 +1234,9 @@ export async function smartLookupWithOffset( } } const counterHints = await buildCounterHintMap([...tapSurfaces], extDb); - const okuriganaStarts = await findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce); + const okuriganaStarts = + precomputedOkuriganaStarts ?? + (await findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce)); let bestOverall: { result: LookupResult; score: number; length: number; start: number } | null = null; @@ -1365,9 +1420,12 @@ export async function autoLookupWithOffset( dictDb: ReaderSqlDb, extDb?: ReaderSqlDb | null, ): Promise { + // Both walks have to refuse the same starts, or the name walk answers a span + // the word walk just refused: 注文したから came back as the surname たから. + const okuriganaStarts = await okuriganaStartsForTap(text, tapOffset, dictDb); const [wordResults, nameResults] = await Promise.all([ - smartLookupWithOffset(text, tapOffset, dictDb, extDb), - extDb ? nameLookupWithOffset(text, tapOffset, extDb) : Promise.resolve([]), + smartLookupWithOffset(text, tapOffset, dictDb, extDb, okuriganaStarts), + extDb ? nameLookupWithOffset(text, tapOffset, extDb, okuriganaStarts) : Promise.resolve([]), ]); return chooseAutoLookupResults(wordResults, nameResults); } @@ -1418,12 +1476,14 @@ export async function nameLookupWithOffset( text: string, tapOffset: number, extDb: ReaderSqlDb, + okuriganaStarts?: ReadonlySet, ): Promise { for (let len = Math.min(text.length, 15); len >= 1; len--) { const minStart = Math.max(0, tapOffset - len + 1); const maxStart = Math.min(tapOffset, text.length - len); for (let start = Math.min(tapOffset, maxStart); start >= minStart; start--) { + if (okuriganaStarts?.has(start)) continue; const substr = text.slice(start, start + len); const names = await lookupExactName(extDb, substr); if (names.length > 0) { From db49524aad701bb7ce6f8e74a90199408011da2a Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 14:52:59 +0800 Subject: [PATCH 08/14] Answer a drag selection with what it selected, first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selecting 展開 in チェーン展開の新古書店 answered チェーン展開: the expansion step runs before anything else and returns on its first hit, so a selection that is itself a word was never asked about. A tap has no boundary and has to guess one. A drag is the boundary. The selection's own spelling is looked up first and emitted first; the expansion still runs and lands under it, so 色い still reaches 茶色い and a long selection still walks word by word into its suffix. The selection cases in lib/smart-lookup.test.ts re-implement the walk instead of calling it, so they cannot see this; lookup.selection.test.ts runs the shipping function. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 30 ++++++++ .../src/lookup.selection.test.ts | 70 +++++++++++++++++++ packages/japanese-reader/src/lookup.ts | 32 ++++----- 3 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 packages/japanese-reader/src/lookup.selection.test.ts diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 06a9c17..38cca06 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -1003,6 +1003,36 @@ junk either way). Gate 126 disagreeing pairs → **123**. its め is not okurigana and nothing proves the position closed; separating that from a real word boundary needs segmentation, not a rule. +### A drag selection is answered with what it selected, first + +Selecting **展開** in チェーン展開の新古書店 answered チェーン展開. The expansion +step — which tries substrings of the selection plus ten characters either side, +longest first — runs before anything else and returns on its first hit, so a +selection that is itself a word was never asked about. Reported as: "when I +drag EXACTLY a region I want the lookup restricted to that region, the opposite +of greedy walking both directions when I just tap." + +A tap has no boundary, so it has to guess one. A drag is the boundary. The +selection's own spelling is now looked up first and emitted first, and the +expansion still runs and lands under it: + +| selected | prefix / suffix | was | now | +| ---------------------- | --------------------- | ---------------------------- | --------------------------- | +| 展開 | チェーン / の新古書店 | チェーン展開 | **展開**, then チェーン展開 | +| べた | 食 / 。 | 食べた | **べた**, then 食べた | +| 色い | 茶 / シャツ | 茶色い | 茶色い | +| くもない | 若 / という | 若くもない | 若くもない | +| 若くもないというのに姿 | / 勢がよく | 若くもない・というのに・姿勢 | unchanged | + +Nothing is lost: a selection that is not a word still reaches past its own +edges, and a long selection still walks word by word with the suffix attached, +which is what 姿 → 姿勢 depends on. + +**The selection cases in `lib/smart-lookup.test.ts` re-implement the walk** in +the test file rather than calling it, so they cannot see this change at all — +they pass before and after. `lookup.selection.test.ts` runs the shipping +function. + ## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no diff --git a/packages/japanese-reader/src/lookup.selection.test.ts b/packages/japanese-reader/src/lookup.selection.test.ts new file mode 100644 index 0000000..58f16e6 --- /dev/null +++ b/packages/japanese-reader/src/lookup.selection.test.ts @@ -0,0 +1,70 @@ +/** + * What a drag selection is answered with. + * + * Selecting 展開 in チェーン展開の新古書店 answered チェーン展開: the expansion + * step runs first and returns on its first hit, so a selection that is itself a + * word never got asked about. A drag says where the word ends. + * + * These run the shipping function. The cases in lib/smart-lookup.test.ts + * re-implement the walk to check its shape, which cannot see this. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { selectionLookup } from "./lookup"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function select(text: string, prefix = "", suffix = ""): Promise { + const out: string[] = []; + await selectionLookup(text, wrap(dict!), (result) => out.push(result.matchedText), { + prefix, + suffix, + extendedDb: wrap(ext!), + }); + return out; +} + +describe.skipIf(!hasBothDbs)("a selection that is itself a word", () => { + it("is answered with itself, first", async () => { + expect(await select("展開", "チェーン", "の新古書店")).toEqual(["展開", "チェーン展開"]); + }); + + it("keeps the longer word as the second answer, not the only one", async () => { + expect(await select("べた", "食", "。")).toEqual(["べた", "食べた"]); + }); + + it("does not repeat itself when nothing longer is there", async () => { + expect(await select("散歩", "の", "に誘わ")).toEqual(["散歩"]); + }); +}); + +describe.skipIf(!hasBothDbs)("a selection that is not a word", () => { + it("still reaches past its edges", async () => { + expect(await select("色い", "茶", "シャツ")).toEqual(["茶色い"]); + expect(await select("くもない", "若", "という")).toEqual(["若くもない"]); + }); + + it("still walks a long selection word by word", async () => { + expect(await select("若くもないというのに姿", "", "勢がよく")).toEqual([ + "若くもない", + "というのに", + "姿勢", + ]); + }); +}); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index a9cb1ac..5dac3a5 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -874,6 +874,20 @@ export async function selectionLookup( options?.extendedDb, ); + // A drag says where the word ends. チェーン展開 is the longer word, but a + // selection of 展開 is a question about 展開, so what the selection itself + // spells is answered first and the expansion below lands under it. + const exactSelection = await lookupExactJapanese(dictDb, trimmed); + if (exactSelection.length > 0) { + onResult( + asWordLookupResult({ + matchedText: trimmed, + entries: sortEntriesForMatchedSurface(exactSelection, trimmed, counterHints.get(trimmed)), + deinflectReasons: [], + }), + ); + } + // Expansion step: try substrings of expanded text that fully contain the selection if (prefix.length > 0 || suffix.length > 0) { const expanded = prefix + trimmed + suffix; @@ -925,23 +939,7 @@ export async function selectionLookup( } } - // Try the full selected text as a single lookup first - const fullEntries = await lookupExactJapanese(dictDb, trimmed); - if (fullEntries.length > 0) { - const sortedEntries = sortEntriesForMatchedSurface( - fullEntries, - trimmed, - counterHints.get(trimmed), - ); - onResult( - asWordLookupResult({ - matchedText: trimmed, - entries: sortedEntries, - deinflectReasons: [], - }), - ); - return; - } + if (exactSelection.length > 0) return; // Try deinflecting the full selected text const fullCandidates = deinflect(trimmed); From 93bf68efa7e41f2a33d5f0f1f4831c102d0a6660 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 16:20:14 +0800 Subject: [PATCH 09/14] Hold the new negative rules to the rule the repo already had MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review found the ～ぬ / ～なく rules speaking over words the dictionary has: 問題でなく answered 出る, 弛まぬ answered 緩む, 馬がいななく answered 犬, せわしなく answered 世話. 53 regressions over the 4,155 ～ない-adverbial and ～ぬ surfaces — a population Bocchan does not contain, which is why the committed sweep showed none of it. Undoing one of these rules is a guess, and a reading that needs no guess wins first. The rules carry `guess: true`, deinflect propagates it, and the gate reaches the three walks that never had one — the drag-selection expansion, findFirstWord and findBoundaryWord. The furigana resolver asks it as a preference rather than a weight, because commonness pays +120 there against −80 for a deinflection. Also: the name guard narrows to a kanji followed by a PARTICLE. Refusing every kana tail took the ruby off 十勝ダム and 一戸トンネル entirely and garbled 三条通【さんじょうどお】り into 三条通【さんじょうどおり】り. NAME mode gets the okurigana starts it was never passed. Rules that emit ANY now emit ANY_OUTPUT, so the suru-noun rule cannot re-grant RAW. Against origin/master: 107 of 11,296 word-mode taps change, all wins; 593 auto-mode, 477 of them a kana run answered with a JMnedict name and now answered with a word; 57 of 67,299 surfaces — 44 gain furigana, 13 lose a junk name reading, 0 read differently. Gate 128 pairs → 123. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 166 +++++++---- lib/deinflect.test.ts | 4 +- .../src/deinflect.negative-nu.test.ts | 24 +- packages/japanese-reader/src/deinflect.ts | 267 +++++++++++++++--- .../src/furigana.name-trailing-kana.test.ts | 34 ++- packages/japanese-reader/src/furigana.ts | 61 ++-- .../src/lookup.selection.test.ts | 20 +- .../src/lookup.span-starts.test.ts | 41 ++- packages/japanese-reader/src/lookup.ts | 39 ++- .../src/use-japanese-reader.ts | 10 +- 10 files changed, 535 insertions(+), 131 deletions(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 38cca06..c4ac35b 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -38,6 +38,13 @@ of argued about. `it.fails`, so the suite is green today and goes red the moment one is fixed. +**None of this runs in CI.** The dictionaries are build artifacts and the +workflow does not fetch them, so `test/dictionary-db.ts` makes every suite that +queries them `describe.skipIf` — three of the five files added for the lookup +work skip entirely there, and `yarn check:tap-consistency` is not a CI step at +all. The sweeps above and those suites are local gates. What CI does gate is +the pure rule table: `deinflect.*.test.ts` and the scoring helpers. + A change is shippable when the diff is enumerable and every entry in it is an improvement, neutral, or a regression named and accepted in this document. A regression nobody wrote down is not accepted, it is unnoticed. @@ -861,7 +868,7 @@ take a past. That was reading **any** entry with an `exp` sense as classless, and 棒 is `n,exp` — so ぼって was painted across the corpus as a te-form of the noun 棒. `exp` now has to be the only class recorded. -#### Negative ～ぬ, ～なく and ～なくて +### Negative ～ぬ, ～なく and ～なくて, and the guess they are Reported from a novel: 広げた風呂敷を**畳まぬ**ようなもの printed じょう over 畳 and answered the tap with the mat たたみ, and 散歩に**誘わなく**なった printed @@ -875,42 +882,61 @@ form was in the rule table. and stops there: nothing is spelled that way, and it types the result ADJ so no verb rule can follow it. -Three guards, each from a measured failure: - -- **`typeIn: RAW` on every one of them.** ～ぬ is also a verb ending — the - te-form rule んで→ぬ makes たくさんで a ぬ-verb — so without it a second pass - strips the ending the first one produced: たくさんで answered 託す, んです - answered 酢, and 〜れたんだ answered 劣. All four of those vanish with RAW. +**Every one of them is a guess, and that is the load-bearing part.** The first +version of this change shipped the rules unmarked, and an adversarial review +found what that costs, over a population the corpus cannot reach — all 4,155 +～ない-adverbial and ～ぬ dictionary surfaces, where **237 taps moved and 53 were +regressions**: + +| | was | became | +| -------------------------- | ----------------- | ------ | +| 問題**でなく**単なる誤解だ | で無い | 出る | +| **弛まぬ**努力 | 弛まぬ (たゆまぬ) | 緩む | +| **卒なく**こなす | 卒なく | 終わる | +| 馬が**いななく** | 嘶く | 犬 | +| **せわしなく**歩き回る | 忙しない | 世話 | + +Each of those surfaces is itself a dictionary entry, and the manufactured verb +beat it because `hasCommon` pays 100 against five points a reason. The repo +already had the answer — **a reading that needs no guess wins first**, the rule +that keeps にしては from being answered with にしても — so the rules now carry +`guess: true`, `deinflect` propagates a `guessed` flag (merging it, because a +word reachable without a guess is not one), and the gate was extended to the +three walks that never had it: the drag-selection expansion, `findFirstWord` +and `findBoundaryWord`. The furigana resolver asks the same question as a +**preference, not a weight** — commonness pays +120 there against −80 for a +deinflection, so a score could not express it. + +Checked against every one of the 53: all fixed, and none of 畳まぬ, 知らぬ, +誘わなく, 出なく, 心配しなく, 消えぬ, 知れぬ is itself an entry, so every win +still speaks. Ordering cannot defeat the gate — the surface is candidate 0 and +the ADJ block precedes the negative block, measured over 747,558 surfaces with +zero counterexamples. + +Three further guards, each from a measured failure: + +- **`typeIn: RAW` on every rule.** ～ぬ is also a verb ending — the te-form rule + んで→ぬ makes たくさんで a ぬ-verb — so without it a second pass strips the + ending the first one produced: たくさんで answered 託す, んです answered 酢. - **`stemEnd: V1_STEM` on the three bare ichidan rules**, a new field holding - what the character before the ending has to be. お小遣い**が**なくて is a - particle and ない, but が passes a length floor, so the bare rule reached がる - and answered a particle with a verb (4 taps). Only an え/い-row kana or a - kanji can be an ichidan stem; が is あ-row. + what the character before the ending has to be. お小遣い**が**なくて passes a + length floor, so the bare rule reached がる. - **`minStem: 1` on the row rules, and none on せぬ / こぬ / しなく / こなく**, - which are whole words with no stem at all. The test caught that: せぬ reached - せる and す but never する. - -Measured. **Tap sweep: 33 of 3,772 corpus taps change.** 21 are the point — -知らぬ→知る (was 糠, 等, 知/ち), 知れぬ→知れる, 切れぬ→切れる, 消えぬ→消える, -使えぬ→使う, 出揃わぬ→出揃う, 出なく→出る. **Furigana sweep: 44 of 67,299 -surfaces gain a reading, 0 lose one, 0 read differently** — 似ぬ, 心得ぬ, -思えぬ, 分らなく, 聞かなくて, 鳴らなくて and the rest. Gate 98.0%, 128 -disagreeing pairs → **126**. - -**Accepted regressions**, all read in that diff: - -- 何にもせぬ (5 taps). おやじは何にもせぬ男で: the span now swallows せぬ and - shows 何にも, because せぬ→する→the suru-noun rule lands back on 何にも. The - entry shown is still the right one and two of those taps were junk before - (燃やす for せ); the ideal split, 何にも + する, needs the suru-noun rule to - refuse a negative it did not undo itself. -- はせぬ → 蓮/はす (3 taps). そんな依怙贔負はせぬ男だ is は + せぬ. Junk for junk: - it answered 蓮 in its imperative before. -- なくなった → ナウ (3 taps). 水が出なくなった now answers 出なく on the first - three characters, which is right, and the ナウ is what tapping the leftover - なった already gave everywhere else. - -### A name spelling that ends in kana is not evidence about the kanji before it + which are whole words with no stem at all. + +**A bit leak the same review found, fixed here:** `ANY` is `0xff` and `RAW` is +`0x80`, so a rule whose output is typed `ANY` re-granted RAW — よぎなくさせる +reached 余儀る through the suru-noun rule, three rules deep. Rules that emit +`ANY` now emit `ANY_OUTPUT`, which is `ANY & ~RAW`. Latent today (余儀る and +程る are not words) and found by sweeping 827,842 dictionary forms. + +**Not shipped here, deliberately:** ～ん (読まん), ～ざる (行かざる) and ～なくちゃ +resolve to nothing at all. ～ん needs its own collision sweep before it goes +near the table — it collides with sentence-final ん, with のだ→んだ and with +every noun ending in ん, which is the over-broad shape that produced +たくさんで→託す in the first place. + +### A kanji followed by a particle is the particle 花のつぼみのような printed **あや** over 花, and 水の流れ printed **にず** over 水. JMnedict holds 花の (Ayano) and 水の (Nizuno); the furigana path takes a @@ -919,27 +945,42 @@ surface, and two characters beat one. Both spellings have no observed frequency at all — nobody has been seen reading 水の as にずの. The ruby that came out is the tell: `stripOkurigana` had already cut the の -off, so the page got a reading over 花 alone that only the whole name has. A -name's trailing kana is part of its spelling, not okurigana, so a surface -ending in kana is now never considered for a name reading -(`shouldConsiderNameFuriganaSurface`). - -Measured over the corpus: **15 of 67,299 surfaces change, all of them losses -of a name reading, every one junk** — 上の, 仲の, 坂の, 木の, 東の, 松の, 水の, -浜の, 理の, 育の, 高の, 魚の (a kanji and the particle), plus 夢か, 新し and -帝国ホテル. Nothing gains or changes a reading. - -The whole population, not just the corpus: of the **7,596** name spellings in -JMnedict that end in kana, 7,247 stop resolving as a surface. **4,410 of those -print exactly the same ruby anyway**, because the kanji run before the kana -carries the name on its own — 帝国ホテル still reads 帝国/ていこく, and -龍ヶ嶽トンネル still reads 龍ヶ嶽. The remaining 2,837 are spellings with the -kanji buried inside kana (い乃り, み津ゑ, しら坂トンネル): prewar given-name -orthography and signposted place names, which the reader will not meet, and -where the old ruby covered a leading fragment. - -五十嵐, 杏子, 丸木, 後味 and 西條 are all unmoved — nothing written in kanji -is touched. +off, so the page got a reading over 花 alone that only the whole name has. + +**Refusing every kana-tailed name was too wide, and review caught it before +this shipped.** Two separate costs, neither visible in the corpus: + +- **33 place and person names lost their ruby entirely** — 十勝ダム, 一戸トンネル, + 三輪ひとみ — because their kanji prefix is also a `counter_readings` form, and + a counter rejected by the `showCounters` setting shadows the whole span. With + counters on, 17 of them printed the counter instead: 一戸【いっこ】. +- **The ◯◯通り street class was garbled, which is worse than absent.** + 三条通【さんじょうどお】り became 三条通【さんじょうどおり】り — the reading + absorbs the り and the page reprints it. 94 of the 101 通り spellings changed. + +So the guard is what the bug actually was: a name is refused only when the +surface ends in a **single-kana particle** (の, が, を, に, は, へ, と, も, や, か) +directly after a kanji. り, め and み are the name. + +Measured over the corpus: **13 of 67,299 surfaces change, every one a junk name +reading going away** — 上の, 仲の, 坂の, 夢か, 木の, 東の, 松の, 水の, 浜の, 理の, +育の, 高の, 魚の. Nothing gains or reads differently. Over the whole kana-tailed +population (7,467 spellings) the narrowed guard changes 308 and loses **none**, +against 2,785 changed and 33 lost for the wide one; 帝国ホテル, 龍ヶ嶽トンネル, +三条通り, お染め, 晴み and 鉞り are all back to what they read before. + +**Accepted, and inherited rather than introduced:** a handful of single-kanji +given names spelled with one trailing particle still lose — 秋を (Akio) reads +秋【とき】を. All carry a frequency of 1 or 2, and the fallback is the kanji's +own word reading. + +**One thing this does not reach:** a name entry short-circuits the JLPT level +filters and the word entry that replaces it does not, so under a restricted +level config 165 surfaces lose ruby that the name was carrying past the filter. +That is `entry.isName ||` in `shouldShowFuriganaForSurface`, not this guard. + +五十嵐, 杏子, 丸木, 後味 and 西條 are unmoved — nothing written in kanji is +touched. ### A two-kanji name the counts have seen may override a one-kanji word @@ -982,6 +1023,12 @@ guard, so with the word span refused, 注文したから came back as the surnam たから. Auto mode now computes the starts once and hands them to both walks (`okuriganaStartsForTap`). +**And NAME mode, which review caught after the fact.** `autoLookupWithOffset` +computes the starts and hands them to both walks, but the reader calls +`nameLookupWithOffset` directly when the user switches to Names, and that call +had no starts at all — so name mode went on answering 注文したから with たから. +It computes them too now, when the word dictionary is open. + **And then the name walk's own hole.** A name may override a shorter word outright when the surface is "an exact name match", and that accepted a match on the name's **reading**: ためだった holds the reading of 為田, したから holds @@ -1031,7 +1078,12 @@ which is what 姿 → 姿勢 depends on. **The selection cases in `lib/smart-lookup.test.ts` re-implement the walk** in the test file rather than calling it, so they cannot see this change at all — they pass before and after. `lookup.selection.test.ts` runs the shipping -function. +function — including `autoSelectionLookup`, which is what the reader's default +mode actually calls and which nothing covered until review said so. + +**The expansion step had no literal-first gate either**, which is where the +negative rules above would have spoken over an entry on a drag. It has one now, +as do `findFirstWord` and `findBoundaryWord`. ## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way diff --git a/lib/deinflect.test.ts b/lib/deinflect.test.ts index fa3ab78..2d27c22 100644 --- a/lib/deinflect.test.ts +++ b/lib/deinflect.test.ts @@ -667,7 +667,9 @@ describe("Deinflection algorithm properties", () => { test("empty string returns just itself", () => { const result = deinflect(""); - expect(result).toEqual([{ word: "", typeMask: 0xff, entryMask: 0xff, reasons: [] }]); + expect(result).toEqual([ + { word: "", typeMask: 0xff, entryMask: 0xff, reasons: [], guessed: false }, + ]); }); test("non-Japanese text returns just itself", () => { diff --git a/packages/japanese-reader/src/deinflect.negative-nu.test.ts b/packages/japanese-reader/src/deinflect.negative-nu.test.ts index 1c4ff1b..3044db6 100644 --- a/packages/japanese-reader/src/deinflect.negative-nu.test.ts +++ b/packages/japanese-reader/src/deinflect.negative-nu.test.ts @@ -36,11 +36,33 @@ describe("negative ～ぬ", () => { // word that already ends in ぬ. Without RAW a second pass strips the ending // the first one produced and たくさんで comes back as 託す. it("fires on the text as written, never on another rule's output", () => { - expect(bases("たくさんで")).not.toContain("託す"); + // The te-form rule んで→ぬ types these as ぬ-verbs; without RAW the negative + // rules strip the ending it just produced. たくす is the word 託す is + // spelled with, and asserting on 託す instead would be unfailable — + // deinflect only ever returns the script it was handed. + expect(bases("たくさんで")).not.toContain("たくす"); + expect(bases("飲んで")).not.toContain("飲る"); + expect(bases("読んで")).not.toContain("読る"); + expect(bases("言われたんだ")).not.toContain("言われつ"); // 死る is the ichidan rule reading 死 as a stem. No dictionary holds it, so // it costs a lookup and nothing else; 死ぬ keeps answering as itself. expect(bases("死ぬ")).toEqual(["死ぬ", "死る"]); }); + + // These rules are guesses: 弛まぬ, でなく and いななく are themselves entries, + // and a manufactured verb must not be allowed to answer them. + it("marks what it produces as a guess", () => { + const guessed = (word: string, base: string) => + deinflect(word).find((c) => c.word === base)?.guessed; + expect(guessed("畳まぬ", "畳む")).toBe(true); + expect(guessed("誘わなく", "誘う")).toBe(true); + // でる, not 出る: deinflect returns the script it was handed. + expect(guessed("でなく", "でる")).toBe(true); + // The path that does not guess keeps its standing. + expect(guessed("でなく", "でない")).toBe(false); + expect(guessed("せわしなく", "せわしない")).toBe(false); + expect(guessed("畳まぬ", "畳まぬ")).toBe(false); + }); }); // 誘わなくなった had no path at all: the adjective rule く→い reaches 誘わない, diff --git a/packages/japanese-reader/src/deinflect.ts b/packages/japanese-reader/src/deinflect.ts index 4c60df1..6755b1a 100644 --- a/packages/japanese-reader/src/deinflect.ts +++ b/packages/japanese-reader/src/deinflect.ts @@ -41,6 +41,12 @@ const NOT_MASU_STEM = ANY & ~V5; * span starting inside よっぽど. */ const RAW = 0x80; +/** + * `ANY` as a rule's OUTPUT. `ANY` includes `RAW`, so a rule that emits it + * hands the next rule a surface that was never written — which is how + * よぎなくさせる reached 余儀る through the suru-noun rule. + */ +const ANY_OUTPUT = ANY & ~RAW; /** An ichidan stem's last character: an え/い-row kana, or any kanji. */ const V1_STEM = @@ -62,6 +68,12 @@ interface DeinflectRule { reason: string; /** Minimum stem length left after stripping `from`. Guards 1-char inputs. */ minStem?: number; + /** + * Undoing this rule is a guess about what the page meant, so a reading that + * needs no guess wins first: 弛まぬ and でなく are entries, and answering + * them with 緩む and 出る is reaching past a word the dictionary has. + */ + guess?: boolean; /** * What the character before the ending has to be. Only an え/い-row kana or a * kanji can be an ichidan stem, so without it お小遣い**が**なくて deinflects @@ -285,7 +297,7 @@ const RULES: DeinflectRule[] = [ { from: "したり", to: "する", typeIn: SURU, typeOut: SURU, reason: "tari" }, { from: "している", to: "する", typeIn: SURU, typeOut: SURU, reason: "te-iru" }, { from: "してる", to: "する", typeIn: SURU, typeOut: SURU, reason: "te-iru (casual)" }, - { from: "する", to: "", typeIn: SURU, typeOut: ANY, reason: SURU_NOUN_REASON }, + { from: "する", to: "", typeIn: SURU, typeOut: ANY_OUTPUT, reason: SURU_NOUN_REASON }, // ── 来る (kuru) irregular ── { from: "きた", to: "くる", typeIn: KURU, typeOut: KURU, reason: "past" }, @@ -351,7 +363,7 @@ const RULES: DeinflectRule[] = [ // volitional 話そう below; both rules are tried and the dictionary decides. { from: "そう", to: "い", typeIn: ADJ, typeOut: ADJ, reason: "appearance" }, // ── Negative ～ず / ～ずに (written negative, the ～ない of literary prose) ── - { from: "ずに", to: "ず", typeIn: ANY, typeOut: ANY, reason: "without doing" }, + { from: "ずに", to: "ず", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "without doing" }, { from: "ず", to: "る", typeIn: V1, typeOut: V1, reason: "negative", minStem: 1 }, { from: "わず", to: "う", typeIn: V5, typeOut: V5, reason: "negative" }, { from: "かず", to: "く", typeIn: V5, typeOut: V5, reason: "negative" }, @@ -369,15 +381,15 @@ const RULES: DeinflectRule[] = [ // Every rule here fires on the surface as written. ～ぬ is also a verb ending // — the te-form rule んで→ぬ makes たくさんで a ぬ-verb — so without RAW a // second pass strips the ending it just produced and 託す answers たくさんで. - { from: "わぬ", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "かぬ", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "がぬ", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "さぬ", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "たぬ", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "なぬ", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "ばぬ", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "まぬ", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "らぬ", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { from: "わぬ", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "かぬ", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "がぬ", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "さぬ", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "たぬ", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "なぬ", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "ばぬ", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "まぬ", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "らぬ", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, { from: "ぬ", to: "る", @@ -386,22 +398,95 @@ const RULES: DeinflectRule[] = [ reason: "negative", minStem: 1, stemEnd: V1_STEM, + guess: true, }, - { from: "せぬ", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative" }, - { from: "こぬ", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative" }, + { from: "せぬ", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative", guess: true }, + { from: "こぬ", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative", guess: true }, // ～なく is the negative's adverbial, the half of 誘わなくなった that is the // verb. The ADJ rule く→い already reaches 誘わない and stops: nothing is // spelled that way, and its output is typed ADJ so no verb rule follows it. - { from: "わなく", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "かなく", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "がなく", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "さなく", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "たなく", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "ななく", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "ばなく", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "まなく", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, - { from: "らなく", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1 }, + { + from: "わなく", + to: "う", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "かなく", + to: "く", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "がなく", + to: "ぐ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "さなく", + to: "す", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "たなく", + to: "つ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "ななく", + to: "ぬ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "ばなく", + to: "ぶ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "まなく", + to: "む", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "らなく", + to: "る", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, { from: "なく", to: "る", @@ -410,19 +495,92 @@ const RULES: DeinflectRule[] = [ reason: "negative", minStem: 1, stemEnd: V1_STEM, + guess: true, + }, + { from: "しなく", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative", guess: true }, + { from: "こなく", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative", guess: true }, + + { + from: "わなくて", + to: "う", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "かなくて", + to: "く", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "がなくて", + to: "ぐ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "さなくて", + to: "す", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "たなくて", + to: "つ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "ななくて", + to: "ぬ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "ばなくて", + to: "ぶ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "まなくて", + to: "む", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "らなくて", + to: "る", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, }, - { from: "しなく", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative" }, - { from: "こなく", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative" }, - - { from: "わなくて", to: "う", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "かなくて", to: "く", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "がなくて", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "さなくて", to: "す", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "たなくて", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "ななくて", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "ばなくて", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "まなくて", to: "む", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, - { from: "らなくて", to: "る", typeIn: RAW, typeOut: V5, reason: "negative te-form", minStem: 1 }, { from: "なくて", to: "る", @@ -431,15 +589,30 @@ const RULES: DeinflectRule[] = [ reason: "negative te-form", minStem: 1, stemEnd: V1_STEM, + guess: true, + }, + { + from: "しなくて", + to: "する", + typeIn: RAW, + typeOut: SURU, + reason: "negative te-form", + guess: true, + }, + { + from: "こなくて", + to: "くる", + typeIn: RAW, + typeOut: KURU, + reason: "negative te-form", + guess: true, }, - { from: "しなくて", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative te-form" }, - { from: "こなくて", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative te-form" }, // ── Generic te-iru forms (works across verb types after te-form resolution) ── - { from: "ている", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru" }, - { from: "てる", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru (casual)" }, - { from: "でいる", to: "で", typeIn: ANY, typeOut: ANY, reason: "te-iru" }, - { from: "でる", to: "で", typeIn: ANY, typeOut: ANY, reason: "te-iru (casual)" }, + { from: "ている", to: "て", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru" }, + { from: "てる", to: "て", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru (casual)" }, + { from: "でいる", to: "で", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru" }, + { from: "でる", to: "で", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru (casual)" }, // ── ～なさい: the polite imperative, built on the masu-stem ── // Spelled out per row rather than stripping なさい and leaving the stem to the @@ -694,10 +867,18 @@ export interface DeinflectCandidate { */ entryMask: number; reasons: string[]; + /** Some rule on the way here was a guess — see `DeinflectRule.guess`. */ + guessed: boolean; } export function deinflect(word: string): DeinflectCandidate[] { - const first: DeinflectCandidate = { word, typeMask: ANY, entryMask: ANY, reasons: [] }; + const first: DeinflectCandidate = { + word, + typeMask: ANY, + entryMask: ANY, + reasons: [], + guessed: false, + }; const results: DeinflectCandidate[] = [first]; const seen = new Map([[word, first]]); @@ -712,6 +893,7 @@ export function deinflect(word: string): DeinflectCandidate[] { if (!(current.typeMask & rule.typeIn)) continue; const base = current.word.slice(0, stemLen) + rule.to; + const guessed = current.guessed || !!rule.guess; const already = seen.get(base); if (already) { // A second rule reaching the same word is a second reading of the @@ -719,6 +901,8 @@ export function deinflect(word: string): DeinflectCandidate[] { // and the kuru te-form rule, and 来る is kuru. Keeping only the first // rule's class would refuse the entry that is actually there. already.entryMask |= rule.typeOut; + // Reaching the same word without a guess means it was never one. + if (!guessed) already.guessed = false; continue; } const candidate: DeinflectCandidate = { @@ -726,6 +910,7 @@ export function deinflect(word: string): DeinflectCandidate[] { typeMask: rule.typeOut, entryMask: rule.typeOut, reasons: [...current.reasons, rule.reason], + guessed, }; seen.set(base, candidate); results.push(candidate); diff --git a/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts index 32eb0a3..bdb4d54 100644 --- a/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts +++ b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts @@ -1,10 +1,15 @@ /** - * A name spelling that ends in kana is not evidence about the kanji before it. + * A kanji followed by a particle is the particle. * * 花のつぼみ printed あや over 花 and 水の流れ printed にず over 水: JMnedict * holds 花の (Ayano) and 水の (Nizuno), the surface was two characters so it * beat the one-character word, and the ruby that came out covered the kanji * alone — a reading only the whole name has, over a noun and a particle. + * + * Refusing every kana-tailed name was too wide, and review caught it: it took + * the reading off 十勝ダム and 一戸トンネル entirely (their kanji prefix is a + * counter form, and a settings-rejected counter shadows the span) and garbled + * 三条通り into 三条通【さんじょうどおり】り. Only a particle is the bug. */ import Database from "better-sqlite3"; import { afterAll, describe, expect, it } from "vitest"; @@ -33,7 +38,7 @@ async function entry(surface: string): Promise<{ reading?: string; isName?: bool return map[surface] ?? {}; } -describe.skipIf(!hasBothDbs)("a name spelled with trailing kana", () => { +describe.skipIf(!hasBothDbs)("a name spelled with a trailing particle", () => { it("is not read as a name", async () => { expect((await entry("花の")).isName).toBeFalsy(); expect((await entry("水の")).isName).toBeFalsy(); @@ -41,17 +46,38 @@ describe.skipIf(!hasBothDbs)("a name spelled with trailing kana", () => { expect((await entry("木の")).isName).toBeFalsy(); }); + // A guard, not a gate: these read the same before and after the change. it("leaves the kanji run itself to the word it is", async () => { expect((await entry("花")).reading).toBe("はな"); expect((await entry("水")).reading).toBe("みず"); }); - // The kanji half of a place name still carries it, which is what the page - // shows either way: the ruby never covered the katakana. + it("leaves a name whose tail is not a particle alone", async () => { + // り is the name's own okurigana, and the reading covers 三条通 without it. + expect((await entry("三条通り")).reading).toBe("さんじょうどお"); + expect((await entry("十勝ダム")).isName).toBe(true); + expect((await entry("一戸トンネル")).isName).toBe(true); + expect((await entry("お染め")).reading).toBe("おそ"); + }); + + // Guards, not gates: both read the same before and after the change. it("keeps the reading on the kanji the ruby actually covered", async () => { expect((await entry("帝国")).reading).toBe("ていこく"); }); + // The same doctrine on the furigana side: 拠なく and 得なく are 拠ない and + // 得ない, entries, and must not be read as 拠る and 得る because those are + // commoner. `scoreFuriganaWordMatch` pays commonness +120 against a + // deinflection penalty of −80, so this has to be a preference, not a weight. + it("prefers a reading that needs no guess", async () => { + expect((await entry("拠なく")).reading).toBe("よんどころ"); + expect((await entry("得なく")).reading).toBe("え"); + expect((await entry("弛まなく")).reading).toBe("たゆ"); + // And a guess still speaks where nothing else does. + expect((await entry("畳まぬ")).reading).toBe("たた"); + expect((await entry("誘わなく")).reading).toBe("さそ"); + }); + it("leaves a name written in kanji alone", async () => { expect((await entry("西條")).isName).toBe(true); expect((await entry("杏子")).reading).toBe("きょうこ"); diff --git a/packages/japanese-reader/src/furigana.ts b/packages/japanese-reader/src/furigana.ts index 19d7b37..42b6127 100644 --- a/packages/japanese-reader/src/furigana.ts +++ b/packages/japanese-reader/src/furigana.ts @@ -584,15 +584,20 @@ export function nameReadingDominance( return { share: (best.freq ?? 0) / total, total }; } +/** The single-kana particles, the only kana a name's tail is mistaken for. */ +const PARTICLE_KANA = new Set(["の", "が", "を", "に", "は", "へ", "と", "も", "や", "か"]); + function shouldConsiderNameFuriganaSurface(surface: string): boolean { // 四十三 and 五十八 are given names in JMnedict and numbers everywhere else. // A run written only in numerals is a number. if (!hasKanjiText(surface) || isKanjiNumeralRun(surface)) return false; - // A name's trailing kana is part of its spelling, not okurigana, so the ruby - // would cover the kanji alone and claim a reading only the longer name has: - // 花の is the surname Ayano, and in prose it is 花 and the particle. + // A kanji followed by a particle is the particle: 花の is the surname Ayano + // and 水の is Nizuno, but in prose they are 花 and 水 and the の, and the ruby + // that came out claimed a reading over the kanji that only the name has. + // Only a particle — the り of 三条通り and the み of 晴み are the name. const chars = [...surface]; - return !isKana(chars[chars.length - 1]); + const last = chars[chars.length - 1]; + return !(PARTICLE_KANA.has(last) && chars.length > 1 && isKanji(chars[chars.length - 2])); } function resolveLexicalSuffixJlpt( @@ -600,7 +605,9 @@ function resolveLexicalSuffixJlpt( lookupMap: Map, ): number | null | undefined { const tryWords = (candidateSurface: string): number | null | undefined => { - const candidateWords = deinflect(candidateSurface).map((c) => c.word); + const candidateWords = deinflect(candidateSurface) + .filter((c) => !c.guessed) + .map((c) => c.word); for (const word of candidateWords) { const match = lookupMap.get(word); if (match?.jlptLevel != null) return match.jlptLevel; @@ -644,22 +651,31 @@ export async function resolveFuriganaBatch( ): Promise> { const { includeNames = true, includeCounters = true } = options; // Deinflect all surfaces, collect unique search words - const surfaceToDeinflected = new Map(); + const surfaceToDeinflected = new Map(); const allSearchWords = new Set(); for (const surface of surfaces) { - const candidates = deinflect(surface); - const words = candidates.map((c) => c.word); + const words: { word: string; guessed: boolean }[] = []; + const at = new Map(); + // A word reachable both ways is not a guess, so the flag is merged rather + // than taken from whichever path was seen first. + const add = (word: string, guessed: boolean) => { + const seen = at.get(word); + if (seen === undefined) { + at.set(word, words.length); + words.push({ word, guessed }); + } else if (!guessed) { + words[seen].guessed = false; + } + }; + for (const c of deinflect(surface)) add(c.word, c.guessed); // Also try digit→kanji normalized forms (e.g. １人 → 一人) const normalized = normalizeDigitsToKanji(surface); if (normalized !== surface) { - const normCandidates = deinflect(normalized); - for (const c of normCandidates) { - if (!words.includes(c.word)) words.push(c.word); - } + for (const c of deinflect(normalized)) add(c.word, c.guessed); } surfaceToDeinflected.set(surface, words); - for (const w of words) allSearchWords.add(w); + for (const w of words) allSearchWords.add(w.word); } // Batch lookup @@ -683,12 +699,19 @@ export async function resolveFuriganaBatch( deinflectedWord: string; } | null = null; - for (const word of deinflected) { - const match = lookupMap.get(word); - if (!match || !match.kanaForm) continue; - const score = scoreFuriganaWordMatch(surface, match, word); - if (!bestWordMatch || score > bestWordMatch.score) { - bestWordMatch = { match, score, deinflectedWord: word }; + // Two passes, not one score: a guess is only allowed to speak when + // reading the surface as written found nothing. 弛まなく is 弛まない, an + // entry, and must not be answered with 弛む because 弛む is commoner. + for (const pass of [false, true]) { + if (bestWordMatch) break; + for (const { word, guessed } of deinflected) { + if (guessed !== pass) continue; + const match = lookupMap.get(word); + if (!match || !match.kanaForm) continue; + const score = scoreFuriganaWordMatch(surface, match, word); + if (!bestWordMatch || score > bestWordMatch.score) { + bestWordMatch = { match, score, deinflectedWord: word }; + } } } diff --git a/packages/japanese-reader/src/lookup.selection.test.ts b/packages/japanese-reader/src/lookup.selection.test.ts index 58f16e6..6bde45c 100644 --- a/packages/japanese-reader/src/lookup.selection.test.ts +++ b/packages/japanese-reader/src/lookup.selection.test.ts @@ -12,7 +12,16 @@ import Database from "better-sqlite3"; import { afterAll, describe, expect, it } from "vitest"; import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; -import { selectionLookup } from "./lookup"; +import { autoSelectionLookup as autoSelection, selectionLookup } from "./lookup"; + +async function autoSelectionLookup( + dict: ReturnType, + text: string, + prefix: string, + suffix: string, +) { + return autoSelection(text, dict, wrap(ext!), { prefix, suffix }); +} const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; @@ -54,6 +63,15 @@ describe.skipIf(!hasBothDbs)("a selection that is itself a word", () => { }); }); +describe.skipIf(!hasBothDbs)("auto mode, which is what a drag actually runs", () => { + // The reader's default mode calls autoSelectionLookup, not selectionLookup, + // and nothing covered it until review pointed that out. + it("answers the selection first there too", async () => { + const results = await autoSelectionLookup(wrap(dict!), "展開", "チェーン", "の新古書店"); + expect(results.map((r) => r.matchedText)).toEqual(["展開", "チェーン展開"]); + }); +}); + describe.skipIf(!hasBothDbs)("a selection that is not a word", () => { it("still reaches past its edges", async () => { expect(await select("色い", "茶", "シャツ")).toEqual(["茶色い"]); diff --git a/packages/japanese-reader/src/lookup.span-starts.test.ts b/packages/japanese-reader/src/lookup.span-starts.test.ts index 8bffef0..19b9625 100644 --- a/packages/japanese-reader/src/lookup.span-starts.test.ts +++ b/packages/japanese-reader/src/lookup.span-starts.test.ts @@ -14,7 +14,12 @@ import Database from "better-sqlite3"; import { afterAll, describe, expect, it } from "vitest"; import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; -import { autoLookupWithOffset, smartLookupWithOffset } from "./lookup"; +import { + autoLookupWithOffset, + nameLookupWithOffset, + okuriganaStartsForTap, + smartLookupWithOffset, +} from "./lookup"; const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; @@ -50,8 +55,38 @@ describe.skipIf(!hasBothDbs)("a suru-verb noun proves its own kana tail", () => }); it("leaves a noun's trailing particle alone", async () => { - // 注意に is not 注意する in any form, so に stays open. - expect((await word("注意にも程がある", 3))?.matchedText).not.toBe("注意に"); + // 注意に is not 注意する in any form, so に stays open: the kanji answer the + // noun and the kana answer the particle. Asserting only "not 注意に" passed + // before this change too, which is no gate at all. + expect((await word("注意にも程がある", 0))?.matchedText).toBe("注意"); + expect((await word("注意にも程がある", 2))?.matchedText).toBe("にも"); + }); +}); + +describe.skipIf(!hasBothDbs)("name mode walks the same spans", () => { + // Name mode has its own walk, and it got the starts only after review: + // without them it answered 注文したから with the surname たから and + // したためだった with 為田, from inside words the word walk had closed. + it("refuses a start inside a word", async () => { + const starts = await okuriganaStartsForTap( + "切ってみろと注文したから、何だ指ぐらい", + 10, + wrap(dict!), + ); + const guarded = await nameLookupWithOffset( + "切ってみろと注文したから、何だ指ぐらい", + 10, + wrap(ext!), + starts, + ); + expect(guarded[0]?.matchedText).not.toBe("たから"); + }); + + it("still answers a name that starts at a word boundary", async () => { + const text = "演じた西條さんがそばに来て"; + const starts = await okuriganaStartsForTap(text, 3, wrap(dict!)); + const hit = await nameLookupWithOffset(text, 3, wrap(ext!), starts); + expect(hit[0]?.matchedText).toBe("西條"); }); }); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index 5dac3a5..d750c93 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -42,12 +42,31 @@ interface EntrySortContext { type KanjiReadingRecord = Awaited>; +/** + * Undoing a contraction, swapping a particle and the literary negatives are + * all guesses about what the page meant, and a reading that needs no guess + * wins first — so a guess only speaks when the span found nothing else. + */ +function candidateIsGuess(candidate: { + reasons: string[]; + particleSwapped?: boolean; + guessed?: boolean; +}): boolean { + return ( + !!candidate.particleSwapped || + !!candidate.guessed || + candidate.reasons.some((reason) => CONTRACTION_REASONS.has(reason)) + ); +} + /** One spelling the tap walk will ask the dictionary about, and why. */ interface TapCandidate { word: string; reasons: string[]; spelledKanji?: string; particleSwapped?: boolean; + /** A rule on the way here was a guess — see `DeinflectRule.guess`. */ + guessed?: boolean; } const DIGIT_TO_KANJI: Record = { @@ -905,10 +924,14 @@ export async function selectionLookup( let best: LookupResult | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; const entries = await lookupExactJapanese(dictDb, candidate.word); if (entries.length > 0) { + if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( entries, substr, @@ -1023,12 +1046,16 @@ async function findFirstWord( // Try all candidates for this substring length, prefer common entries let best: { result: LookupResult; matchLength: number } | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { + if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1094,12 +1121,16 @@ async function findBoundaryWord( let best: LookupResult | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { + if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1154,13 +1185,17 @@ export async function smartLookup( for (const substr of substrings) { const candidates = deinflect(substr); + let literalMatched = false; for (const candidate of candidates) { + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { + if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1280,9 +1315,7 @@ export async function smartLookupWithOffset( // commoner of the two. let literalMatched = false; for (const candidate of candidates) { - const guessed = - candidate.particleSwapped || - candidate.reasons.some((reason) => CONTRACTION_REASONS.has(reason)); + const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; let entries = await lookupOnce(candidate.word); // Rewriting a kanji as kana claims the entry is that word spelled diff --git a/packages/japanese-reader/src/use-japanese-reader.ts b/packages/japanese-reader/src/use-japanese-reader.ts index 7a97236..338e713 100644 --- a/packages/japanese-reader/src/use-japanese-reader.ts +++ b/packages/japanese-reader/src/use-japanese-reader.ts @@ -42,6 +42,7 @@ import { autoSelectionLookup, nameLookup, nameLookupWithOffset, + okuriganaStartsForTap, selectionLookup, smartLookup, smartLookupWithOffset, @@ -1602,9 +1603,16 @@ export function useJapaneseReader({ } } else { const tapOffset = msg.tapOffset as number | undefined; + // Name mode walks spans too, so it refuses the same starts a word + // walk does: without this, 注文したから answers the surname たから + // from inside 注文した. + const nameModeStarts = + isNameMode && dictDb && tapOffset && tapOffset > 0 + ? await okuriganaStartsForTap(text, tapOffset, dictDb) + : undefined; const results = isNameMode ? tapOffset && tapOffset > 0 - ? await nameLookupWithOffset(text, tapOffset, extendedDb!) + ? await nameLookupWithOffset(text, tapOffset, extendedDb!, nameModeStarts) : await nameLookup(text, extendedDb!) : isAutoMode ? tapOffset && tapOffset > 0 From 497db6af863390e3f95a14a5be3a533dcead3c32 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 16:29:20 +0800 Subject: [PATCH 10/14] A number is a count until the counts say otherwise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round two found the two-kanji override handing 鳥が二羽いた to the name ふたば, 全三巻 to みつまき, 犬が二戸 to にと and 第三章 to さんしょう — every one a span of the app's own counter_readings table, every one also a name somebody has been seen with once or twice, and every one returned as the only result with no word to fall back to. A span carrying a kanji numeral now needs three sightings instead of one. That clears all of those at 1-2 and costs nothing measurable: 一郎 is 393, 三郎 334, 七海 38, and the corpus diff is unmoved at 12 name wins. Also from that review: the wider bound is now recorded in the doc rather than left at the two straddles it was argued from — 36,405 two-kanji spellings that are not words can outrank a one-character word, 18,212 on a single sighting. The cache refactor's dead binding is gone, and the stem guard reads a code point rather than a UTF-16 unit, which was refusing a legitimate ichidan stem written with a surrogate pair. 616 tests across 32 suites; gate unmoved at 123 disagreeing pairs. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 26 +++++++++++++++- .../src/auto-name.override.test.ts | 31 ++++++++++++++++--- packages/japanese-reader/src/auto-name.ts | 23 ++++++++++---- packages/japanese-reader/src/deinflect.ts | 6 +++- packages/japanese-reader/src/lookup.ts | 6 ++-- 5 files changed, 76 insertions(+), 16 deletions(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index c4ac35b..630cfd5 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -1005,6 +1005,30 @@ names in the book that a tap used to answer with a single kanji** — 茂作 (wa The floor itself is untouched above two characters, and an uncounted two-kanji spelling is still refused — which is the whole of what keeps 田先 out. +**A number is a count until the counts say otherwise.** Review found the +relaxation handing 鳥が**二羽**いた to ふたば, 全**三巻** to みつまき, 犬が**二戸** +to にと and 第**三章** to さんしょう — each a real span of the app's own +`counter_readings` table, each also a name somebody has been seen with once or +twice, and each returned as the _only_ result with no word to fall back to. A +span carrying a kanji numeral now needs **three** sightings rather than one +(`AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ`). That clears every one of those at 1–2 +and costs nothing real: 一郎 is 393, 三郎 334, 七海 38, and the corpus diff is +unmoved at 12. 1,334 of the 2,242 numeral-carrying two-kanji spellings lose the +override; 908 keep it. + +**The bound, recorded because it is much wider than the two straddles above.** +**36,405** two-kanji spellings that are not dictionary words now outrank a +one-character word, **18,212 of them on a single sighting**. A synthetic frame +— `その{A}{B}を見た。` over the 11,248 pairs whose characters are each a common +standalone word and whose pair is not a word — flips **21,598 of 22,496 taps** +from word to name. Taken as shipped, because the frame is the thing being +measured there, not the language: two adjacent single-kanji words with no kana +between them is rare in real prose, which is why the same relaxation moves 12 +taps in 11,296 of Bocchan and every one is a name in the book. If this ever +reads wrong in the wild, the fix is a frequency floor set from the +distribution, not another guard. `yarn check:tap-consistency` is word-mode and +cannot see any of this. + ### A suru-verb noun proves its own kana tail, and the name walk refuses the same starts 海外へ赴任した**ため**だった answered したため — 認む, "to write down", in its @@ -1023,7 +1047,7 @@ guard, so with the word span refused, 注文したから came back as the surnam たから. Auto mode now computes the starts once and hands them to both walks (`okuriganaStartsForTap`). -**And NAME mode, which review caught after the fact.** `autoLookupWithOffset` +**And NAME mode, which review caught twice.** `autoLookupWithOffset` computes the starts and hands them to both walks, but the reader calls `nameLookupWithOffset` directly when the user switches to Names, and that call had no starts at all — so name mode went on answering 注文したから with たから. diff --git a/packages/japanese-reader/src/auto-name.override.test.ts b/packages/japanese-reader/src/auto-name.override.test.ts index d301e9f..51415db 100644 --- a/packages/japanese-reader/src/auto-name.override.test.ts +++ b/packages/japanese-reader/src/auto-name.override.test.ts @@ -36,17 +36,30 @@ async function tap(text: string, at: number) { describe("the override floor", () => { it("lets a counted two-kanji name through", () => { - expect(nameMayOverrideShorterWord(2, 24)).toBe(true); - expect(nameMayOverrideShorterWord(2, 1)).toBe(true); + expect(nameMayOverrideShorterWord("西條", 24)).toBe(true); + expect(nameMayOverrideShorterWord("箱根", 1)).toBe(true); }); it("keeps an uncounted one out", () => { - expect(nameMayOverrideShorterWord(2, null)).toBe(false); - expect(nameMayOverrideShorterWord(2, 0)).toBe(false); + // An extended DB built before the frequency column has no counts at all, + // and every two-kanji override silently switches off rather than guessing. + expect(nameMayOverrideShorterWord("田先", null)).toBe(false); + expect(nameMayOverrideShorterWord("中電", 0)).toBe(false); }); it("leaves three characters where they were", () => { - expect(nameMayOverrideShorterWord(3, null)).toBe(true); + expect(nameMayOverrideShorterWord("大泉学園", null)).toBe(true); + }); + + // 二羽 is two birds, 三巻 is volume three, 三章 is chapter three — each also + // a name somebody has been seen with once. A number is a count until the + // counts say otherwise. + it("asks a span carrying a numeral for more than one sighting", () => { + expect(nameMayOverrideShorterWord("二羽", 2)).toBe(false); + expect(nameMayOverrideShorterWord("三章", 1)).toBe(false); + expect(nameMayOverrideShorterWord("計三", 2)).toBe(false); + expect(nameMayOverrideShorterWord("一郎", 393)).toBe(true); + expect(nameMayOverrideShorterWord("七海", 38)).toBe(true); }); }); @@ -66,8 +79,16 @@ describe.skipIf(!hasBothDbs)("a two-kanji surname under the finger", () => { expect((await tap("同じ数学の教師に堀田というのが居た", 8))?.matchedText).toBe("堀田"); }); + // A guard, not a gate: this is the pre-existing floor's behaviour, kept + // under the relaxation rather than proved by it. it("still refuses a span straddling a word boundary", async () => { expect((await tap("山田先生のお宅へ行った", 1))?.matchedText).not.toBe("田先"); expect((await tap("食事中電話が鳴った", 2))?.matchedText).not.toBe("中電"); }); + + it("answers a count with the counter, not with a name", async () => { + expect((await tap("鳥が二羽いた。", 3))?.lookupKind).not.toBe("name"); + expect((await tap("全三巻を買った。", 2))?.lookupKind).not.toBe("name"); + expect((await tap("第三章まで進んだ。", 2))?.lookupKind).not.toBe("name"); + }); }); diff --git a/packages/japanese-reader/src/auto-name.ts b/packages/japanese-reader/src/auto-name.ts index 5ead431..f748b64 100644 --- a/packages/japanese-reader/src/auto-name.ts +++ b/packages/japanese-reader/src/auto-name.ts @@ -27,16 +27,27 @@ export const AUTO_NAME_OVERRIDE_MIN_LENGTH = 3; */ export const AUTO_NAME_OVERRIDE_COUNTED_LENGTH = 2; export const AUTO_NAME_OVERRIDE_MIN_FREQ = 1; +/** + * What a span carrying a numeral has to show instead. 二羽 is two birds, 三巻 + * is volume three and 三章 is chapter three, and each is also a name somebody + * has been seen with once or twice — so a number is a count until the counts + * say otherwise. 一郎 (393), 七海 (38) and 三郎 (334) clear this comfortably. + */ +export const AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ = 3; + +const KANJI_NUMERALS = new Set([..."一二三四五六七八九十百千万億〇零壱弐参"]); export function nameMayOverrideShorterWord( - matchedLength: number, + matchedText: string, topFreq: number | null | undefined, ): boolean { - if (matchedLength >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; - return ( - matchedLength >= AUTO_NAME_OVERRIDE_COUNTED_LENGTH && - (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_MIN_FREQ - ); + const chars = [...matchedText]; + if (chars.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; + if (chars.length < AUTO_NAME_OVERRIDE_COUNTED_LENGTH) return false; + const floor = chars.some((ch) => KANJI_NUMERALS.has(ch)) + ? AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ + : AUTO_NAME_OVERRIDE_MIN_FREQ; + return (topFreq ?? 0) >= floor; } export interface AutoNameWordCandidate { diff --git a/packages/japanese-reader/src/deinflect.ts b/packages/japanese-reader/src/deinflect.ts index 6755b1a..64a2b8b 100644 --- a/packages/japanese-reader/src/deinflect.ts +++ b/packages/japanese-reader/src/deinflect.ts @@ -889,7 +889,11 @@ export function deinflect(word: string): DeinflectCandidate[] { const stemLen = current.word.length - rule.from.length; if (stemLen + rule.to.length <= 0) continue; if (rule.minStem != null && stemLen < rule.minStem) continue; - if (rule.stemEnd && !rule.stemEnd.test(current.word[stemLen - 1] ?? "")) continue; + // By code point: a surrogate pair indexed as a code unit tests half a + // kanji, and the guard would refuse a legitimate stem. + if (rule.stemEnd && !rule.stemEnd.test([...current.word.slice(0, stemLen)].pop() ?? "")) { + continue; + } if (!(current.typeMask & rule.typeIn)) continue; const base = current.word.slice(0, stemLen) + rule.to; diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index d750c93..c096df4 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -602,7 +602,7 @@ function makeEntryCache(dictDb: ReaderSqlDb) { const found = await lookupExactJapaneseMany(dictDb, missing); for (const word of missing) entryCache.set(word, found.get(word) ?? []); }; - return { entryCache, lookupOnce, prefetch }; + return { lookupOnce, prefetch }; } /** @@ -784,7 +784,7 @@ function chooseAutoLookupVariants( nameIsLonger && (nameConfidence >= nameOnlyConfidence || (nameSpellingIsSurface(bestName) && - nameMayOverrideShorterWord(bestName.matchedText.length, bestName.nameMatches?.[0]?.freq))) + nameMayOverrideShorterWord(bestName.matchedText, bestName.nameMatches?.[0]?.freq))) ) { return taggedNames; } @@ -1237,7 +1237,7 @@ export async function smartLookupWithOffset( if (negativeScopeParticle) return [asWordLookupResult(negativeScopeParticle)]; const kanjiCache = new Map(); - const { entryCache, lookupOnce, prefetch } = makeEntryCache(dictDb); + const { lookupOnce, prefetch } = makeEntryCache(dictDb); /** * Spellings of `substr` with its one kanji written as a reading instead. From 65cea5ec137091e2d8410727f86a3f0a0bef1923 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 17:12:07 +0800 Subject: [PATCH 11/14] A guess has to land on a word it could have inflected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sweeping the whole corpus rather than the 12,000-character slice — the reviewer's point, and it was right — found the ～なく rules reaching する and the suru-noun strip handing back whatever stood in front of it: 釣や猟をしなくっちゃ answered the particle を, 足音もしなく answered 喪, ものでなくっちゃ answered ので, 何にもせぬ swallowed せぬ to answer the adverb 何にも. A guess claims the surface is an inflected word, so what it lands on has to be one. The scope was measured twice before it was set: over every suru-noun strip the same filter moves 869 taps and takes 正直にしていれば from 直に to the broth 煮汁; over the contractions and particle swaps, which have their own filters already, it takes 虫の好かない to 良く and すると to と. On the literary negatives alone it moves 16. Measured whole, 37,411 taps against origin/master: 233 word-mode changes across 126 transitions and 2,036 auto-mode, every transition read and the costs named in the doc — chiefly 安心したらしかった, where 安心したら is a conditional that proves ら closed and nothing in the kana says otherwise. Gate 128 disagreeing pairs → 125; 619 tests. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 65 +++++++++++++++++-- .../src/lookup.span-starts.test.ts | 30 +++++++++ packages/japanese-reader/src/lookup.ts | 14 ++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 630cfd5..0170e2d 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -936,6 +936,63 @@ near the table — it collides with sentence-final ん, with のだ→んだ and every noun ending in ん, which is the over-broad shape that produced たくさんで→託す in the first place. +### What a guess is allowed to land on, and the branch measured whole + +A reviewer pointed out that the sections above each measure a 12,000-character +slice, not the corpus, and that the whole file is four times that. Measured +whole — **37,411 taps**, every character of `test/corpus/bocchan.txt`, this +branch against `origin/master`: + +- **Word mode: 233 change**, 126 distinct transitions, all read. +- **Auto mode: 2,036 change** — 1,752 a kana run that was answered with a + JMnedict name and is now answered with a word, 220 the word-mode changes, 60 + a real name the tap used to answer with a single kanji, 2 lateral. +- **`yarn check:tap-consistency`: 128 disagreeing pairs → 125.** The per-commit + numbers above (126, 123) were each measured on their own commit; the branch + as a whole lands at 125, the two pairs being the entry filter below refusing + a span the gate had counted as agreeing. + +That sweep also found the last defect, which the slice could not: the ～なく +rules reach する, the suru-noun rule then strips it, and what came back was +whatever stood in front — 釣や猟を**しなく**っちゃ answered the particle を, +足音も**しなく** answered 喪, ものでなく**っちゃ** answered ので, and 何にもせぬ +swallowed せぬ to answer the adverb 何にも. A guess claims the surface is an +inflected word, so the entry it lands on now has to be one (`entryInflects` or +`entryTakesSuru`). + +**That filter is on the literary negatives only, and the scope was measured +twice.** Over every suru-noun strip it moves 869 taps and takes +正直**にしていれば** from 直に to the broth 煮汁, 出たり**はいったり**して from +入る to 配流. Over the contractions and particle swaps — which have their own +filters already — it takes 虫の好かない from the phrase to 良く and すると to the +particle と. Narrowed to candidates this branch introduced, it moves 16. + +**The costs in those 233, named.** All but one are the suru-noun proof closing +a position that is genuinely ambiguous, which is the trade `11eeebc` already +records: + +- 安心した**らしかった** → しかった (4 taps). 安心したら is the conditional and + proves ら closed; here した is past and らしかった is 〜らしい. Nothing in the + kana separates them. +- 遠慮**するがいい** → がい (6), 説明**しようがない** → がない (3), + 勘当**すると** → と (4), 心配**しようと** → と (2), **しろと** → と (2), + **したく** → くらい, **したに** → に, **したて** → て (3 between them). Same + shape: a suru-verb's own kana closing a position a longer entry wanted. +- 足音**もしなく** → 模する (2). もする is a real suru verb, so both filters pass + it; `origin/master` answered もし there, which is no better. A commonness + requirement would clear it and would cost 出揃わぬ, which is uncommon and + right. +- なくなった → なった/ナウ (9), **くっちゃ寝** → ちゃ (3), なくてもいい → いいん + (3), なくては → は (2). Tails left over where a longer correct span now wins + in front of them. + +Against roughly 185 taps that are the point: 知らぬ, 入らぬ, 知れぬ, 切れぬ, +消えぬ, 思えぬ, 心得ぬ, 似ぬ, 劣らぬ, 分らぬ, わからぬ, 出揃わぬ, もせぬ, はせぬ +(馳せる, where it was 蓮), 差支えなく, 行かなく, 聞かなく, 聞かなくて, 心配しなく, +鳴らなくて, 及ぼさなくて, 仰がれなくて, 待ってなくて, 動かなく, 殺さなく, 笑わなく, +寝なく, 癪に障らなく, 出来なく, 出なく, and the suru-verb splits やる, やろう, +やった, くれ, くれた, から, もの. + ### A kanji followed by a particle is the particle 花のつぼみのような printed **あや** over 花, and 水の流れ printed **にず** over @@ -997,8 +1054,8 @@ in 34, while 田先 and 中電 have never been observed as names at all. A two-kanji name with any observed frequency may now override (`nameMayOverrideShorterWord`). -Measured over **11,296** auto-mode corpus taps: **12 change, all of them -names in the book that a tap used to answer with a single kanji** — 茂作 (was +Measured over the first **11,296** auto-mode corpus taps: **12 change, all of +them names in the book that a tap used to answer with a single kanji** — 茂作 (was 作), 箱根 (箱, 根), 小倉 (小, 倉), 古賀 (古, 賀), 浅井 (井, three times) and 堀田 (堀, 田). Nothing else moves, in either direction. @@ -1060,8 +1117,8 @@ on the name's **reading**: ためだった holds the reading of 為田, した its kanji form, or the whole spelling for a name written in kana, so ひろし and たけし still resolve (`nameSpellingIsSurface`). -Measured over 11,296 corpus taps. **Word mode: 15 change, all of them a span -that was cutting into a suru-verb** — 注文したから→から, 勉強してやろう→やろう, +Measured over the first 11,296 corpus taps. **Word mode: 15 change, all of +them a span that was cutting into a suru-verb** — 注文したから→から, 勉強してやろう→やろう, 合点したもの→もの, 紹介してやる→やる, 心配しなくって→って, 珍重してく→くれた. **Auto mode: 489 change — 472 of them a kana run that was being answered with a JMnedict name and is now answered with a word**: たから→から (24 taps), diff --git a/packages/japanese-reader/src/lookup.span-starts.test.ts b/packages/japanese-reader/src/lookup.span-starts.test.ts index 19b9625..d67c38e 100644 --- a/packages/japanese-reader/src/lookup.span-starts.test.ts +++ b/packages/japanese-reader/src/lookup.span-starts.test.ts @@ -63,6 +63,36 @@ describe.skipIf(!hasBothDbs)("a suru-verb noun proves its own kana tail", () => }); }); +describe.skipIf(!hasBothDbs)("what a guess is allowed to land on", () => { + // The ～なく rules reach する, and the suru-noun strip then hands back + // whatever stood in front of it. A guess claims the surface is an inflected + // word, so the entry it lands on has to be one. + it("refuses a guess that lands on a particle or an adverb", async () => { + expect((await word("る方が楽に極まってる。釣や猟をしなくっちゃ活計が", 14))?.matchedText).toBe( + "を", + ); + expect((await word("と三人で暮していた。おやじは何にもせぬ男で、人の", 14))?.matchedText).toBe( + "何にも", + ); + expect((await word("つは魚に相違ない。生きてるものでなくっちゃ、こう", 13))?.matchedText).toBe( + "もの", + ); + }); + + // Only on a guess: the same filter over every suru-noun strip moves 869 + // corpus taps and takes 直にしていれば from 直に to the broth 煮汁. + it("leaves the suru-noun strip alone where nothing guessed", async () => { + expect((await word("正直にしていれば誰が乗じ", 2))?.matchedText).toBe("直にしていれば"); + expect((await word("勧誘されるのを断った", 0))?.entries[0]?.kanji[0]?.text).toBe("勧誘"); + expect((await word("出たりはいったりして、よう", 5))?.entries[0]?.kanji[0]?.text).toBe("入る"); + }); + + it("still lets a guess speak when it lands on a verb", async () => { + expect((await word("心配しなくてもいいと云い", 0))?.matchedText).toBe("心配しなくて"); + expect((await word("散歩に誘わなくなった", 3))?.matchedText).toBe("誘わなく"); + }); +}); + describe.skipIf(!hasBothDbs)("name mode walks the same spans", () => { // Name mode has its own walk, and it got the starts only after review: // without them it answered 注文したから with the surname たから and diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index c096df4..0fcd689 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -1336,6 +1336,20 @@ export async function smartLookupWithOffset( entry.senses.some((sense) => sense.partOfSpeech?.includes("exp")), ); } + // A guess claims the surface is an inflected word, so what it lands on + // has to be one. Without this the ～なく rules reach する and the + // suru-noun strip hands back whatever was in front of it: 釣や猟を + // しなくっちゃ answered the particle を, ものでなくっちゃ answered ので, + // 何にもせぬ swallowed せぬ to answer the adverb 何にも. + // + // Only on the literary negatives, which is where the claim comes from. + // The same filter over every suru-noun strip moves 869 corpus taps and + // takes 直にしていれば from 直に to the broth 煮汁; over the contractions + // and particle swaps, which have their own filters above, it takes + // 虫の好かない from the phrase to 良く and すると to the particle と. + if (candidate.guessed) { + entries = entries.filter((entry) => entryInflects(entry) || entryTakesSuru(entry)); + } if (entries.length > 0) { let sortedEntries = sortEntriesForMatchedSurface( entries, From b69e444fb97a483eab8bde83d3cf470ee143a984 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 17:36:18 +0800 Subject: [PATCH 12/14] A name may not cut a word in half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round five, against 236,778 characters of modern prose rather than Bocchan: the counted two-kanji override answered 東京府と東京市 with the given name きょういち, 外来音 with らいね, 提出元 with でもと — 45 wrong taps in 2,556, every one a span cutting a word the walk already knew about. The counts cannot separate those from a surname, since 京市 and 定家 are both a single sighting and only one is a name, but the surrounding text can: a common entry covering the span's first character and starting before it, or its last and ending after it. Refuses 11 of 20 real straddles and loses 0 of 42 confirmed wins, and only for the two-kanji override, because a longer name overlaps ordinary words all the time — asked of every override it takes フランクリン to フランク. Also from that round: the numeral floor is asked before the length shortcut, or 千万人 and 東洋一 ride in with no sightings; literalMatched is raised by the literal itself rather than by what survives the entries already shown; ね, よ and さ join the particle set, which takes 17 more junk name readings off kanji and loses none; and the guess gate, ANY_OUTPUT and the archaic 寝ぬ regression all now have tests or doc entries. The obvious generalisation — asking every deinflected candidate to land on something inflectable — was measured and rejected: it takes 正直にしていれば from 直に to nothing and 出たりはいったりして from 入る to 配流. Word mode unmoved at 233 of 37,411; auto 2,050; gate 125; 620 tests. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 41 +++++++++- .../src/auto-name.override.test.ts | 11 +++ packages/japanese-reader/src/auto-name.ts | 12 +-- .../src/deinflect.negative-nu.test.ts | 4 + packages/japanese-reader/src/furigana.ts | 23 +++++- .../src/lookup.span-starts.test.ts | 8 ++ packages/japanese-reader/src/lookup.ts | 78 ++++++++++++++++--- 7 files changed, 157 insertions(+), 20 deletions(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 0170e2d..ddc6989 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -944,7 +944,7 @@ whole — **37,411 taps**, every character of `test/corpus/bocchan.txt`, this branch against `origin/master`: - **Word mode: 233 change**, 126 distinct transitions, all read. -- **Auto mode: 2,036 change** — 1,752 a kana run that was answered with a +- **Auto mode: 2,050 change** — 1,766 a kana run that was answered with a JMnedict name and is now answered with a word, 220 the word-mode changes, 60 a real name the tap used to answer with a single kanji, 2 lateral. - **`yarn check:tap-consistency`: 128 disagreeing pairs → 125.** The per-commit @@ -978,6 +978,11 @@ records: 勘当**すると** → と (4), 心配**しようと** → と (2), **しろと** → と (2), **したく** → くらい, **したに** → に, **したて** → て (3 between them). Same shape: a suru-verb's own kana closing a position a longer entry wanted. +- まだ**寝ぬ** → 寝ぬ/いぬ, an all-`arch` classical entry, where it used to reach + 寝る. The sole case in 34,527 synthetic ～ぬ/～なく surfaces built from every + v5/v1 entry; the other ten literal-entry wins (叶わぬ, 思わぬ, 変わらぬ, + 漏れなく) are improvements. Letting an archaic literal lose to a guess would + clear it, and nobody has measured that rule. - 足音**もしなく** → 模する (2). もする is a real suru verb, so both filters pass it; `origin/master` answered もし there, which is no better. A commonness requirement would clear it and would cost 出揃わぬ, which is uncommon and @@ -1026,6 +1031,11 @@ population (7,467 spellings) the narrowed guard changes 308 and loses **none**, against 2,785 changed and 33 lost for the wide one; 帝国ホテル, 龍ヶ嶽トンネル, 三条通り, お染め, 晴み and 鉞り are all back to what they read before. +**ね, よ and さ are in the set too**, which review measured at 17 more junk name +readings going away (多ね, 晴さ, 嘉よ, 寧ね, 理さ, 里よ) and none lost. で earns no +place: no name spelling ends in a kanji plus で. The corpus is neutral to all of +them — the 13 surfaces it moves are the の cases. + **Accepted, and inherited rather than introduced:** a handful of single-kanji given names spelled with one trailing particle still lose — 秋を (Akio) reads 秋【とき】を. All carry a frequency of 1 or 2, and the fallback is the kanji's @@ -1070,9 +1080,36 @@ twice, and each returned as the _only_ result with no word to fall back to. A span carrying a kanji numeral now needs **three** sightings rather than one (`AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ`). That clears every one of those at 1–2 and costs nothing real: 一郎 is 393, 三郎 334, 七海 38, and the corpus diff is -unmoved at 12. 1,334 of the 2,242 numeral-carrying two-kanji spellings lose the +unmoved at 12. The floor is asked **before** the three-character shortcut, or +千万人 (no sightings at all) and 東洋一 ride in on length alone; over the corpus +that takes 五畳敷, 二十歩, 五十人 and 五万石 off the name walk, counts in all 14 +of their taps. 1,334 of the 2,242 numeral-carrying two-kanji spellings lose the override; 908 keep it. +**A name may not cut a word in half.** Reviewed against 236,778 characters of +modern prose rather than Bocchan — Wikipedia, bibliography-dense, the hardest +case for a name walk — the counted override answered 東京府と東京**市** with the +given name きょういち (15 taps), 外来**音** with らいね, 提出**元** with でもと and +分野**別** with のべつ: 45 wrong taps in 2,556, every one a span cutting a word +the walk already knew about (東京|市, 外来|音, 提出|元). The counts cannot +separate those from a surname — 京市 and 定家 are both one sighting and only one +is a name — but the surrounding text can, which is the question +`findOkuriganaStarts` asks of kana asked of kanji: a **common** entry that +covers the span's first character and starts before it, or its last and ends +after it, three characters out either way (`spanCutsAWord`). + +Measured on the reviewer's 20 real-prose straddles and 42 confirmed wins: it +refuses **11 of 20** straddles and loses **0 of 42** wins — 西條, 堀田, 華山, +吉川, 浅井, 茂作, 箱根, 小倉, 古賀, 池上, 定家, 鈴木, 佐藤 all keep theirs. Only +common entries answer, because any entry at reach 3 refuses 14 and costs 滝山 +(to 山城) and 家康 (to 徳川家). The nine it still allows are a name cutting +another name — 杉[藤]美代子, 桜[井]茂治 — where the neighbour is in no dictionary +at all, and catching those would be one name vouching for another. + +**Only for the two-kanji override.** A longer name overlaps an ordinary word all +the time: asking the same question of every override takes フランクリン to +フランク and ゴルキ to ゴ, 125 taps of real names in this corpus alone. + **The bound, recorded because it is much wider than the two straddles above.** **36,405** two-kanji spellings that are not dictionary words now outrank a one-character word, **18,212 of them on a single sighting**. A synthetic frame diff --git a/packages/japanese-reader/src/auto-name.override.test.ts b/packages/japanese-reader/src/auto-name.override.test.ts index 51415db..91c090f 100644 --- a/packages/japanese-reader/src/auto-name.override.test.ts +++ b/packages/japanese-reader/src/auto-name.override.test.ts @@ -86,6 +86,17 @@ describe.skipIf(!hasBothDbs)("a two-kanji surname under the finger", () => { expect((await tap("食事中電話が鳴った", 2))?.matchedText).not.toBe("中電"); }); + // 東京|市 is not the given name きょういち. The counts cannot tell a straddle + // from a surname — 京市 and 定家 are both one sighting — but the text can. + it("refuses a two-kanji name that cuts a word in half", async () => { + expect((await tap("これにより東京府と東京市が廃止され", 11))?.lookupKind).not.toBe("name"); + expect((await tap("その他の外来音を含める場合は", 6))?.lookupKind).not.toBe("name"); + }); + + it("leaves a longer name alone, which overlaps words all the time", async () => { + expect((await tap("云うならフランクリンの自伝", 4))?.matchedText).toBe("フランクリン"); + }); + it("answers a count with the counter, not with a name", async () => { expect((await tap("鳥が二羽いた。", 3))?.lookupKind).not.toBe("name"); expect((await tap("全三巻を買った。", 2))?.lookupKind).not.toBe("name"); diff --git a/packages/japanese-reader/src/auto-name.ts b/packages/japanese-reader/src/auto-name.ts index f748b64..94de666 100644 --- a/packages/japanese-reader/src/auto-name.ts +++ b/packages/japanese-reader/src/auto-name.ts @@ -42,12 +42,14 @@ export function nameMayOverrideShorterWord( topFreq: number | null | undefined, ): boolean { const chars = [...matchedText]; - if (chars.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; if (chars.length < AUTO_NAME_OVERRIDE_COUNTED_LENGTH) return false; - const floor = chars.some((ch) => KANJI_NUMERALS.has(ch)) - ? AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ - : AUTO_NAME_OVERRIDE_MIN_FREQ; - return (topFreq ?? 0) >= floor; + // Asked before the length shortcut, or 千万人 and 東洋一 ride in on it with + // no sightings at all. + if (chars.some((ch) => KANJI_NUMERALS.has(ch))) { + return (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ; + } + if (chars.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; + return (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_MIN_FREQ; } export interface AutoNameWordCandidate { diff --git a/packages/japanese-reader/src/deinflect.negative-nu.test.ts b/packages/japanese-reader/src/deinflect.negative-nu.test.ts index 3044db6..9d40f3b 100644 --- a/packages/japanese-reader/src/deinflect.negative-nu.test.ts +++ b/packages/japanese-reader/src/deinflect.negative-nu.test.ts @@ -44,6 +44,10 @@ describe("negative ～ぬ", () => { expect(bases("飲んで")).not.toContain("飲る"); expect(bases("読んで")).not.toContain("読る"); expect(bases("言われたんだ")).not.toContain("言われつ"); + // ANY includes RAW, so a rule that emits it used to hand the next one a + // surface that was never written: よぎなくさせる reached 余儀る three rules + // deep, through the suru-noun strip. + expect(bases("よぎなくさせる")).not.toContain("よぎる"); // 死る is the ichidan rule reading 死 as a stem. No dictionary holds it, so // it costs a lookup and nothing else; 死ぬ keeps answering as itself. expect(bases("死ぬ")).toEqual(["死ぬ", "死る"]); diff --git a/packages/japanese-reader/src/furigana.ts b/packages/japanese-reader/src/furigana.ts index 42b6127..d18b221 100644 --- a/packages/japanese-reader/src/furigana.ts +++ b/packages/japanese-reader/src/furigana.ts @@ -584,8 +584,27 @@ export function nameReadingDominance( return { share: (best.freq ?? 0) / total, total }; } -/** The single-kana particles, the only kana a name's tail is mistaken for. */ -const PARTICLE_KANA = new Set(["の", "が", "を", "に", "は", "へ", "と", "も", "や", "か"]); +/** + * The single-kana particles, the only kana a name's tail is mistaken for. + * ね, よ and さ earn their place: they take 17 more junk name readings off + * kanji (多ね, 晴さ, 嘉よ) and lose none. で is a no-op — no name spelling + * ends in kanji + で. + */ +const PARTICLE_KANA = new Set([ + "の", + "が", + "を", + "に", + "は", + "へ", + "と", + "も", + "や", + "か", + "ね", + "よ", + "さ", +]); function shouldConsiderNameFuriganaSurface(surface: string): boolean { // 四十三 and 五十八 are given names in JMnedict and numbers everywhere else. diff --git a/packages/japanese-reader/src/lookup.span-starts.test.ts b/packages/japanese-reader/src/lookup.span-starts.test.ts index d67c38e..0da0141 100644 --- a/packages/japanese-reader/src/lookup.span-starts.test.ts +++ b/packages/japanese-reader/src/lookup.span-starts.test.ts @@ -87,6 +87,14 @@ describe.skipIf(!hasBothDbs)("what a guess is allowed to land on", () => { expect((await word("出たりはいったりして、よう", 5))?.entries[0]?.kanji[0]?.text).toBe("入る"); }); + // The gate itself, not the entry filter: でなく and 弛まぬ are entries, and + // the walk must stop asking once a reading that needed no guess matched. + it("stops at a reading that needed no guess", async () => { + expect((await word("問題でなく単なる誤解だ", 2))?.entries[0]?.kanji[0]?.text).toBe("で無い"); + expect((await word("弛まぬ努力を続けた", 0))?.entries[0]?.kanji[0]?.text).toBe("弛まぬ"); + expect((await word("馬がいななく声がした", 2))?.entries[0]?.kanji[0]?.text).toBe("嘶く"); + }); + it("still lets a guess speak when it lands on a verb", async () => { expect((await word("心配しなくてもいいと云い", 0))?.matchedText).toBe("心配しなくて"); expect((await word("散歩に誘わなくなった", 3))?.matchedText).toBe("誘わなく"); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index 0fcd689..cf3528f 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -4,6 +4,7 @@ import { AUTO_DUAL_MIN_MATCH_LENGTH, AUTO_NAME_DUAL_CONFIDENCE, AUTO_NAME_ONLY_CONFIDENCE, + AUTO_NAME_OVERRIDE_MIN_LENGTH, AUTO_NAME_ONLY_WITH_EXACT_WORD_CONFIDENCE, computeAutoNameConfidence, nameMayOverrideShorterWord, @@ -748,6 +749,7 @@ function scoreTapCandidate( function chooseAutoLookupVariants( wordResult: LookupResult, nameResults: LookupResult[], + nameCutsAWord = false, ): LookupResult[] { const taggedWord = asWordLookupResult(wordResult); if (nameResults.length === 0) return [taggedWord]; @@ -780,10 +782,16 @@ function chooseAutoLookupVariants( : AUTO_NAME_ONLY_CONFIDENCE; const nameIsLonger = bestName.matchedText.length > taggedWord.matchedText.length; + // The cut test is for the counted two-kanji override only. A longer name + // overlaps an ordinary word all the time — フランクリン contains フランク, + // ゴルキ contains ゴ — and asking it there costs real names. + const cutsAWord = + nameCutsAWord && [...bestName.matchedText].length < AUTO_NAME_OVERRIDE_MIN_LENGTH; if ( nameIsLonger && (nameConfidence >= nameOnlyConfidence || - (nameSpellingIsSurface(bestName) && + (!cutsAWord && + nameSpellingIsSurface(bestName) && nameMayOverrideShorterWord(bestName.matchedText, bestName.nameMatches?.[0]?.freq))) ) { return taggedNames; @@ -846,9 +854,45 @@ function attachAutoSelectionAlternates( }); } +/** + * Whether a span cuts a word at either end — 東京|市 for 京市, 外来|音 for 来音. + * + * The counts cannot tell a straddle from a surname: 京市 and 定家 are both one + * sighting and only one of them is a name. The text can. This is the question + * `findOkuriganaStarts` asks of kana, asked of kanji, and only common entries + * answer it, because a rare one vouching for a boundary is how 滝山 would lose + * to 山城. + */ +async function spanCutsAWord( + text: string, + start: number, + length: number, + lookupMany: (words: string[]) => Promise>, +): Promise { + const end = start + length; + const words: string[] = []; + for (let reach = 1; reach <= 3; reach++) { + for (let into = 1; into <= 3; into++) { + if (start - reach >= 0 && start + into <= text.length) { + words.push(text.slice(start - reach, start + into)); + } + if (end + reach <= text.length && end - into >= 0) { + words.push(text.slice(end - into, end + reach)); + } + } + } + if (words.length === 0) return false; + const found = await lookupMany([...new Set(words)]); + for (const entries of found.values()) { + if (entries.some((entry) => entry.common)) return true; + } + return false; +} + export function chooseAutoLookupResults( wordResults: LookupResult[], nameResults: LookupResult[], + nameCutsAWord = false, ): LookupResult[] { const taggedWordResults = wordResults.map(asWordLookupResult); const taggedNameResults = nameResults.map(asNameLookupResult); @@ -856,7 +900,7 @@ export function chooseAutoLookupResults( if (taggedWordResults.length === 0) return taggedNameResults; if (taggedNameResults.length === 0) return taggedWordResults; - const variants = chooseAutoLookupVariants(taggedWordResults[0], taggedNameResults); + const variants = chooseAutoLookupVariants(taggedWordResults[0], taggedNameResults, nameCutsAWord); if (variants.length <= 1) { return variants[0]?.lookupKind === "name" ? taggedNameResults : taggedWordResults; } @@ -1052,10 +1096,13 @@ async function findFirstWord( const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); + // Raised by the literal itself, not by what is left after the entries + // already shown are removed: a literal whose entries were all shown is + // still a reading that needed no guess. + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { - if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1127,10 +1174,10 @@ async function findBoundaryWord( const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { - if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1191,11 +1238,11 @@ export async function smartLookup( const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; const allEntries = await lookupExactJapanese(dictDb, candidate.word); + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { - if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( newEntries, substr, @@ -1343,11 +1390,12 @@ export async function smartLookupWithOffset( // 何にもせぬ swallowed せぬ to answer the adverb 何にも. // // Only on the literary negatives, which is where the claim comes from. - // The same filter over every suru-noun strip moves 869 corpus taps and - // takes 直にしていれば from 直に to the broth 煮汁; over the contractions - // and particle swaps, which have their own filters above, it takes - // 虫の好かない from the phrase to 良く and すると to the particle と. - if (candidate.guessed) { + // Asking it of every deinflected candidate — the obvious generalisation + // — takes 正直にしていれば from 直に to nothing and 出たりはいったりして + // from 入る to 配流, because an adverb reached by stripping する is a + // perfectly good answer. Over the contractions and particle swaps, + // which have their own guards above, it takes 虫の好かない to 良く. + if (candidate.guessed && !candidate.particleSwapped) { entries = entries.filter((entry) => entryInflects(entry) || entryTakesSuru(entry)); } if (entries.length > 0) { @@ -1472,7 +1520,15 @@ export async function autoLookupWithOffset( smartLookupWithOffset(text, tapOffset, dictDb, extDb, okuriganaStarts), extDb ? nameLookupWithOffset(text, tapOffset, extDb, okuriganaStarts) : Promise.resolve([]), ]); - return chooseAutoLookupResults(wordResults, nameResults); + // 東京|市 is not きょういち. Only asked where a name could override, which is + // a minority of taps, and it is one batched lookup. + const topName = nameResults[0]; + const cutsAWord = + topName?.matchStart != null && + (await spanCutsAWord(text, topName.matchStart, topName.matchedText.length, (words) => + lookupExactJapaneseMany(dictDb, words), + )); + return chooseAutoLookupResults(wordResults, nameResults, cutsAWord); } export async function autoSelectionLookup( From c5c4505143e43b1d47e6760bea862ffc1677ca9c Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 17:37:46 +0800 Subject: [PATCH 13/14] A one-character drag is not a boundary anybody drew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader emits a selection for a single character, so "answer the selection first" was answering 意 ahead of 注意, 本 ahead of 日本 and 気 ahead of 人気のある — and in auto mode the primary result lost its name alternate as well, since alternates are attached per matched text. Two characters is the length at which a drag says something the tap walk could not. 展開 in チェーン展開, the case the change was for, is unmoved. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 7 +++++++ packages/japanese-reader/src/lookup.selection.test.ts | 7 +++++++ packages/japanese-reader/src/lookup.ts | 6 +++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index ddc6989..b1c9248 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -1199,6 +1199,13 @@ they pass before and after. `lookup.selection.test.ts` runs the shipping function — including `autoSelectionLookup`, which is what the reader's default mode actually calls and which nothing covered until review said so. +**A one-character drag does not count.** The reader emits a selection for a +single character (`touch.ts` fires on any non-empty range), and "the selection +first" there answers 意 ahead of 注意, 本 ahead of 日本, 気 ahead of 人気のある — +and in auto mode the primary result loses its name alternate too, because those +are attached per matched text. Two characters is the length at which a drag +says something the tap walk could not. + **The expansion step had no literal-first gate either**, which is where the negative rules above would have spoken over an entry on a drag. It has one now, as do `findFirstWord` and `findBoundaryWord`. diff --git a/packages/japanese-reader/src/lookup.selection.test.ts b/packages/japanese-reader/src/lookup.selection.test.ts index 6bde45c..cec3f50 100644 --- a/packages/japanese-reader/src/lookup.selection.test.ts +++ b/packages/japanese-reader/src/lookup.selection.test.ts @@ -58,6 +58,13 @@ describe.skipIf(!hasBothDbs)("a selection that is itself a word", () => { expect(await select("べた", "食", "。")).toEqual(["べた", "食べた"]); }); + // A one-character drag is not a boundary anybody drew: the reader emits a + // selection for a single character, and the expansion is what it is for. + it("does not apply to a one-character drag", async () => { + expect(await select("意", "注", "にも程がある")).toEqual(["注意"]); + expect(await select("本", "日", "の文化")).toEqual(["日本"]); + }); + it("does not repeat itself when nothing longer is there", async () => { expect(await select("散歩", "の", "に誘わ")).toEqual(["散歩"]); }); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index cf3528f..26e708e 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -940,7 +940,11 @@ export async function selectionLookup( // A drag says where the word ends. チェーン展開 is the longer word, but a // selection of 展開 is a question about 展開, so what the selection itself // spells is answered first and the expansion below lands under it. - const exactSelection = await lookupExactJapanese(dictDb, trimmed); + // + // Two characters, because one is not a boundary anybody drew on purpose — + // the reader emits a selection for a single character, and 注|意 would then + // answer 意 ahead of 注意, which is what the expansion is for. + const exactSelection = [...trimmed].length >= 2 ? await lookupExactJapanese(dictDb, trimmed) : []; if (exactSelection.length > 0) { onResult( asWordLookupResult({ From 33caa5e2a94cf1ce9f127e75ba6fc2372dc97214 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 9 Oct 2026 17:51:13 +0800 Subject: [PATCH 14/14] The guess gate is a question about the tap, not about one span MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declared per span, it only ever compared a guess with a literal at the same characters — and a common guess one character in then outscores a rare longer literal, since commonness pays 120 and the extra character 100. 要求 にこたえられなく answered 耐える off たえられなく, having already thrown away こたえる, which is 応える and what the sentence wants. Hoisted to tap scope, where the walk is longest-first so the longer literal is always seen first: one corpus tap changes, かもしれぬ to もし, which is what master answers. A flat score penalty instead of the skip was measured and is the wrong shape — it reverts six ～ぬ wins and breaks 手はない, gate 131. The scope was the bug, not the strength. The entry filter also moves into every walk rather than the tap alone: a drag still answered 釣や猟をしなくっちゃ with the particle を. 232 word-mode taps of 37,411 against origin/master, auto unmoved at 2,050, gate 125, 623 tests. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reader-lookup-decisions.md | 29 ++++++++++++- packages/japanese-reader/src/lookup.ts | 58 +++++++++++++++++++------- 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index b1c9248..d0268ae 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -943,7 +943,7 @@ slice, not the corpus, and that the whole file is four times that. Measured whole — **37,411 taps**, every character of `test/corpus/bocchan.txt`, this branch against `origin/master`: -- **Word mode: 233 change**, 126 distinct transitions, all read. +- **Word mode: 232 change**, all read. - **Auto mode: 2,050 change** — 1,766 a kana run that was answered with a JMnedict name and is now answered with a word, 220 the word-mode changes, 60 a real name the tap used to answer with a single kanji, 2 lateral. @@ -960,6 +960,23 @@ swallowed せぬ to answer the adverb 何にも. A guess claims the surface is a inflected word, so the entry it lands on now has to be one (`entryInflects` or `entryTakesSuru`). +**The flag is a question about the tap, not about one span.** It was declared +inside the per-length loop, so a guess was only ever compared with a literal at +the _same_ characters — and a common guess one character in then outscores a +rare longer literal, because commonness pays 120 and the extra character 100. +要求に**こたえられなく** answered 耐える off たえられなく, having thrown away +こたえる (応える, the reading the sentence wants) at its own span. Hoisted to tap +scope, where the walk is longest-first so the longer literal is always seen +first: **one corpus tap changes** (かもしれぬ → もし, which is what master +answers), and the gate reads 125. + +A flat −150 score penalty instead of the skip was measured and is the wrong +shape: it reverts six of the ～ぬ wins and breaks 手はない, 訳はない and 気はせく, +gate 131. The scope was the bug, not the strength. + +**The filter runs in every walk**, not only the tap: a drag answered +釣や猟を**しなく**っちゃ with the particle を long after the tap stopped doing it. + **That filter is on the literary negatives only, and the scope was measured twice.** Over every suru-noun strip it moves 869 taps and takes 正直**にしていれば** from 直に to the broth 煮汁, 出たり**はいったり**して from @@ -1164,6 +1181,16 @@ tail that reaches real words — いなが→云い, きなが→泣き, きで みもな→望み, しもなか→少しも, うがい→使う. Two are lateral (いなひと→ひとは, junk either way). Gate 126 disagreeing pairs → **123**. +**Open, and pre-existing: a deinflected candidate that is not a guess is still +accepted on the dictionary alone.** Nothing requires the entry to be a word that +could have carried the inflection, so a past or a negative lands on a noun: +なった → ナウ (adj-na), 安心したらしかった → 死活, 選ばなくてはならない → 離る +(a classical bigrade verb that cannot take modern ない), といった → と言う (exp). +The obvious filter is in the file already (`entryInflects`) and the obvious +scope — every candidate with a reason — was measured here and rejected, because +an adverb reached by stripping する is a perfectly good answer. A narrower scope +that exempts the suru-noun step has not been measured. + **What this does not fix:** めだった still answers 目立つ. ため is a noun, so its め is not okurigana and nothing proves the position closed; separating that from a real word boundary needs segmentation, not a rule. diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index 26e708e..0fce771 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -43,6 +43,23 @@ interface EntrySortContext { type KanjiReadingRecord = Awaited>; +/** + * What a guess is allowed to land on. Undoing one of the literary negatives + * claims the surface is an inflected word, so the entry has to be one: the + * ～なく rules otherwise reach する, the suru-noun strip hands back whatever + * stood in front of it, and 釣や猟をしなくっちゃ answers the particle を. + * + * Not for a particle swap, which is not an inflection and has its own guard — + * 虫が好かない is `exp` and nothing else. + */ +function entriesAGuessMayLandOn( + candidate: TapCandidate, + entries: ReaderDictEntry[], +): ReaderDictEntry[] { + if (!candidate.guessed || candidate.particleSwapped) return entries; + return entries.filter((entry) => entryInflects(entry) || entryTakesSuru(entry)); +} + /** * Undoing a contraction, swapping a particle and the literary negatives are * all guesses about what the page meant, and a reading that needs no guess @@ -977,7 +994,10 @@ export async function selectionLookup( for (const candidate of candidates) { const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; - const entries = await lookupExactJapanese(dictDb, candidate.word); + const entries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); if (entries.length > 0) { if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( @@ -1099,7 +1119,10 @@ async function findFirstWord( for (const candidate of candidates) { const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); // Raised by the literal itself, not by what is left after the entries // already shown are removed: a literal whose entries were all shown is // still a reading that needed no guess. @@ -1177,7 +1200,10 @@ async function findBoundaryWord( for (const candidate of candidates) { const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); @@ -1241,7 +1267,10 @@ export async function smartLookup( for (const candidate of candidates) { const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); @@ -1323,6 +1352,12 @@ export async function smartLookupWithOffset( (await findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce)); let bestOverall: { result: LookupResult; score: number; length: number; start: number } | null = null; + // Tap scope, not span scope. Per span, a guess is only ever compared with a + // literal at the SAME characters, and a common guess one character in then + // outscores a rare longer literal — commonness pays 120, the extra character + // 100 — so 要求にこたえられなく answered 耐える off たえられなく. The walk is + // longest-first, so the longer literal is always seen first. + let literalMatched = false; for (let len = Math.min(text.length, 15); len >= 1; len--) { // Valid start positions: substring must contain the tap position @@ -1364,7 +1399,6 @@ export async function smartLookupWithOffset( // span as written found nothing — にしては is an entry in its own right // and must not be answered with にしても, which merely happens to be the // commoner of the two. - let literalMatched = false; for (const candidate of candidates) { const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; @@ -1393,15 +1427,11 @@ export async function smartLookupWithOffset( // しなくっちゃ answered the particle を, ものでなくっちゃ answered ので, // 何にもせぬ swallowed せぬ to answer the adverb 何にも. // - // Only on the literary negatives, which is where the claim comes from. - // Asking it of every deinflected candidate — the obvious generalisation - // — takes 正直にしていれば from 直に to nothing and 出たりはいったりして - // from 入る to 配流, because an adverb reached by stripping する is a - // perfectly good answer. Over the contractions and particle swaps, - // which have their own guards above, it takes 虫の好かない to 良く. - if (candidate.guessed && !candidate.particleSwapped) { - entries = entries.filter((entry) => entryInflects(entry) || entryTakesSuru(entry)); - } + // Asking this of every deinflected candidate — the obvious + // generalisation — takes 正直にしていれば from 直に to nothing and + // 出たりはいったりして from 入る to 配流, because an adverb reached by + // stripping する is a perfectly good answer. + entries = entriesAGuessMayLandOn(candidate, entries); if (entries.length > 0) { let sortedEntries = sortEntriesForMatchedSurface( entries,