perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185)

src/core/text/han-code-points.test.ts

@@ -0,0 +1,54 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { HAN_CODE_POINT_RANGES, HAN_REGEXP_CLASS_BODY, isHanCodePoint } from './han-code-points';
+
+test('every range boundary is inside the table', () => {
+  for (const [start, end] of HAN_CODE_POINT_RANGES) {
+    for (const codePoint of [start, end]) {
+      assert.ok(isHanCodePoint(codePoint), `expected U+${codePoint.toString(16)} to be Han`);
+    }
+  }
+
+  // Extension J (Unicode 17) and the Compatibility blocks are the ones a
+  // BMP-only table used to miss.
+  assert.ok(isHanCodePoint(0x323b0));
+  assert.ok(isHanCodePoint(0x33479));
+  assert.ok(isHanCodePoint(0xf900));
+  assert.ok(isHanCodePoint(0x2f800));
+});
+
+test('no unified ideograph the runtime knows about falls outside the table', () => {
+  // One direction only: a runtime with older Unicode data simply checks fewer
+  // code points, where asserting the reverse would fail on Extension J.
+  const unifiedIdeograph = /\p{Unified_Ideograph}/u;
+
+  for (let codePoint = 0x3000; codePoint <= 0x40000; codePoint += 1) {
+    if (unifiedIdeograph.test(String.fromCodePoint(codePoint))) {
+      assert.ok(
+        isHanCodePoint(codePoint),
+        `expected unified ideograph U+${codePoint.toString(16)} to be in the table`,
+      );
+    }
+  }
+});
+
+test('code points just outside the table are rejected', () => {
+  for (const codePoint of [0x33ff, 0x4dc0, 0xa000, 0x1f000, 0x3347a]) {
+    assert.equal(
+      isHanCodePoint(codePoint),
+      false,
+      `expected U+${codePoint.toString(16)} not to be Han`,
+    );
+  }
+});
+
+test('the regexp class body matches the same code points as the predicate', () => {
+  const classRegExp = new RegExp(`^[${HAN_REGEXP_CLASS_BODY}]$`, 'u');
+
+  for (const codePoint of [0x3400, 0x4e00, 0x9fff, 0xf900, 0x20000, 0x323b0, 0x33479]) {
+    assert.match(String.fromCodePoint(codePoint), classRegExp);
+  }
+  for (const codePoint of [0x3040, 0x30ff, 0x33fa, 0x3347a]) {
+    assert.doesNotMatch(String.fromCodePoint(codePoint), classRegExp);
+  }
+});

src/core/text/han-code-points.ts

@@ -0,0 +1,31 @@
+// Single source of truth for "this code point is a Han character", shared by
+// the main-process character dictionary and the in-page Yomitan scan runtime.
+// The two used to carry separate range lists, and they drifted: a name written
+// with a supplementary-plane kanji could enter the generated dictionary while
+// the scanner's greedy name pre-pass refused to probe the position.
+//
+// Ranges rather than \p{Script=Han}: the scan walk tests one code point per
+// character of every subtitle line, where an integer compare beats building a
+// string for a regex, and the script is injected as text into a page where a
+// shared helper cannot be imported.
+export const HAN_CODE_POINT_RANGES: ReadonlyArray<readonly [number, number]> = [
+  [0x3400, 0x4dbf], // Extension A
+  [0x4e00, 0x9fff], // CJK Unified Ideographs
+  [0xf900, 0xfaff], // Compatibility Ideographs
+  [0x20000, 0x2a6df], // Extension B
+  [0x2a700, 0x2ebef], // Extensions C-F
+  [0x2ebf0, 0x2ee5f], // Extension I
+  [0x2f800, 0x2fa1f], // Compatibility Ideographs Supplement
+  [0x30000, 0x3134f], // Extension G
+  [0x31350, 0x323af], // Extension H
+  [0x323b0, 0x33479], // Extension J (Unicode 17)
+];
+
+export function isHanCodePoint(codePoint: number): boolean {
+  return HAN_CODE_POINT_RANGES.some(([start, end]) => codePoint >= start && codePoint <= end);
+}
+
+/** The same ranges as a regular expression character class body (needs the `u` flag). */
+export const HAN_REGEXP_CLASS_BODY = HAN_CODE_POINT_RANGES.map(
+  ([start, end]) => `\\u{${start.toString(16)}}-\\u{${end.toString(16)}}`,
+).join('');