sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-07-07 13:08:54 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(anki): reading-aware known-word matching (cache v3) (#142)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-07 00:13:10 -07:00
committed by GitHub
parent 8b9a70c5a6
commit 38ddb29aa0
16 changed files with 701 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/tokenizer.ts
+4 -3
View File
@@ -42,7 +42,7 @@ export interface TokenizerServiceDeps {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean;
isKnownWord: (text: string, reading?: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +77,7 @@ export interface TokenizerDepsRuntimeOptions {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean;
isKnownWord: (text: string, reading?: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +129,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
function getKnownWordLookup(
deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions,
): (text: string) => boolean {
): (text: string, reading?: string) => boolean {
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
return () => false;
}
@@ -723,6 +723,7 @@ async function parseWithYomitanInternalParser(
surface: token.surface,
reading: token.reading,
headword: token.headword,
headwordReading: token.headwordReading,
startPos: token.startPos,
endPos: token.endPos,
partOfSpeech: posMetadata.partOfSpeech,