sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-08-01 07:21:33 -07:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tokenizer): consolidate kana/POS helpers into token-classification (#161)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-12 23:27:25 -07:00
committed by GitHub
parent 14070acceb
commit 21a06c12fb
11 changed files with 302 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/tokenizer.ts
+1 -14
View File
@@ -25,6 +25,7 @@ import {
requestYomitanTermFrequencies,
} from './tokenizer/yomitan-parser-runtime';
import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
import { isKanaChar } from './tokenizer/token-classification';
const logger = createLogger('main:tokenizer');
@@ -322,20 +323,6 @@ function normalizeFrequencyLookupText(rawText: string): string {
return rawText.trim().toLowerCase();
}
function isKanaChar(char: string): boolean {
const code = char.codePointAt(0);
if (code === undefined) {
return false;
}
return (
(code >= 0x3041 && code <= 0x3096) ||
(code >= 0x309b && code <= 0x309f) ||
code === 0x30fc ||
(code >= 0x30a0 && code <= 0x30fa) ||
(code >= 0x30fd && code <= 0x30ff)
);
}
function getTrailingKanaSuffix(surface: string): string {
const chars = Array.from(surface);
let splitIndex = chars.length;