perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185)

This commit is contained in:
2026-08-06 21:44:09 -07:00
committed by GitHub
parent 441ecf3c04
commit dbdf578c68
38 changed files with 4073 additions and 1720 deletions
+3 -35
View File
@@ -2934,44 +2934,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar
return [];
}
if (script.includes('parseText')) {
return [
{
source: 'scanning-parser',
index: 0,
content: [
[
{
text: '取り組んで',
reading: 'とりくんで',
headwords: [[{ term: '取り組む' }]],
},
],
[
{
text: 'もらいます',
reading: 'もらいます',
headwords: [[{ term: 'もらう' }]],
},
],
],
},
];
}
return [
{
surface: '取り',
reading: 'とり',
headword: '取',
surface: '取り組んで',
reading: 'とりくんで',
headword: '取り組む',
startPos: 0,
endPos: 2,
},
{
surface: '組んで',
reading: 'くんで',
headword: '組む',
startPos: 2,
endPos: 5,
},
{