sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-07-28 16:49:50 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(overlay): resolve unspaced Japanese name splits and scan recovery (#146)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-07 02:33:46 -07:00
committed by GitHub
parent e7739de51c
commit 61f39d1e09
19 changed files with 884 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/tokenizer.ts
+2 -1
View File
@@ -150,7 +150,8 @@ function hasAnyAnnotationEnabled(options: TokenizerAnnotationOptions): boolean {
options.knownWordsEnabled ||
options.nPlusOneEnabled ||
options.jlptEnabled ||
options.frequencyEnabled
options.frequencyEnabled ||
options.nameMatchEnabled
);
}
src/core/services/tokenizer/annotation-stage.test.ts
+46 -1
View File
@@ -33,6 +33,50 @@ function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStage
};
}
test('annotateTokens keeps name matches on tokens the POS noise filter would strip', () => {
// MeCab tags 平 as 接頭詞 in contexts like あっ 平 これ…, which is in the
// POS1 exclusion list; a confirmed character-name match must survive it.
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
makeToken({
surface: '平',
headword: '平',
reading: 'ひら',
pos1: '接頭詞',
isNameMatch: false,
jlptLevel: 'N1',
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: true });
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[1]?.isNameMatch, false);
assert.equal(result[1]?.jlptLevel, undefined);
});
test('annotateTokens strips name matches from POS-excluded tokens when name matching is disabled', () => {
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: false });
assert.equal(result[0]?.isNameMatch, false);
});
test('annotateTokens known-word match mode uses headword vs surface', () => {
const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
const isKnownWord = (text: string): boolean => text === '食べる';
@@ -1994,7 +2038,8 @@ test('annotateTokens keeps known status while clearing other annotations for aru
assert.equal(result[0]?.headword, '有る');
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.isNameMatch, false);
// Name matches take precedence over the annotation noise filter.
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
});
src/core/services/tokenizer/annotation-stage.ts
+6 -2
View File
@@ -778,7 +778,13 @@ export function annotateTokens(
: false;
nPlusOneKnownStatuses[index] = isKnownForMatching;
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
// A confirmed character-name match must survive the POS noise filter:
// MeCab can tag a name like 平 as a prefix (接頭詞) depending on context,
// which would otherwise strip the name match and its portrait.
if (
!prioritizedNameMatch &&
sharedShouldExcludeTokenFromSubtitleAnnotations(token, {
pos1Exclusions,
pos2Exclusions,
@@ -794,8 +800,6 @@ export function annotateTokens(
};
}
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
const frequencyRank =
frequencyEnabled && !prioritizedNameMatch
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
src/core/services/tokenizer/yomitan-parser-runtime.test.ts
+99
View File
@@ -975,6 +975,105 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
]);
});
test('requestYomitanScanTokens retries shorter windows when a greedy match has no exact-source headword', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profileIndex: 0,
scanLength: 40,
dictionaries: ['JMdict'],
dictionaryPriorityByName: { JMdict: 0 },
dictionaryFrequencyModeByName: {},
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [{ name: 'JMdict', enabled: true, id: 0 }],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens('平 （平）', deps, {
error: () => undefined,
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (!text.startsWith('平')) {
return { originalTextLength: 0, dictionaryEntries: [] };
}
if (text.length >= 4) {
// Simulates Yomitan normalization consuming punctuation/whitespace:
// the greedy match spans 平 （平 but no headword source equals it.
return {
originalTextLength: 4,
dictionaryEntries: [
{
headwords: [
{
term: '平々',
reading: 'へいへい',
sources: [{ originalText: '平平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
}
return {
originalTextLength: 1,
dictionaryEntries: [
{
headwords: [
{
term: '平',
reading: 'たいら',
sources: [{ originalText: '平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
});
assert.deepEqual(result, [
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 0,
endPos: 1,
isNameMatch: false,
frequencyRank: undefined,
},
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 3,
endPos: 4,
isNameMatch: false,
frequencyRank: undefined,
},
]);
});
test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
src/core/services/tokenizer/yomitan-parser-runtime.ts
+58 -36
View File
@@ -1297,47 +1297,69 @@ ${YOMITAN_SCANNING_HELPERS}
const text = ${JSON.stringify(text)};
const details = {matchType: "exact", deinflect: true};
const tokens = [];
let i = 0;
while (i < text.length) {
const codePoint = text.codePointAt(i);
async function findTokenAt(position, windowLength) {
const codePoint = text.codePointAt(position);
const character = String.fromCodePoint(codePoint);
const substring = text.substring(i, i + ${scanLength});
const substring = text.substring(position, position + windowLength);
const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
if (dictionaryEntries.length > 0 && originalTextLength > 0 && (originalTextLength !== character.length || isCodePointJapanese(codePoint))) {
const source = substring.substring(0, originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (preferredHeadword && typeof preferredHeadword.term === "string") {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: i,
endPos: i + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
tokens.push(tokenPayload);
i += originalTextLength;
continue;
}
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
return { token: null, matchedLength: 0 };
}
i += character.length;
const source = substring.substring(0, originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
return { token: null, matchedLength: originalTextLength };
}
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: position,
endPos: position + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
return { token: tokenPayload, matchedLength: originalTextLength };
}
let i = 0;
while (i < text.length) {
let attempt = await findTokenAt(i, ${scanLength});
// Yomitan text normalization can consume characters (whitespace,
// punctuation) beyond the matched term, leaving no headword whose
// source equals the consumed text. Retry with shorter windows so a
// valid prefix term (e.g. a character name before a paren) still
// tokenizes instead of the position being skipped.
let retryLength = Math.min(attempt.matchedLength, ${scanLength}) - 1;
while (!attempt.token && retryLength >= 1) {
const retry = await findTokenAt(i, retryLength);
if (retry.token) {
attempt = retry;
break;
}
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
}
if (attempt.token) {
tokens.push(attempt.token);
i += attempt.matchedLength;
continue;
}
i += String.fromCodePoint(text.codePointAt(i)).length;
}
return tokens;
})();