mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-04-12 04:19:25 -07:00
feat: stabilize startup sync and overlay/runtime paths
This commit is contained in:
@@ -150,6 +150,76 @@ test('annotateTokens handles JLPT disabled and eligibility exclusion paths', ()
|
||||
assert.equal(excludedLookupCalls, 0);
|
||||
});
|
||||
|
||||
test('annotateTokens prioritizes name matches over n+1, frequency, and JLPT when enabled', () => {
|
||||
let jlptLookupCalls = 0;
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'オリヴィア',
|
||||
reading: 'オリヴィア',
|
||||
headword: 'オリヴィア',
|
||||
isNameMatch: true,
|
||||
frequencyRank: 42,
|
||||
startPos: 0,
|
||||
endPos: 5,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
getJlptLevel: () => {
|
||||
jlptLookupCalls += 1;
|
||||
return 'N2';
|
||||
},
|
||||
}),
|
||||
{
|
||||
nameMatchEnabled: true,
|
||||
minSentenceWordsForNPlusOne: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.jlptLevel, undefined);
|
||||
assert.equal(jlptLookupCalls, 0);
|
||||
});
|
||||
|
||||
test('annotateTokens keeps other annotations for name matches when name highlighting is disabled', () => {
|
||||
let jlptLookupCalls = 0;
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'オリヴィア',
|
||||
reading: 'オリヴィア',
|
||||
headword: 'オリヴィア',
|
||||
isNameMatch: true,
|
||||
frequencyRank: 42,
|
||||
startPos: 0,
|
||||
endPos: 5,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
getJlptLevel: () => {
|
||||
jlptLookupCalls += 1;
|
||||
return 'N2';
|
||||
},
|
||||
}),
|
||||
{
|
||||
nameMatchEnabled: false,
|
||||
minSentenceWordsForNPlusOne: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, true);
|
||||
assert.equal(result[0]?.frequencyRank, 42);
|
||||
assert.equal(result[0]?.jlptLevel, 'N2');
|
||||
assert.equal(jlptLookupCalls, 1);
|
||||
});
|
||||
|
||||
test('annotateTokens N+1 handoff marks expected target when threshold is satisfied', () => {
|
||||
const tokens = [
|
||||
makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),
|
||||
|
||||
@@ -39,6 +39,7 @@ export interface AnnotationStageDeps {
|
||||
|
||||
export interface AnnotationStageOptions {
|
||||
nPlusOneEnabled?: boolean;
|
||||
nameMatchEnabled?: boolean;
|
||||
jlptEnabled?: boolean;
|
||||
frequencyEnabled?: boolean;
|
||||
minSentenceWordsForNPlusOne?: number;
|
||||
@@ -611,6 +612,13 @@ function computeTokenJlptLevel(
|
||||
return level ?? undefined;
|
||||
}
|
||||
|
||||
function hasPrioritizedNameMatch(
|
||||
token: MergedToken,
|
||||
options: Pick<AnnotationStageOptions, 'nameMatchEnabled'>,
|
||||
): boolean {
|
||||
return options.nameMatchEnabled !== false && token.isNameMatch === true;
|
||||
}
|
||||
|
||||
export function annotateTokens(
|
||||
tokens: MergedToken[],
|
||||
deps: AnnotationStageDeps,
|
||||
@@ -619,25 +627,31 @@ export function annotateTokens(
|
||||
const pos1Exclusions = resolvePos1Exclusions(options);
|
||||
const pos2Exclusions = resolvePos2Exclusions(options);
|
||||
const nPlusOneEnabled = options.nPlusOneEnabled !== false;
|
||||
const nameMatchEnabled = options.nameMatchEnabled !== false;
|
||||
const frequencyEnabled = options.frequencyEnabled !== false;
|
||||
const jlptEnabled = options.jlptEnabled !== false;
|
||||
|
||||
// Single pass: compute known word status, frequency filtering, and JLPT level together
|
||||
const annotated = tokens.map((token) => {
|
||||
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
|
||||
const isKnown = nPlusOneEnabled
|
||||
? computeTokenKnownStatus(token, deps.isKnownWord, deps.knownWordMatchMode)
|
||||
: false;
|
||||
|
||||
const frequencyRank = frequencyEnabled
|
||||
const frequencyRank = frequencyEnabled && !prioritizedNameMatch
|
||||
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
|
||||
: undefined;
|
||||
|
||||
const jlptLevel = jlptEnabled ? computeTokenJlptLevel(token, deps.getJlptLevel) : undefined;
|
||||
const jlptLevel =
|
||||
jlptEnabled && !prioritizedNameMatch
|
||||
? computeTokenJlptLevel(token, deps.getJlptLevel)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
...token,
|
||||
isKnown,
|
||||
isNPlusOneTarget: nPlusOneEnabled ? token.isNPlusOneTarget : false,
|
||||
isNPlusOneTarget:
|
||||
nPlusOneEnabled && !prioritizedNameMatch ? token.isNPlusOneTarget : false,
|
||||
frequencyRank,
|
||||
jlptLevel,
|
||||
};
|
||||
@@ -655,10 +669,25 @@ export function annotateTokens(
|
||||
? minSentenceWordsForNPlusOne
|
||||
: 3;
|
||||
|
||||
return markNPlusOneTargets(
|
||||
const nPlusOneMarked = markNPlusOneTargets(
|
||||
annotated,
|
||||
sanitizedMinSentenceWordsForNPlusOne,
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
);
|
||||
|
||||
if (!nameMatchEnabled) {
|
||||
return nPlusOneMarked;
|
||||
}
|
||||
|
||||
return nPlusOneMarked.map((token) =>
|
||||
hasPrioritizedNameMatch(token, options)
|
||||
? {
|
||||
...token,
|
||||
isNPlusOneTarget: false,
|
||||
frequencyRank: undefined,
|
||||
jlptLevel: undefined,
|
||||
}
|
||||
: token,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user