mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 01:01:34 -07:00
feat(animetosho): add English/Japanese subtitle download integration (#159)
This commit is contained in:
GitHub
@@ -0,0 +1,119 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import * as http from 'node:http';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
|
||||
import { animetoshoFetchJson } from './utils.js';
|
||||
|
||||
interface TestServer {
|
||||
baseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
function startServer(handler: http.RequestListener): Promise<TestServer> {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer(handler);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const { port } = server.address() as AddressInfo;
|
||||
resolve({
|
||||
baseUrl: `http://127.0.0.1:${port}`,
|
||||
close: () =>
|
||||
new Promise((done) => {
|
||||
server.closeAllConnections?.();
|
||||
server.close(() => done());
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('animetoshoFetchJson gives up on a hanging response', async () => {
|
||||
const server = await startServer((_req, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
// Never end the response.
|
||||
res.write('[');
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await animetoshoFetchJson(
|
||||
'/json',
|
||||
{},
|
||||
{ baseUrl: server.baseUrl, timeoutMs: 150 },
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
if (!result.ok) {
|
||||
assert.match(result.error.error, /timed out/i);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('animetoshoFetchJson rejects an oversized response', async () => {
|
||||
const server = await startServer((_req, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(Array.from({ length: 2000 }, (_, index) => ({ id: index }))));
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await animetoshoFetchJson(
|
||||
'/json',
|
||||
{},
|
||||
{ baseUrl: server.baseUrl, maxResponseBytes: 512 },
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
if (!result.ok) {
|
||||
assert.match(result.error.error, /too large/i);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('animetoshoFetchJson decodes multi-byte characters split across chunks', async () => {
|
||||
const title = '葬送のフリーレン';
|
||||
const body = Buffer.from(JSON.stringify([{ id: 1, title }]), 'utf8');
|
||||
// Split inside the first Japanese character's 3-byte UTF-8 sequence.
|
||||
const splitAt = body.indexOf(Buffer.from(title, 'utf8')) + 1;
|
||||
|
||||
const server = await startServer((_req, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.write(body.subarray(0, splitAt));
|
||||
setTimeout(() => res.end(body.subarray(splitAt)), 10);
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await animetoshoFetchJson<Array<{ id: number; title: string }>>(
|
||||
'/json',
|
||||
{},
|
||||
{ baseUrl: server.baseUrl },
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
if (result.ok) {
|
||||
assert.equal(result.data[0]!.title, title);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('animetoshoFetchJson still parses a normal response', async () => {
|
||||
const server = await startServer((_req, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify([{ id: 1, title: 'ok' }]));
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await animetoshoFetchJson<Array<{ id: number }>>(
|
||||
'/json',
|
||||
{ q: 'x' },
|
||||
{ baseUrl: server.baseUrl },
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
if (result.ok) {
|
||||
assert.deepEqual(result.data, [{ id: 1, title: 'ok' }]);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
animetoshoLangToFilenameSuffix,
|
||||
animetoshoTrackMatchesLanguages,
|
||||
describeAnimetoshoTabLanguages,
|
||||
normalizeAnimetoshoLangCode,
|
||||
} from './lang.js';
|
||||
|
||||
test('normalizeAnimetoshoLangCode collapses 2/3-letter and region variants', () => {
|
||||
assert.equal(normalizeAnimetoshoLangCode('eng'), 'en');
|
||||
assert.equal(normalizeAnimetoshoLangCode('en'), 'en');
|
||||
assert.equal(normalizeAnimetoshoLangCode('en-US'), 'en');
|
||||
assert.equal(normalizeAnimetoshoLangCode('GER'), 'de');
|
||||
assert.equal(normalizeAnimetoshoLangCode('jpn'), 'ja');
|
||||
assert.equal(normalizeAnimetoshoLangCode('vie'), 'vie');
|
||||
assert.equal(normalizeAnimetoshoLangCode(''), '');
|
||||
});
|
||||
|
||||
test('animetoshoTrackMatchesLanguages matches across code forms', () => {
|
||||
assert.equal(animetoshoTrackMatchesLanguages('eng', ['en']), true);
|
||||
assert.equal(animetoshoTrackMatchesLanguages('eng', ['en', 'eng']), true);
|
||||
assert.equal(animetoshoTrackMatchesLanguages('ger', ['de']), true);
|
||||
assert.equal(animetoshoTrackMatchesLanguages('por', ['en']), false);
|
||||
assert.equal(animetoshoTrackMatchesLanguages('spa', ['en', 'de']), false);
|
||||
});
|
||||
|
||||
test('animetoshoTrackMatchesLanguages keeps unknown-language tracks visible', () => {
|
||||
assert.equal(animetoshoTrackMatchesLanguages('', ['en']), true);
|
||||
assert.equal(animetoshoTrackMatchesLanguages('und', ['en']), true);
|
||||
});
|
||||
|
||||
test('describeAnimetoshoTabLanguages names common languages and dedupes', () => {
|
||||
assert.equal(describeAnimetoshoTabLanguages(['en', 'eng']), 'English');
|
||||
assert.equal(describeAnimetoshoTabLanguages(['de']), 'German');
|
||||
assert.equal(describeAnimetoshoTabLanguages(['en', 'de']), 'English / German');
|
||||
assert.equal(describeAnimetoshoTabLanguages(['vie']), 'VIE');
|
||||
assert.equal(describeAnimetoshoTabLanguages([]), 'English');
|
||||
});
|
||||
|
||||
test('animetoshoLangToFilenameSuffix is re-exported from the pure module', () => {
|
||||
assert.equal(animetoshoLangToFilenameSuffix('jpn'), 'ja');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('eng'), 'en');
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
// Pure language-code helpers shared between the main-process Animetosho client
|
||||
// and the overlay renderer. Must stay free of node imports so the renderer
|
||||
// bundle can use it.
|
||||
|
||||
const LANG_FILENAME_SUFFIXES: Record<string, string> = {
|
||||
eng: 'en',
|
||||
jpn: 'ja',
|
||||
ger: 'de',
|
||||
deu: 'de',
|
||||
spa: 'es',
|
||||
por: 'pt',
|
||||
fre: 'fr',
|
||||
fra: 'fr',
|
||||
ita: 'it',
|
||||
rus: 'ru',
|
||||
ara: 'ar',
|
||||
chi: 'zh',
|
||||
zho: 'zh',
|
||||
kor: 'ko',
|
||||
};
|
||||
|
||||
const LANG_DISPLAY_NAMES: Record<string, string> = {
|
||||
en: 'English',
|
||||
ja: 'Japanese',
|
||||
de: 'German',
|
||||
es: 'Spanish',
|
||||
pt: 'Portuguese',
|
||||
fr: 'French',
|
||||
it: 'Italian',
|
||||
ru: 'Russian',
|
||||
ar: 'Arabic',
|
||||
zh: 'Chinese',
|
||||
ko: 'Korean',
|
||||
};
|
||||
|
||||
export function normalizeAnimetoshoLangCode(code: string): string {
|
||||
const base = code.trim().toLowerCase().split(/[-_]/)[0] ?? '';
|
||||
return LANG_FILENAME_SUFFIXES[base] ?? base;
|
||||
}
|
||||
|
||||
export function animetoshoLangToFilenameSuffix(lang: string | undefined): string {
|
||||
const normalized = normalizeAnimetoshoLangCode(lang ?? '');
|
||||
if (!normalized || normalized === 'und') return 'en';
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function animetoshoTrackMatchesLanguages(
|
||||
trackLang: string,
|
||||
configuredLanguages: string[],
|
||||
): boolean {
|
||||
const normalizedTrack = normalizeAnimetoshoLangCode(trackLang);
|
||||
// Unlabeled tracks cannot be classified; keep them visible rather than
|
||||
// hiding them behind a tab that never shows them.
|
||||
if (!normalizedTrack || normalizedTrack === 'und') return true;
|
||||
return configuredLanguages.some(
|
||||
(candidate) => normalizeAnimetoshoLangCode(candidate) === normalizedTrack,
|
||||
);
|
||||
}
|
||||
|
||||
export function describeAnimetoshoTabLanguages(configuredLanguages: string[]): string {
|
||||
const normalized = [
|
||||
...new Set(
|
||||
configuredLanguages
|
||||
.map((candidate) => normalizeAnimetoshoLangCode(candidate))
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
if (normalized.length === 0) return 'English';
|
||||
return normalized.map((code) => LANG_DISPLAY_NAMES[code] ?? code.toUpperCase()).join(' / ');
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
import {
|
||||
animetoshoLangToFilenameSuffix,
|
||||
buildAnimetoshoAttachmentUrl,
|
||||
decompressXzFile,
|
||||
extractAnimetoshoSubtitleFiles,
|
||||
mapAnimetoshoSearchResults,
|
||||
} from './utils.js';
|
||||
|
||||
test('buildAnimetoshoAttachmentUrl pads the attachment id to 8 hex digits', () => {
|
||||
assert.equal(
|
||||
buildAnimetoshoAttachmentUrl(1955356),
|
||||
'https://animetosho.org/storage/attach/001dd61c/1955356.xz',
|
||||
);
|
||||
});
|
||||
|
||||
test('animetoshoLangToFilenameSuffix maps common ISO 639-2 codes to two-letter suffixes', () => {
|
||||
assert.equal(animetoshoLangToFilenameSuffix('eng'), 'en');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('jpn'), 'ja');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('ger'), 'de');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('spa'), 'es');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('POR'), 'pt');
|
||||
});
|
||||
|
||||
test('animetoshoLangToFilenameSuffix falls back to the raw code, and to en when unknown', () => {
|
||||
assert.equal(animetoshoLangToFilenameSuffix('vie'), 'vie');
|
||||
assert.equal(animetoshoLangToFilenameSuffix(''), 'en');
|
||||
assert.equal(animetoshoLangToFilenameSuffix(undefined), 'en');
|
||||
assert.equal(animetoshoLangToFilenameSuffix('und'), 'en');
|
||||
});
|
||||
|
||||
test('buildAnimetoshoAttachmentUrl rejects non-positive and non-integer ids', () => {
|
||||
assert.equal(buildAnimetoshoAttachmentUrl(0), null);
|
||||
assert.equal(buildAnimetoshoAttachmentUrl(-5), null);
|
||||
assert.equal(buildAnimetoshoAttachmentUrl(1.5), null);
|
||||
assert.equal(buildAnimetoshoAttachmentUrl(Number.NaN), null);
|
||||
});
|
||||
|
||||
test('mapAnimetoshoSearchResults maps valid entries and caps to maxResults', () => {
|
||||
const payload = [
|
||||
{
|
||||
id: 606713,
|
||||
title: '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].mkv',
|
||||
timestamp: 1710000000,
|
||||
total_size: 1490354395,
|
||||
num_files: 1,
|
||||
},
|
||||
{ id: 'bogus', title: 'missing numeric id' },
|
||||
{ id: 606714, title: '[Erai-raws] Sousou no Frieren - 28 [1080p].mkv' },
|
||||
{ id: 606715, title: 'capped away' },
|
||||
];
|
||||
|
||||
const entries = mapAnimetoshoSearchResults(payload, 2);
|
||||
assert.equal(entries.length, 2);
|
||||
assert.deepEqual(entries[0], {
|
||||
id: 606713,
|
||||
title: '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].mkv',
|
||||
timestamp: 1710000000,
|
||||
totalSize: 1490354395,
|
||||
numFiles: 1,
|
||||
});
|
||||
assert.equal(entries[1]!.id, 606714);
|
||||
assert.equal(entries[1]!.totalSize, null);
|
||||
assert.equal(entries[1]!.numFiles, null);
|
||||
});
|
||||
|
||||
test('mapAnimetoshoSearchResults returns empty list for non-array payloads', () => {
|
||||
assert.deepEqual(mapAnimetoshoSearchResults({ error: 'nope' }, 10), []);
|
||||
assert.deepEqual(mapAnimetoshoSearchResults(null, 10), []);
|
||||
});
|
||||
|
||||
const DETAIL_PAYLOAD = {
|
||||
id: 606713,
|
||||
title: '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].mkv',
|
||||
files: [
|
||||
{
|
||||
id: 1151711,
|
||||
filename: '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].mkv',
|
||||
attachments: [
|
||||
{
|
||||
id: 1955355,
|
||||
type: 'font',
|
||||
info: { name: 'arial.ttf' },
|
||||
size: 300000,
|
||||
},
|
||||
{
|
||||
id: 1955356,
|
||||
type: 'subtitle',
|
||||
info: { codec: 'ASS', lang: 'eng', name: 'English subs', trackid: 2 },
|
||||
size: 33075,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
test('extractAnimetoshoSubtitleFiles keeps only text subtitle attachments with download urls', () => {
|
||||
const files = extractAnimetoshoSubtitleFiles(DETAIL_PAYLOAD);
|
||||
assert.equal(files.length, 1);
|
||||
const file = files[0]!;
|
||||
assert.equal(file.attachmentId, 1955356);
|
||||
assert.equal(file.lang, 'eng');
|
||||
assert.equal(file.trackName, 'English subs');
|
||||
assert.equal(file.size, 33075);
|
||||
assert.equal(file.url, 'https://animetosho.org/storage/attach/001dd61c/1955356.xz');
|
||||
assert.equal(file.sourceFilename, '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].mkv');
|
||||
assert.equal(file.filename, '[SubsPlease] Sousou no Frieren - 28 (1080p) [8BBBC28C].eng.ass');
|
||||
});
|
||||
|
||||
test('extractAnimetoshoSubtitleFiles skips image-based subtitle codecs', () => {
|
||||
const files = extractAnimetoshoSubtitleFiles({
|
||||
files: [
|
||||
{
|
||||
id: 1,
|
||||
filename: 'movie.mkv',
|
||||
attachments: [
|
||||
{ id: 10, type: 'subtitle', info: { codec: 'PGS', lang: 'eng' }, size: 100 },
|
||||
{ id: 11, type: 'subtitle', info: { codec: 'VobSub', lang: 'eng' }, size: 100 },
|
||||
{ id: 12, type: 'subtitle', info: { codec: 'SRT', lang: 'eng' }, size: 100 },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.deepEqual(
|
||||
files.map((f) => f.attachmentId),
|
||||
[12],
|
||||
);
|
||||
assert.equal(files[0]!.filename, 'movie.eng.srt');
|
||||
});
|
||||
|
||||
test('extractAnimetoshoSubtitleFiles sorts English tracks first and disambiguates duplicates', () => {
|
||||
const files = extractAnimetoshoSubtitleFiles({
|
||||
files: [
|
||||
{
|
||||
id: 1,
|
||||
filename: 'episode.mkv',
|
||||
attachments: [
|
||||
{
|
||||
id: 21,
|
||||
type: 'subtitle',
|
||||
info: { codec: 'ASS', lang: 'ger', name: 'Deutsch' },
|
||||
size: 1,
|
||||
},
|
||||
{
|
||||
id: 22,
|
||||
type: 'subtitle',
|
||||
info: { codec: 'ASS', lang: 'eng', name: 'Signs & Songs' },
|
||||
size: 2,
|
||||
},
|
||||
{
|
||||
id: 23,
|
||||
type: 'subtitle',
|
||||
info: { codec: 'ASS', lang: 'eng', name: 'Full Subtitles' },
|
||||
size: 3,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
files.map((f) => f.attachmentId),
|
||||
[22, 23, 21],
|
||||
);
|
||||
assert.equal(files[0]!.filename, 'episode.eng.signs-songs.ass');
|
||||
assert.equal(files[1]!.filename, 'episode.eng.full-subtitles.ass');
|
||||
assert.equal(files[2]!.filename, 'episode.ger.ass');
|
||||
});
|
||||
|
||||
test('extractAnimetoshoSubtitleFiles tolerates missing info fields', () => {
|
||||
const files = extractAnimetoshoSubtitleFiles({
|
||||
files: [
|
||||
{
|
||||
id: 1,
|
||||
attachments: [{ id: 31, type: 'subtitle', info: { codec: 'ASS' }, size: 5 }],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(files.length, 1);
|
||||
assert.equal(files[0]!.lang, '');
|
||||
assert.equal(files[0]!.trackName, null);
|
||||
assert.equal(files[0]!.filename, 'subtitle.ass');
|
||||
});
|
||||
|
||||
const hasXz = (() => {
|
||||
try {
|
||||
execFileSync('xz', ['--version'], { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
test('decompressXzFile round-trips an xz-compressed subtitle', { skip: !hasXz }, async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-animetosho-test-'));
|
||||
try {
|
||||
const plainPath = path.join(dir, 'sub.ass');
|
||||
const content = '[Script Info]\nTitle: test\n';
|
||||
fs.writeFileSync(plainPath, content, 'utf8');
|
||||
execFileSync('xz', ['-z', plainPath]);
|
||||
|
||||
const destPath = path.join(dir, 'out.ass');
|
||||
const result = await decompressXzFile(`${plainPath}.xz`, destPath);
|
||||
assert.equal(result.ok, true);
|
||||
if (result.ok) {
|
||||
assert.equal(result.path, destPath);
|
||||
}
|
||||
assert.equal(fs.readFileSync(destPath, 'utf8'), content);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('decompressXzFile reports an error for corrupt input', { skip: !hasXz }, async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-animetosho-test-'));
|
||||
try {
|
||||
const srcPath = path.join(dir, 'broken.xz');
|
||||
fs.writeFileSync(srcPath, 'not xz data');
|
||||
const result = await decompressXzFile(srcPath, path.join(dir, 'out.ass'));
|
||||
assert.equal(result.ok, false);
|
||||
if (!result.ok) {
|
||||
assert.match(result.error.error, /xz|decompress/i);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Binary file not shown.
Reference in New Issue
Block a user