mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-02-28 06:22:45 -08:00
1016 lines
39 KiB
JavaScript
1016 lines
39 KiB
JavaScript
/*
|
|
* Copyright (C) 2023-2025 Yomitan Authors
|
|
* Copyright (C) 2020-2022 Yomichan Authors
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import * as ajvSchemas0 from '../../lib/validate-schemas.js';
|
|
import {
|
|
BlobWriter as BlobWriter0,
|
|
TextWriter as TextWriter0,
|
|
Uint8ArrayReader as Uint8ArrayReader0,
|
|
ZipReader as ZipReader0,
|
|
configure,
|
|
} from '../../lib/zip.js';
|
|
import {ExtensionError} from '../core/extension-error.js';
|
|
import {parseJson} from '../core/json.js';
|
|
import {toError} from '../core/to-error.js';
|
|
import {stringReverse} from '../core/utilities.js';
|
|
import {getFileExtensionFromImageMediaType, getImageMediaTypeFromFileName} from '../media/media-util.js';
|
|
import {compareRevisions} from './dictionary-data-util.js';
|
|
|
|
const ajvSchemas = /** @type {import('dictionary-importer').CompiledSchemaValidators} */ (/** @type {unknown} */ (ajvSchemas0));
|
|
const BlobWriter = /** @type {typeof import('@zip.js/zip.js').BlobWriter} */ (/** @type {unknown} */ (BlobWriter0));
|
|
const TextWriter = /** @type {typeof import('@zip.js/zip.js').TextWriter} */ (/** @type {unknown} */ (TextWriter0));
|
|
const Uint8ArrayReader = /** @type {typeof import('@zip.js/zip.js').Uint8ArrayReader} */ (/** @type {unknown} */ (Uint8ArrayReader0));
|
|
const ZipReader = /** @type {typeof import('@zip.js/zip.js').ZipReader} */ (/** @type {unknown} */ (ZipReader0));
|
|
|
|
const INDEX_FILE_NAME = 'index.json';
|
|
|
|
export class DictionaryImporter {
|
|
/**
|
|
* @param {import('dictionary-importer-media-loader').GenericMediaLoader} mediaLoader
|
|
* @param {import('dictionary-importer').OnProgressCallback} [onProgress]
|
|
*/
|
|
constructor(mediaLoader, onProgress) {
|
|
/** @type {import('dictionary-importer-media-loader').GenericMediaLoader} */
|
|
this._mediaLoader = mediaLoader;
|
|
/** @type {import('dictionary-importer').OnProgressCallback} */
|
|
this._onProgress = typeof onProgress === 'function' ? onProgress : () => {};
|
|
/** @type {import('dictionary-importer').ProgressData} */
|
|
this._progressData = this._createProgressData();
|
|
}
|
|
|
|
/**
|
|
* @param {import('./dictionary-database.js').DictionaryDatabase} dictionaryDatabase
|
|
* @param {ArrayBuffer} archiveContent
|
|
* @param {import('dictionary-importer').ImportDetails} details
|
|
* @returns {Promise<import('dictionary-importer').ImportResult>}
|
|
*/
|
|
async importDictionary(dictionaryDatabase, archiveContent, details) {
|
|
if (!dictionaryDatabase) {
|
|
throw new Error('Invalid database');
|
|
}
|
|
if (!dictionaryDatabase.isPrepared()) {
|
|
throw new Error('Database is not ready');
|
|
}
|
|
|
|
/** @type {Error[]} */
|
|
const errors = [];
|
|
const maxTransactionLength = 1000;
|
|
const bulkAddProgressAllowance = 1000;
|
|
|
|
/**
|
|
* @template {import('dictionary-database').ObjectStoreName} T
|
|
* @param {T} objectStoreName
|
|
* @param {import('dictionary-database').ObjectStoreData<T>[]} entries
|
|
*/
|
|
const bulkAdd = async (objectStoreName, entries) => {
|
|
const entryCount = entries.length;
|
|
|
|
let progressIndexIncrease = bulkAddProgressAllowance / Math.ceil(entryCount / maxTransactionLength);
|
|
if (entryCount < maxTransactionLength) { progressIndexIncrease = bulkAddProgressAllowance; }
|
|
if (entryCount === 0) { this._progressData.index += progressIndexIncrease; }
|
|
|
|
for (let i = 0; i < entryCount; i += maxTransactionLength) {
|
|
const count = Math.min(maxTransactionLength, entryCount - i);
|
|
|
|
try {
|
|
await dictionaryDatabase.bulkAdd(objectStoreName, entries, i, count);
|
|
} catch (e) {
|
|
errors.push(toError(e));
|
|
}
|
|
|
|
this._progressData.index += progressIndexIncrease;
|
|
this._progress();
|
|
}
|
|
};
|
|
|
|
this._progressReset();
|
|
|
|
configure({
|
|
workerScripts: {
|
|
deflate: ['../../lib/z-worker.js'],
|
|
inflate: ['../../lib/z-worker.js'],
|
|
},
|
|
});
|
|
|
|
// Read archive
|
|
const fileMap = await this._getFilesFromArchive(archiveContent);
|
|
const index = await this._readAndValidateIndex(fileMap);
|
|
|
|
const dictionaryTitle = index.title;
|
|
const version = /** @type {import('dictionary-data').IndexVersion} */ (index.version);
|
|
|
|
// Verify database is not already imported
|
|
if (await dictionaryDatabase.dictionaryExists(dictionaryTitle)) {
|
|
return {
|
|
errors: [new Error(`Dictionary ${dictionaryTitle} is already imported, skipped it.`)],
|
|
result: null,
|
|
};
|
|
}
|
|
|
|
// Load schemas
|
|
this._progressNextStep(0);
|
|
const dataBankSchemas = this._getDataBankSchemas(version);
|
|
|
|
// Files
|
|
/** @type {import('dictionary-importer').QueryDetails} */
|
|
const queryDetails = [
|
|
['termFiles', /^term_bank_(\d+)\.json$/],
|
|
['termMetaFiles', /^term_meta_bank_(\d+)\.json$/],
|
|
['kanjiFiles', /^kanji_bank_(\d+)\.json$/],
|
|
['kanjiMetaFiles', /^kanji_meta_bank_(\d+)\.json$/],
|
|
['tagFiles', /^tag_bank_(\d+)\.json$/],
|
|
];
|
|
const {termFiles, termMetaFiles, kanjiFiles, kanjiMetaFiles, tagFiles} = Object.fromEntries(this._getArchiveFiles(fileMap, queryDetails));
|
|
|
|
// Load data
|
|
const prefixWildcardsSupported = !!details.prefixWildcardsSupported;
|
|
|
|
this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length);
|
|
|
|
for (const termFile of termFiles) { await this._validateFile(termFile, dataBankSchemas[0]); }
|
|
for (const termMetaFile of termMetaFiles) { await this._validateFile(termMetaFile, dataBankSchemas[1]); }
|
|
for (const kanjiFile of kanjiFiles) { await this._validateFile(kanjiFile, dataBankSchemas[2]); }
|
|
for (const kanjiMetaFile of kanjiMetaFiles) { await this._validateFile(kanjiMetaFile, dataBankSchemas[3]); }
|
|
for (const tagFile of tagFiles) { await this._validateFile(tagFile, dataBankSchemas[4]); }
|
|
|
|
// termFiles is doubled due to media importing
|
|
this._progressNextStep((termFiles.length * 2 + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length) * bulkAddProgressAllowance);
|
|
|
|
let importSuccess = false;
|
|
|
|
/** @type {import('dictionary-importer').SummaryCounts} */
|
|
const counts = {
|
|
terms: {total: 0},
|
|
termMeta: {total: 0},
|
|
kanji: {total: 0},
|
|
kanjiMeta: {total: 0},
|
|
tagMeta: {total: 0},
|
|
media: {total: 0},
|
|
};
|
|
|
|
const yomitanVersion = details.yomitanVersion;
|
|
/** @type {import('dictionary-importer').SummaryDetails} */
|
|
let summaryDetails = {prefixWildcardsSupported, counts, styles: '', yomitanVersion, importSuccess};
|
|
|
|
let summary = this._createSummary(dictionaryTitle, version, index, summaryDetails);
|
|
const dictionarySummaryAdd = await dictionaryDatabase.addWithResult('dictionaries', summary);
|
|
/** @type {Promise<IDBValidKey>} */
|
|
const dictionarySummaryResult = new Promise((resolve, reject) => {
|
|
dictionarySummaryAdd.onerror = () => reject(void 0);
|
|
dictionarySummaryAdd.onsuccess = () => resolve(dictionarySummaryAdd.result);
|
|
});
|
|
|
|
try {
|
|
const uniqueMediaPaths = new Set();
|
|
for (const termFile of termFiles) {
|
|
/** @type {import('dictionary-importer').ImportRequirement[]} */
|
|
const requirements = [];
|
|
let termList = await (
|
|
version === 1 ?
|
|
this._readFileSequence([termFile], this._convertTermBankEntryV1.bind(this), dictionaryTitle) :
|
|
this._readFileSequence([termFile], this._convertTermBankEntryV3.bind(this), dictionaryTitle)
|
|
);
|
|
|
|
// Prefix wildcard support
|
|
if (prefixWildcardsSupported) {
|
|
for (const entry of termList) {
|
|
entry.expressionReverse = stringReverse(entry.expression);
|
|
entry.readingReverse = stringReverse(entry.reading);
|
|
}
|
|
}
|
|
|
|
// Extended data support
|
|
for (let i = 0, ii = termList.length; i < ii; ++i) {
|
|
const entry = termList[i];
|
|
const glossaryList = entry.glossary;
|
|
for (let j = 0, jj = glossaryList.length; j < jj; ++j) {
|
|
const glossary = glossaryList[j];
|
|
if (typeof glossary !== 'object' || glossary === null || Array.isArray(glossary)) { continue; }
|
|
glossaryList[j] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements);
|
|
}
|
|
}
|
|
|
|
const alreadyAddedRequirements = requirements.filter((x) => { return uniqueMediaPaths.has(x.source.path); });
|
|
const notAddedRequirements = requirements.filter((x) => { return !uniqueMediaPaths.has(x.source.path); });
|
|
for (const requirement of requirements) { uniqueMediaPaths.add(requirement.source.path); }
|
|
|
|
await this._resolveAsyncRequirements(alreadyAddedRequirements, fileMap); // already added must also be resolved for the term dict to have correct data
|
|
let {media} = await this._resolveAsyncRequirements(notAddedRequirements, fileMap);
|
|
await bulkAdd('media', media);
|
|
counts.media.total += media.length;
|
|
|
|
this._progress();
|
|
|
|
await bulkAdd('terms', termList);
|
|
counts.terms.total += termList.length;
|
|
|
|
this._progress();
|
|
|
|
termList = [];
|
|
media = [];
|
|
}
|
|
|
|
for (const termMetaFile of termMetaFiles) {
|
|
let termMetaList = await this._readFileSequence([termMetaFile], this._convertTermMetaBankEntry.bind(this), dictionaryTitle);
|
|
|
|
await bulkAdd('termMeta', termMetaList);
|
|
for (const [key, value] of Object.entries(this._getMetaCounts(termMetaList))) {
|
|
if (key in counts.termMeta) {
|
|
counts.termMeta[key] += value;
|
|
} else {
|
|
counts.termMeta[key] = value;
|
|
}
|
|
}
|
|
|
|
this._progress();
|
|
|
|
termMetaList = [];
|
|
}
|
|
|
|
for (const kanjiFile of kanjiFiles) {
|
|
let kanjiList = await (
|
|
version === 1 ?
|
|
this._readFileSequence([kanjiFile], this._convertKanjiBankEntryV1.bind(this), dictionaryTitle) :
|
|
this._readFileSequence([kanjiFile], this._convertKanjiBankEntryV3.bind(this), dictionaryTitle)
|
|
);
|
|
|
|
await bulkAdd('kanji', kanjiList);
|
|
counts.kanji.total += kanjiList.length;
|
|
|
|
this._progress();
|
|
|
|
kanjiList = [];
|
|
}
|
|
|
|
for (const kanjiMetaFile of kanjiMetaFiles) {
|
|
let kanjiMetaList = await this._readFileSequence([kanjiMetaFile], this._convertKanjiMetaBankEntry.bind(this), dictionaryTitle);
|
|
|
|
await bulkAdd('kanjiMeta', kanjiMetaList);
|
|
for (const [key, value] of Object.entries(this._getMetaCounts(kanjiMetaList))) {
|
|
if (key in counts.kanjiMeta) {
|
|
counts.kanjiMeta[key] += value;
|
|
} else {
|
|
counts.kanjiMeta[key] = value;
|
|
}
|
|
}
|
|
|
|
this._progress();
|
|
|
|
kanjiMetaList = [];
|
|
}
|
|
|
|
for (const tagFile of tagFiles) {
|
|
let tagList = await this._readFileSequence([tagFile], this._convertTagBankEntry.bind(this), dictionaryTitle);
|
|
this._addOldIndexTags(index, tagList, dictionaryTitle);
|
|
|
|
await bulkAdd('tagMeta', tagList);
|
|
counts.tagMeta.total += tagList.length;
|
|
|
|
this._progress();
|
|
|
|
tagList = [];
|
|
}
|
|
|
|
importSuccess = true;
|
|
} catch (e) {
|
|
errors.push(toError(e));
|
|
}
|
|
|
|
// Update dictionary descriptor
|
|
this._progressNextStep(0);
|
|
|
|
const stylesFileName = 'styles.css';
|
|
const stylesFile = fileMap.get(stylesFileName);
|
|
let styles = '';
|
|
if (typeof stylesFile !== 'undefined') {
|
|
styles = await this._getData(stylesFile, new TextWriter());
|
|
const cssErrors = this._validateCss(styles);
|
|
if (cssErrors.length > 0) {
|
|
return {
|
|
errors: cssErrors,
|
|
result: null,
|
|
};
|
|
}
|
|
}
|
|
|
|
summaryDetails = {prefixWildcardsSupported, counts, styles, yomitanVersion, importSuccess};
|
|
summary = this._createSummary(dictionaryTitle, version, index, summaryDetails);
|
|
const primaryKey = await dictionarySummaryResult;
|
|
await dictionaryDatabase.bulkUpdate('dictionaries', [{data: summary, primaryKey}], 0, 1);
|
|
|
|
this._progress();
|
|
|
|
return {result: summary, errors};
|
|
}
|
|
|
|
/**
|
|
* @param {ArrayBuffer} archiveContent
|
|
* @returns {Promise<import('dictionary-importer').ArchiveFileMap>}
|
|
*/
|
|
async _getFilesFromArchive(archiveContent) {
|
|
const zipFileReader = new Uint8ArrayReader(new Uint8Array(archiveContent));
|
|
const zipReader = new ZipReader(zipFileReader);
|
|
const zipEntries = await zipReader.getEntries();
|
|
/** @type {import('dictionary-importer').ArchiveFileMap} */
|
|
const fileMap = new Map();
|
|
for (const entry of zipEntries) {
|
|
fileMap.set(entry.filename, entry);
|
|
}
|
|
return fileMap;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ArchiveFileMap} fileMap
|
|
* @returns {?string}
|
|
*/
|
|
_findRedundantDirectories(fileMap) {
|
|
let indexPath = '';
|
|
for (const file of fileMap) {
|
|
if (file[0].replace(/.*\//, '') === INDEX_FILE_NAME) {
|
|
indexPath = file[0];
|
|
}
|
|
}
|
|
const redundantDirectoriesRegex = new RegExp(`.*(?=${INDEX_FILE_NAME.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`);
|
|
const redundantDirectories = indexPath.match(redundantDirectoriesRegex);
|
|
return redundantDirectories ? redundantDirectories[0] : null;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ArchiveFileMap} fileMap
|
|
* @returns {Promise<import('dictionary-data').Index>}
|
|
* @throws {Error}
|
|
*/
|
|
async _readAndValidateIndex(fileMap) {
|
|
const indexFile = fileMap.get(INDEX_FILE_NAME);
|
|
if (typeof indexFile === 'undefined') {
|
|
const redundantDirectories = this._findRedundantDirectories(fileMap);
|
|
if (redundantDirectories) {
|
|
throw new Error('Dictionary index found nested in redundant directories: "' + redundantDirectories + '" when it must be in the archive\'s root directory');
|
|
}
|
|
throw new Error('No dictionary index found in archive');
|
|
}
|
|
const indexFile2 = /** @type {import('@zip.js/zip.js').Entry} */ (indexFile);
|
|
|
|
const indexContent = await this._getData(indexFile2, new TextWriter());
|
|
const index = /** @type {unknown} */ (parseJson(indexContent));
|
|
|
|
if (!ajvSchemas.dictionaryIndex(index)) {
|
|
throw this._formatAjvSchemaError(ajvSchemas.dictionaryIndex, INDEX_FILE_NAME);
|
|
}
|
|
|
|
const validIndex = /** @type {import('dictionary-data').Index} */ (index);
|
|
|
|
const version = typeof validIndex.format === 'number' ? validIndex.format : validIndex.version;
|
|
validIndex.version = version;
|
|
|
|
const {title, revision} = validIndex;
|
|
if (typeof version !== 'number' || !title || !revision) {
|
|
throw new Error('Unrecognized dictionary format');
|
|
}
|
|
|
|
return validIndex;
|
|
}
|
|
|
|
/**
|
|
* @returns {import('dictionary-importer').ProgressData}
|
|
*/
|
|
_createProgressData() {
|
|
return {
|
|
index: 0,
|
|
count: 0,
|
|
};
|
|
}
|
|
|
|
/** */
|
|
_progressReset() {
|
|
this._progressData = this._createProgressData();
|
|
this._progress(true);
|
|
}
|
|
|
|
/**
|
|
* @param {number} count
|
|
*/
|
|
_progressNextStep(count) {
|
|
this._progressData.index = 0;
|
|
this._progressData.count = count;
|
|
this._progress(true);
|
|
}
|
|
|
|
/**
|
|
* @param {boolean} nextStep
|
|
*/
|
|
_progress(nextStep = false) {
|
|
this._onProgress({...this._progressData, nextStep});
|
|
}
|
|
|
|
/**
|
|
* @param {string} dictionaryTitle
|
|
* @param {number} version
|
|
* @param {import('dictionary-data').Index} index
|
|
* @param {import('dictionary-importer').SummaryDetails} details
|
|
* @returns {import('dictionary-importer').Summary}
|
|
* @throws {Error}
|
|
*/
|
|
_createSummary(dictionaryTitle, version, index, details) {
|
|
const indexSequenced = index.sequenced;
|
|
const {prefixWildcardsSupported, counts, styles, importSuccess} = details;
|
|
/** @type {import('dictionary-importer').Summary} */
|
|
const summary = {
|
|
title: dictionaryTitle,
|
|
revision: index.revision,
|
|
sequenced: typeof indexSequenced === 'boolean' && indexSequenced,
|
|
version,
|
|
importDate: Date.now(),
|
|
prefixWildcardsSupported,
|
|
counts,
|
|
styles,
|
|
importSuccess,
|
|
};
|
|
|
|
const {minimumYomitanVersion, author, url, description, attribution, frequencyMode, isUpdatable, sourceLanguage, targetLanguage} = index;
|
|
if (typeof minimumYomitanVersion === 'string') {
|
|
if (details.yomitanVersion === '0.0.0.0') {
|
|
// Running a development version of Yomitan
|
|
} else if (compareRevisions(details.yomitanVersion, minimumYomitanVersion)) {
|
|
throw new Error(`Dictionary is incompatible with this version of Yomitan (${details.yomitanVersion}; minimum required: ${minimumYomitanVersion})`);
|
|
}
|
|
summary.minimumYomitanVersion = minimumYomitanVersion;
|
|
}
|
|
if (typeof author === 'string') { summary.author = author; }
|
|
if (typeof url === 'string') { summary.url = url; }
|
|
if (typeof description === 'string') { summary.description = description; }
|
|
if (typeof attribution === 'string') { summary.attribution = attribution; }
|
|
if (typeof frequencyMode === 'string') { summary.frequencyMode = frequencyMode; }
|
|
if (typeof sourceLanguage === 'string') { summary.sourceLanguage = sourceLanguage; }
|
|
if (typeof targetLanguage === 'string') { summary.targetLanguage = targetLanguage; }
|
|
if (typeof isUpdatable === 'boolean') {
|
|
const {indexUrl, downloadUrl} = index;
|
|
if (!isUpdatable || !this._validateUrl(indexUrl) || !this._validateUrl(downloadUrl)) {
|
|
throw new Error('Invalid index data for updatable dictionary');
|
|
}
|
|
summary.isUpdatable = isUpdatable;
|
|
summary.indexUrl = indexUrl;
|
|
summary.downloadUrl = downloadUrl;
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
/**
|
|
* @param {string|undefined} string
|
|
* @returns {boolean}
|
|
*/
|
|
_validateUrl(string) {
|
|
if (typeof string !== 'string') {
|
|
return false;
|
|
}
|
|
|
|
let url;
|
|
try {
|
|
url = new URL(string);
|
|
} catch (_) {
|
|
return false;
|
|
}
|
|
|
|
return url.protocol === 'http:' || url.protocol === 'https:';
|
|
}
|
|
|
|
/**
|
|
* @param {import('ajv').ValidateFunction} schema
|
|
* @param {string} fileName
|
|
* @returns {ExtensionError}
|
|
*/
|
|
_formatAjvSchemaError(schema, fileName) {
|
|
const e = new ExtensionError(`Dictionary has invalid data in '${fileName}' '${JSON.stringify(schema.errors)}'`);
|
|
e.data = schema.errors;
|
|
return e;
|
|
}
|
|
|
|
/**
|
|
* @param {number} version
|
|
* @returns {import('dictionary-importer').CompiledSchemaNameArray}
|
|
*/
|
|
_getDataBankSchemas(version) {
|
|
const termBank = (
|
|
version === 1 ?
|
|
'dictionaryTermBankV1' :
|
|
'dictionaryTermBankV3'
|
|
);
|
|
const termMetaBank = 'dictionaryTermMetaBankV3';
|
|
const kanjiBank = (
|
|
version === 1 ?
|
|
'dictionaryKanjiBankV1' :
|
|
'dictionaryKanjiBankV3'
|
|
);
|
|
const kanjiMetaBank = 'dictionaryKanjiMetaBankV3';
|
|
const tagBank = 'dictionaryTagBankV3';
|
|
|
|
return [termBank, termMetaBank, kanjiBank, kanjiMetaBank, tagBank];
|
|
}
|
|
|
|
/**
|
|
* @param {string} css
|
|
* @returns {Error[]}
|
|
*/
|
|
_validateCss(css) {
|
|
return css ? [] : [new Error('No styles found')];
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermGlossaryText|import('dictionary-data').TermGlossaryImage|import('dictionary-data').TermGlossaryStructuredContent} data
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @returns {import('dictionary-data').TermGlossary}
|
|
* @throws {Error}
|
|
*/
|
|
_formatDictionaryTermGlossaryObject(data, entry, requirements) {
|
|
switch (data.type) {
|
|
case 'text':
|
|
return data.text;
|
|
case 'image':
|
|
return this._formatDictionaryTermGlossaryImage(data, entry, requirements);
|
|
case 'structured-content':
|
|
return this._formatStructuredContent(data, entry, requirements);
|
|
default:
|
|
throw new Error(`Unhandled data type: ${/** @type {import('core').SerializableObject} */ (data).type}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermGlossaryImage} data
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @returns {import('dictionary-data').TermGlossaryImage}
|
|
*/
|
|
_formatDictionaryTermGlossaryImage(data, entry, requirements) {
|
|
/** @type {import('dictionary-data').TermGlossaryImage} */
|
|
const target = {
|
|
type: 'image',
|
|
path: '', // Will be populated during requirement resolution
|
|
};
|
|
requirements.push({type: 'image', target, source: data, entry});
|
|
return target;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermGlossaryStructuredContent} data
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @returns {import('dictionary-data').TermGlossaryStructuredContent}
|
|
*/
|
|
_formatStructuredContent(data, entry, requirements) {
|
|
const content = this._prepareStructuredContent(data.content, entry, requirements);
|
|
return {
|
|
type: 'structured-content',
|
|
content,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {import('structured-content').Content} content
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @returns {import('structured-content').Content}
|
|
*/
|
|
_prepareStructuredContent(content, entry, requirements) {
|
|
if (typeof content === 'string' || !(typeof content === 'object' && content !== null)) {
|
|
return content;
|
|
}
|
|
if (Array.isArray(content)) {
|
|
for (let i = 0, ii = content.length; i < ii; ++i) {
|
|
content[i] = this._prepareStructuredContent(content[i], entry, requirements);
|
|
}
|
|
return content;
|
|
}
|
|
const {tag} = content;
|
|
switch (tag) {
|
|
case 'img':
|
|
return this._prepareStructuredContentImage(content, entry, requirements);
|
|
}
|
|
const childContent = content.content;
|
|
if (typeof childContent !== 'undefined') {
|
|
content.content = this._prepareStructuredContent(childContent, entry, requirements);
|
|
}
|
|
return content;
|
|
}
|
|
|
|
/**
|
|
* @param {import('structured-content').ImageElement} content
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @returns {import('structured-content').ImageElement}
|
|
*/
|
|
_prepareStructuredContentImage(content, entry, requirements) {
|
|
/** @type {import('structured-content').ImageElement} */
|
|
const target = {
|
|
tag: 'img',
|
|
path: '', // Will be populated during requirement resolution
|
|
};
|
|
requirements.push({type: 'structured-content-image', target, source: content, entry});
|
|
return target;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirement[]} requirements
|
|
* @param {import('dictionary-importer').ArchiveFileMap} fileMap
|
|
* @returns {Promise<{media: import('dictionary-database').MediaDataArrayBufferContent[]}>}
|
|
*/
|
|
async _resolveAsyncRequirements(requirements, fileMap) {
|
|
/** @type {Map<string, import('dictionary-database').MediaDataArrayBufferContent>} */
|
|
const media = new Map();
|
|
/** @type {import('dictionary-importer').ImportRequirementContext} */
|
|
const context = {fileMap, media};
|
|
|
|
for (const requirement of requirements) {
|
|
await this._resolveAsyncRequirement(context, requirement);
|
|
}
|
|
|
|
return {
|
|
media: [...media.values()],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirementContext} context
|
|
* @param {import('dictionary-importer').ImportRequirement} requirement
|
|
*/
|
|
async _resolveAsyncRequirement(context, requirement) {
|
|
switch (requirement.type) {
|
|
case 'image':
|
|
await this._resolveDictionaryTermGlossaryImage(
|
|
context,
|
|
requirement.target,
|
|
requirement.source,
|
|
requirement.entry,
|
|
);
|
|
break;
|
|
case 'structured-content-image':
|
|
await this._resolveStructuredContentImage(
|
|
context,
|
|
requirement.target,
|
|
requirement.source,
|
|
requirement.entry,
|
|
);
|
|
break;
|
|
default:
|
|
return;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirementContext} context
|
|
* @param {import('dictionary-data').TermGlossaryImage} target
|
|
* @param {import('dictionary-data').TermGlossaryImage} source
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
*/
|
|
async _resolveDictionaryTermGlossaryImage(context, target, source, entry) {
|
|
await this._createImageData(context, target, source, entry);
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirementContext} context
|
|
* @param {import('structured-content').ImageElement} target
|
|
* @param {import('structured-content').ImageElement} source
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
*/
|
|
async _resolveStructuredContentImage(context, target, source, entry) {
|
|
const {
|
|
verticalAlign,
|
|
border,
|
|
borderRadius,
|
|
sizeUnits,
|
|
} = source;
|
|
await this._createImageData(context, target, source, entry);
|
|
if (typeof verticalAlign === 'string') { target.verticalAlign = verticalAlign; }
|
|
if (typeof border === 'string') { target.border = border; }
|
|
if (typeof borderRadius === 'string') { target.borderRadius = borderRadius; }
|
|
if (typeof sizeUnits === 'string') { target.sizeUnits = sizeUnits; }
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirementContext} context
|
|
* @param {import('structured-content').ImageElementBase} target
|
|
* @param {import('structured-content').ImageElementBase} source
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
*/
|
|
async _createImageData(context, target, source, entry) {
|
|
const {
|
|
path,
|
|
width: preferredWidth,
|
|
height: preferredHeight,
|
|
title,
|
|
alt,
|
|
description,
|
|
pixelated,
|
|
imageRendering,
|
|
appearance,
|
|
background,
|
|
collapsed,
|
|
collapsible,
|
|
} = source;
|
|
const {width, height} = await this._getImageMedia(context, path, entry);
|
|
target.path = path;
|
|
target.width = width;
|
|
target.height = height;
|
|
if (typeof preferredWidth === 'number') { target.preferredWidth = preferredWidth; }
|
|
if (typeof preferredHeight === 'number') { target.preferredHeight = preferredHeight; }
|
|
if (typeof title === 'string') { target.title = title; }
|
|
if (typeof alt === 'string') { target.alt = alt; }
|
|
if (typeof description === 'string') { target.description = description; }
|
|
if (typeof pixelated === 'boolean') { target.pixelated = pixelated; }
|
|
if (typeof imageRendering === 'string') { target.imageRendering = imageRendering; }
|
|
if (typeof appearance === 'string') { target.appearance = appearance; }
|
|
if (typeof background === 'boolean') { target.background = background; }
|
|
if (typeof collapsed === 'boolean') { target.collapsed = collapsed; }
|
|
if (typeof collapsible === 'boolean') { target.collapsible = collapsible; }
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ImportRequirementContext} context
|
|
* @param {string} path
|
|
* @param {import('dictionary-database').DatabaseTermEntry} entry
|
|
* @returns {Promise<import('dictionary-database').MediaDataArrayBufferContent>}
|
|
*/
|
|
async _getImageMedia(context, path, entry) {
|
|
const {media} = context;
|
|
const {dictionary} = entry;
|
|
|
|
/**
|
|
* @param {string} message
|
|
* @returns {Error}
|
|
*/
|
|
const createError = (message) => {
|
|
const {expression, reading} = entry;
|
|
const readingSource = reading.length > 0 ? ` (${reading})` : '';
|
|
return new Error(`${message} at path ${JSON.stringify(path)} for ${expression}${readingSource} in ${dictionary}`);
|
|
};
|
|
|
|
// Check if already added
|
|
let mediaData = media.get(path);
|
|
if (typeof mediaData !== 'undefined') {
|
|
if (getFileExtensionFromImageMediaType(mediaData.mediaType) === null) {
|
|
throw createError('Media file is not a valid image');
|
|
}
|
|
return mediaData;
|
|
}
|
|
|
|
// Find file in archive
|
|
const file = context.fileMap.get(path);
|
|
if (typeof file === 'undefined') {
|
|
throw createError('Could not find image');
|
|
}
|
|
|
|
// Load file content
|
|
let content = await (await this._getData(file, new BlobWriter())).arrayBuffer();
|
|
|
|
const mediaType = getImageMediaTypeFromFileName(path);
|
|
if (mediaType === null) {
|
|
throw createError('Could not determine media type for image');
|
|
}
|
|
|
|
// Load image data
|
|
let width;
|
|
let height;
|
|
try {
|
|
({content, width, height} = await this._mediaLoader.getImageDetails(content, mediaType));
|
|
} catch (e) {
|
|
throw createError('Could not load image');
|
|
}
|
|
|
|
// Create image data
|
|
mediaData = {
|
|
dictionary,
|
|
path,
|
|
mediaType,
|
|
width,
|
|
height,
|
|
content,
|
|
};
|
|
media.set(path, mediaData);
|
|
|
|
return mediaData;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermV1} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseTermEntry}
|
|
*/
|
|
_convertTermBankEntryV1(entry, dictionary) {
|
|
let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
|
|
reading = reading.length > 0 ? reading : expression;
|
|
return {expression, reading, definitionTags, rules, score, glossary, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermV3} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseTermEntry}
|
|
*/
|
|
_convertTermBankEntryV3(entry, dictionary) {
|
|
let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
|
|
reading = reading.length > 0 ? reading : expression;
|
|
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').TermMeta} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseTermMeta}
|
|
*/
|
|
_convertTermMetaBankEntry(entry, dictionary) {
|
|
const [expression, mode, data] = entry;
|
|
return /** @type {import('dictionary-database').DatabaseTermMeta} */ ({expression, mode, data, dictionary});
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').KanjiV1} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseKanjiEntry}
|
|
*/
|
|
_convertKanjiBankEntryV1(entry, dictionary) {
|
|
const [character, onyomi, kunyomi, tags, ...meanings] = entry;
|
|
return {character, onyomi, kunyomi, tags, meanings, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').KanjiV3} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseKanjiEntry}
|
|
*/
|
|
_convertKanjiBankEntryV3(entry, dictionary) {
|
|
const [character, onyomi, kunyomi, tags, meanings, stats] = entry;
|
|
return {character, onyomi, kunyomi, tags, meanings, stats, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').KanjiMeta} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').DatabaseKanjiMeta}
|
|
*/
|
|
_convertKanjiMetaBankEntry(entry, dictionary) {
|
|
const [character, mode, data] = entry;
|
|
return {character, mode, data, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').Tag} entry
|
|
* @param {string} dictionary
|
|
* @returns {import('dictionary-database').Tag}
|
|
*/
|
|
_convertTagBankEntry(entry, dictionary) {
|
|
const [name, category, order, notes, score] = entry;
|
|
return {name, category, order, notes, score, dictionary};
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-data').Index} index
|
|
* @param {import('dictionary-database').Tag[]} results
|
|
* @param {string} dictionary
|
|
*/
|
|
_addOldIndexTags(index, results, dictionary) {
|
|
const {tagMeta} = index;
|
|
if (typeof tagMeta !== 'object' || tagMeta === null) { return; }
|
|
for (const [name, value] of Object.entries(tagMeta)) {
|
|
const {category, order, notes, score} = value;
|
|
results.push({name, category, order, notes, score, dictionary});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-importer').ArchiveFileMap} fileMap
|
|
* @param {import('dictionary-importer').QueryDetails} queryDetails
|
|
* @returns {import('dictionary-importer').QueryResult}
|
|
*/
|
|
_getArchiveFiles(fileMap, queryDetails) {
|
|
/** @type {import('dictionary-importer').QueryResult} */
|
|
const results = new Map();
|
|
|
|
for (const [fileType] of queryDetails) {
|
|
results.set(fileType, []);
|
|
}
|
|
|
|
for (const [fileName, fileEntry] of fileMap.entries()) {
|
|
for (const [fileType, fileNameFormat] of queryDetails) {
|
|
if (!fileNameFormat.test(fileName)) { continue; }
|
|
const entries = results.get(fileType);
|
|
|
|
if (typeof entries !== 'undefined') {
|
|
entries.push(fileEntry);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* @template [TEntry=unknown]
|
|
* @template [TResult=unknown]
|
|
* @param {import('@zip.js/zip.js').Entry[]} files
|
|
* @param {(entry: TEntry, dictionaryTitle: string) => TResult} convertEntry
|
|
* @param {string} dictionaryTitle
|
|
* @returns {Promise<TResult[]>}
|
|
*/
|
|
async _readFileSequence(files, convertEntry, dictionaryTitle) {
|
|
const results = [];
|
|
for (const file of files) {
|
|
const content = await this._getData(file, new TextWriter());
|
|
let entries;
|
|
|
|
try {
|
|
/** @type {unknown} */
|
|
entries = parseJson(content);
|
|
} catch (error) {
|
|
if (error instanceof Error) {
|
|
throw new Error(error.message + ` in '${file.filename}'`);
|
|
}
|
|
}
|
|
|
|
if (Array.isArray(entries)) {
|
|
for (const entry of /** @type {TEntry[]} */ (entries)) {
|
|
results.push(convertEntry(entry, dictionaryTitle));
|
|
}
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* @param {import('@zip.js/zip.js').Entry} file
|
|
* @param {import('dictionary-importer').CompiledSchemaName} schemaName
|
|
* @returns {Promise<boolean>}
|
|
*/
|
|
async _validateFile(file, schemaName) {
|
|
const content = await this._getData(file, new TextWriter());
|
|
let entries;
|
|
|
|
try {
|
|
/** @type {unknown} */
|
|
entries = parseJson(content);
|
|
} catch (error) {
|
|
if (error instanceof Error) {
|
|
throw new Error(error.message + ` in '${file.filename}'`);
|
|
}
|
|
}
|
|
|
|
const schema = ajvSchemas[schemaName];
|
|
if (!schema(entries)) {
|
|
throw this._formatAjvSchemaError(schema, file.filename);
|
|
}
|
|
|
|
++this._progressData.index;
|
|
this._progress();
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param {import('dictionary-database').DatabaseTermMeta[]|import('dictionary-database').DatabaseKanjiMeta[]} metaList
|
|
* @returns {import('dictionary-importer').SummaryMetaCount}
|
|
*/
|
|
_getMetaCounts(metaList) {
|
|
/** @type {Map<string, number>} */
|
|
const countsMap = new Map();
|
|
for (const {mode} of metaList) {
|
|
let count = countsMap.get(mode);
|
|
count = typeof count !== 'undefined' ? count + 1 : 1;
|
|
countsMap.set(mode, count);
|
|
}
|
|
/** @type {import('dictionary-importer').SummaryMetaCount} */
|
|
const counts = {total: metaList.length};
|
|
for (const [key, value] of countsMap.entries()) {
|
|
if (Object.prototype.hasOwnProperty.call(counts, key)) { continue; }
|
|
counts[key] = value;
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
/**
|
|
* @template [T=unknown]
|
|
* @param {import('@zip.js/zip.js').Entry} entry
|
|
* @param {import('@zip.js/zip.js').Writer<T>|import('@zip.js/zip.js').WritableWriter} writer
|
|
* @returns {Promise<T>}
|
|
*/
|
|
async _getData(entry, writer) {
|
|
if (typeof entry.getData === 'undefined') {
|
|
throw new Error(`Cannot read ${entry.filename}`);
|
|
}
|
|
return await entry.getData(writer);
|
|
}
|
|
}
|