sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-02-27 18:22:41 -08:00
Code Issues Packages Projects Releases Wiki Activity

chore: bootstrap repository tooling and release automation

Browse Source
This commit is contained in:
Kyle Yasuda
2026-02-22 21:43:43 -08:00
commit 448ce03fd4
27 changed files with 6197 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
scripts/build-macos-helper.sh Executable file
View File

@@ -0,0 +1,54 @@
#!/bin/bash
# Build macOS window tracking helper binary
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SWIFT_SOURCE="$SCRIPT_DIR/get-mpv-window-macos.swift"
OUTPUT_DIR="$SCRIPT_DIR/../dist/scripts"
OUTPUT_BINARY="$OUTPUT_DIR/get-mpv-window-macos"
OUTPUT_SOURCE_COPY="$OUTPUT_DIR/get-mpv-window-macos.swift"
fallback_to_source() {
echo "Falling back to source fallback: $OUTPUT_SOURCE_COPY"
mkdir -p "$OUTPUT_DIR"
cp "$SWIFT_SOURCE" "$OUTPUT_SOURCE_COPY"
}
build_swift_helper() {
echo "Compiling macOS window tracking helper..."
if ! command -v swiftc >/dev/null 2>&1; then
echo "swiftc not found in PATH; skipping compilation."
return 1
fi
if ! swiftc -O "$SWIFT_SOURCE" -o "$OUTPUT_BINARY"; then
return 1
fi
chmod +x "$OUTPUT_BINARY"
echo "✓ Built $OUTPUT_BINARY"
return 0
}
# Optional skip flag for non-macOS CI/dev environments
if [[ "${SUBMINER_SKIP_MACOS_HELPER_BUILD:-}" == "1" ]]; then
echo "Skipping macOS helper build (SUBMINER_SKIP_MACOS_HELPER_BUILD=1)"
fallback_to_source
exit 0
fi
# Only build on macOS
if [[ "$(uname)" != "Darwin" ]]; then
echo "Skipping macOS helper build (not on macOS)"
fallback_to_source
exit 0
fi
# Create output directory
mkdir -p "$OUTPUT_DIR"
# Compile Swift script to binary, fallback to source if unavailable or compilation fails
if ! build_swift_helper; then
fallback_to_source
fi

165
scripts/docs-sweep-once.sh Executable file
View File

@@ -0,0 +1,165 @@
#!/usr/bin/env bash
set -euo pipefail
REPO="${REPO:-$HOME/projects/japanese/SubMiner}"
LOCK_FILE="${LOCK_FILE:-/tmp/subminer-doc-sweep.lock}"
STATE_FILE="${STATE_FILE:-/tmp/subminer-doc-sweep.state}"
LOG_FILE="${LOG_FILE:-$REPO/.codex-doc-sweep.log}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-240}"
SUBAGENT_ROOT="${SUBAGENT_ROOT:-$REPO/docs/subagents}"
SUBAGENT_INDEX_FILE="${SUBAGENT_INDEX_FILE:-$SUBAGENT_ROOT/INDEX.md}"
SUBAGENT_COLLAB_FILE="${SUBAGENT_COLLAB_FILE:-$SUBAGENT_ROOT/collaboration.md}"
SUBAGENT_AGENTS_DIR="${SUBAGENT_AGENTS_DIR:-$SUBAGENT_ROOT/agents}"
LEGACY_SUBAGENT_FILE="${LEGACY_SUBAGENT_FILE:-$REPO/docs/subagent.md}"
AGENT_ID="${AGENT_ID:-docs-sweep}"
AGENT_ALIAS="${AGENT_ALIAS:-Docs Sweep}"
AGENT_MISSION="${AGENT_MISSION:-Docs drift cleanup and coordination updates}"
# Non-interactive agent command used to run the prompt.
# Example:
# AGENT_CMD='codex exec'
# AGENT_CMD='opencode run'
AGENT_CMD="${AGENT_CMD:-codex exec}"
AGENT_ID_SAFE="$(printf '%s' "$AGENT_ID" | tr -c 'A-Za-z0-9._-' '_')"
AGENT_FILE="${SUBAGENT_AGENTS_DIR}/${AGENT_ID_SAFE}.md"
mkdir -p "$(dirname "$LOCK_FILE")"
mkdir -p "$(dirname "$STATE_FILE")"
mkdir -p "$SUBAGENT_ROOT" "$SUBAGENT_AGENTS_DIR" "$SUBAGENT_ROOT/archive"
exec 9> "$LOCK_FILE"
if ! flock -n 9; then
exit 0
fi
cd "$REPO"
current_state="$({
git status --porcelain=v1
git ls-files --others --exclude-standard
} | sha256sum | cut -d' ' -f1)"
previous_state="$(cat "$STATE_FILE" 2> /dev/null || true)"
if [[ "$current_state" == "$previous_state" ]]; then
exit 0
fi
printf '%s' "$current_state" > "$STATE_FILE"
run_started_at="$(date -Is)"
run_started_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "[RUN] [$run_started_at] docs sweep running (agent_id=$AGENT_ID alias=$AGENT_ALIAS)"
echo "[$run_started_at] state changed; starting docs sweep (agent_id=$AGENT_ID alias=$AGENT_ALIAS)" >> "$LOG_FILE"
if [[ ! -f "$SUBAGENT_INDEX_FILE" ]]; then
cat > "$SUBAGENT_INDEX_FILE" << 'EOF'
# Subagents Index
Read first. Keep concise.
| agent_id | alias | mission | status | file | last_update_utc |
| --- | --- | --- | --- | --- | --- |
EOF
fi
if [[ ! -f "$SUBAGENT_COLLAB_FILE" ]]; then
cat > "$SUBAGENT_COLLAB_FILE" << 'EOF'
# Subagents Collaboration
Shared notes. Append-only.
- [YYYY-MM-DDTHH:MM:SSZ] [agent_id|alias] note, question, dependency, conflict, decision.
EOF
fi
if [[ ! -f "$AGENT_FILE" ]]; then
cat > "$AGENT_FILE" << EOF
# Agent: $AGENT_ID
- alias: $AGENT_ALIAS
- mission: $AGENT_MISSION
- status: planning
- branch: unknown
- started_at: $run_started_utc
- heartbeat_minutes: 20
## Current Work (newest first)
- [$run_started_utc] intent: initialize section
## Files Touched
- none yet
## Assumptions
- none yet
## Open Questions / Blockers
- none
## Next Step
- continue run
EOF
fi
if [[ -f "$LEGACY_SUBAGENT_FILE" ]]; then
echo "[WARN] [$run_started_at] legacy file exists; prefer sharded layout: $LEGACY_SUBAGENT_FILE" | tee -a "$LOG_FILE"
fi
read -r -d '' PROMPT << EOF || true
Watch for in-flight refactors. If repo changes introduced drift, update only:
- README.md
- AGENTS.md
- docs/**/*.md
- config.example.jsonc
- docs/public/config.example.jsonc <-- generated automatically with make generate-example-config / bun run generate:config-example
- package.json scripts/config references (only if needed)
Coordination protocol:
- Read in order before edits:
1) \`$SUBAGENT_INDEX_FILE\`
2) \`$SUBAGENT_COLLAB_FILE\`
3) \`$AGENT_FILE\`
- Edit scope:
- MAY edit own file: \`$AGENT_FILE\`
- MAY append to collaboration: \`$SUBAGENT_COLLAB_FILE\`
- MAY update own row in index: \`$SUBAGENT_INDEX_FILE\`
- MUST NOT edit other agent files in \`$SUBAGENT_AGENTS_DIR\`
- Ensure own file has updated: alias, mission, status, branch, started_at, heartbeat_minutes.
- Add UTC ISO entries in "Current Work (newest first)" for intent/progress/handoff for this run.
- Keep own file sections current: Files Touched, assumptions, blockers, next step.
- Ensure index row for \`$AGENT_ID\` reflects alias/mission/status/file/last_update_utc.
- If file conflict/dependency seen, append note in collaboration.
Run metadata:
- run_started_at_utc: $run_started_utc
- repo: $REPO
- agent_id: $AGENT_ID
- agent_alias: $AGENT_ALIAS
- agent_file: $AGENT_FILE
Rules:
- Keep edits minimal and accurate to current code.
- Do not commit.
- Do not push.
- If ambiguous, do not guess; skip and report uncertainty.
- Print concise summary with:
1) files changed + why
2) coordination updates made (\`$SUBAGENT_INDEX_FILE\`, \`$SUBAGENT_COLLAB_FILE\`, \`$AGENT_FILE\`)
3) open questions/blockers
EOF
quoted_prompt="$(printf '%q' "$PROMPT")"
job_status=0
if timeout "${TIMEOUT_SECONDS}s" bash -lc "$AGENT_CMD $quoted_prompt" >> "$LOG_FILE" 2>&1; then
run_finished_at="$(date -Is)"
echo "[OK] [$run_finished_at] docs sweep complete (agent_id=$AGENT_ID)"
echo "[$run_finished_at] docs sweep complete (agent_id=$AGENT_ID)" >> "$LOG_FILE"
else
run_failed_at="$(date -Is)"
exit_code=$?
job_status=$exit_code
echo "[FAIL] [$run_failed_at] docs sweep failed (exit $exit_code, agent_id=$AGENT_ID)"
echo "[$run_failed_at] docs sweep failed (exit $exit_code, agent_id=$AGENT_ID)" >> "$LOG_FILE"
fi
exit "$job_status"

192
scripts/docs-sweep-watch.sh Executable file
View File

@@ -0,0 +1,192 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RUN_ONCE_SCRIPT="$SCRIPT_DIR/docs-sweep-once.sh"
INTERVAL_SECONDS="${INTERVAL_SECONDS:-300}"
REPO="${REPO:-$HOME/projects/japanese/SubMiner}"
LOG_FILE="${LOG_FILE:-$REPO/.codex-doc-sweep.log}"
SUBAGENT_ROOT="${SUBAGENT_ROOT:-$REPO/docs/subagents}"
SUBAGENT_INDEX_FILE="${SUBAGENT_INDEX_FILE:-$SUBAGENT_ROOT/INDEX.md}"
SUBAGENT_COLLAB_FILE="${SUBAGENT_COLLAB_FILE:-$SUBAGENT_ROOT/collaboration.md}"
SUBAGENT_AGENTS_DIR="${SUBAGENT_AGENTS_DIR:-$SUBAGENT_ROOT/agents}"
AGENT_ID="${AGENT_ID:-docs-sweep}"
AGENT_ID_SAFE="$(printf '%s' "$AGENT_ID" | tr -c 'A-Za-z0-9._-' '_')"
AGENT_FILE="${AGENT_FILE:-$SUBAGENT_AGENTS_DIR/${AGENT_ID_SAFE}.md}"
REPORT_WITH_CODEX=false
REPORT_TIMEOUT_SECONDS="${REPORT_TIMEOUT_SECONDS:-120}"
REPORT_AGENT_CMD="${REPORT_AGENT_CMD:-codex exec}"
if [[ ! -x "$RUN_ONCE_SCRIPT" ]]; then
echo "Missing executable: $RUN_ONCE_SCRIPT"
echo "Run: chmod +x scripts/docs-sweep-once.sh"
exit 1
fi
usage() {
cat << 'EOF'
Usage: scripts/docs-sweep-watch.sh [options]
Options:
-r, --report One-off: summarize current log with Codex and exit.
-h, --help Show this help message.
Environment:
AGENT_ID Stable agent id (default: docs-sweep)
AGENT_ALIAS Human label shown in logs/coordination (default: Docs Sweep)
AGENT_MISSION One-line focus for this run
SUBAGENT_ROOT Coordination root (default: docs/subagents)
EOF
}
trim_log_runs() {
# Keep only the last 50 docs-sweep runs in the shared log file.
if [[ ! -f "$LOG_FILE" ]]; then
return
fi
local keep_runs=50
local start_line
start_line="$(
awk -v max="$keep_runs" '
/state changed; starting docs sweep/ { lines[++count] = NR }
END {
if (count > max) print lines[count - max + 1]
else print 0
}
' "$LOG_FILE"
)"
if [[ "$start_line" =~ ^[0-9]+$ ]] && (( start_line > 0 )); then
local tmp_file
tmp_file="$(mktemp "${LOG_FILE}.XXXXXX")"
tail -n +"$start_line" "$LOG_FILE" > "$tmp_file"
mv "$tmp_file" "$LOG_FILE"
fi
}
run_report() {
local has_log=false
local has_index=false
local has_collab=false
local has_agent_file=false
if [[ -s "$LOG_FILE" ]]; then
has_log=true
fi
if [[ -s "$SUBAGENT_INDEX_FILE" ]]; then
has_index=true
fi
if [[ -s "$SUBAGENT_COLLAB_FILE" ]]; then
has_collab=true
fi
if [[ -s "$AGENT_FILE" ]]; then
has_agent_file=true
fi
if [[ "$has_log" != "true" && "$has_index" != "true" && "$has_collab" != "true" && "$has_agent_file" != "true" ]]; then
echo "[REPORT] no inputs; missing/empty files:"
echo "[REPORT] - $LOG_FILE"
echo "[REPORT] - $SUBAGENT_INDEX_FILE"
echo "[REPORT] - $SUBAGENT_COLLAB_FILE"
echo "[REPORT] - $AGENT_FILE"
return
fi
local report_prompt
read -r -d '' report_prompt << EOF || true
Summarize docs sweep state. Output:
- Changes made (short bullets; file-focused when possible)
- Agent coordination updates from sharded docs/subagents files
- Open questions / uncertainty
- Left undone / follow-up items
Constraints:
- Be concise.
- If uncertain, say uncertain.
Read these files directly if present:
$LOG_FILE
$SUBAGENT_INDEX_FILE
$SUBAGENT_COLLAB_FILE
$AGENT_FILE
EOF
echo "[REPORT] codex summary start"
local report_file
local report_stderr
report_file="$(mktemp /tmp/docs-sweep-report.XXXXXX)"
report_stderr="$(mktemp /tmp/docs-sweep-report-stderr.XXXXXX)"
(
cd "$REPO"
timeout "${REPORT_TIMEOUT_SECONDS}s" bash -lc "$REPORT_AGENT_CMD -o $(printf '%q' "$report_file") $(printf '%q' "$report_prompt")" > /dev/null 2> "$report_stderr"
)
local report_exit=$?
if (( report_exit != 0 )); then
echo "[REPORT] codex summary failed (exit $report_exit)"
cat "$report_stderr"
echo
echo "[REPORT] codex summary end"
return
fi
if [[ -s "$report_file" ]]; then
cat "$report_file"
else
echo "[REPORT] codex produced no final message"
fi
echo
echo "[REPORT] codex summary end"
}
while (( $# > 0 )); do
case "$1" in
-r|--report)
REPORT_WITH_CODEX=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown option: $1"
usage
exit 1
;;
esac
done
if [[ "$REPORT_WITH_CODEX" == "true" ]]; then
trim_log_runs
run_report
exit 0
fi
stop_requested=false
trap 'stop_requested=true' INT TERM
echo "Starting docs sweep watcher (interval: ${INTERVAL_SECONDS}s, subagent_root: ${SUBAGENT_ROOT}). Press Ctrl+C to stop."
while true; do
run_started_at="$(date -Is)"
echo "[RUN] [$run_started_at] docs sweep cycle running"
if "$RUN_ONCE_SCRIPT"; then
run_finished_at="$(date -Is)"
echo "[OK] [$run_finished_at] docs sweep cycle complete"
else
run_failed_at="$(date -Is)"
exit_code=$?
echo "[FAIL] [$run_failed_at] docs sweep cycle failed (exit $exit_code)"
fi
trim_log_runs
if [[ "$stop_requested" == "true" ]]; then
break
fi
sleep "$INTERVAL_SECONDS" &
wait $!
if [[ "$stop_requested" == "true" ]]; then
break
fi
done
echo "Docs sweep watcher stopped."

239
scripts/get-mpv-window-macos.swift Normal file
View File

@@ -0,0 +1,239 @@
#!/usr/bin/env swift
//
// get-mpv-window-macos.swift
// SubMiner - Get mpv window geometry on macOS
//
// This script uses Core Graphics APIs to find mpv windows system-wide.
// It works with both bundled and unbundled mpv installations.
//
// Usage: swift get-mpv-window-macos.swift
// Output: "x,y,width,height" or "not-found"
//
import Cocoa
import Foundation
private struct WindowGeometry {
let x: Int
let y: Int
let width: Int
let height: Int
}
private let targetMpvSocketPath: String? = {
guard CommandLine.arguments.count > 1 else {
return nil
}
let value = CommandLine.arguments[1].trimmingCharacters(in: .whitespacesAndNewlines)
return value.isEmpty ? nil : value
}()
private func windowCommandLineForPid(_ pid: pid_t) -> String? {
let process = Process()
process.executableURL = URL(fileURLWithPath: "/bin/ps")
process.arguments = ["-p", "\(pid)", "-o", "args="]
let pipe = Pipe()
process.standardOutput = pipe
process.standardError = pipe
do {
try process.run()
process.waitUntilExit()
} catch {
return nil
}
if process.terminationStatus != 0 {
return nil
}
let data = pipe.fileHandleForReading.readDataToEndOfFile()
guard let commandLine = String(data: data, encoding: .utf8) else {
return nil
}
return commandLine
.trimmingCharacters(in: .whitespacesAndNewlines)
.replacingOccurrences(of: "\n", with: " ")
}
private func windowHasTargetSocket(_ pid: pid_t) -> Bool {
guard let socketPath = targetMpvSocketPath else {
return true
}
guard let commandLine = windowCommandLineForPid(pid) else {
return false
}
return commandLine.contains("--input-ipc-server=\(socketPath)") ||
commandLine.contains("--input-ipc-server \(socketPath)")
}
private func geometryFromRect(x: CGFloat, y: CGFloat, width: CGFloat, height: CGFloat) -> WindowGeometry {
let minX = Int(floor(x))
let minY = Int(floor(y))
let maxX = Int(ceil(x + width))
let maxY = Int(ceil(y + height))
return WindowGeometry(
x: minX,
y: minY,
width: max(0, maxX - minX),
height: max(0, maxY - minY)
)
}
private func normalizedMpvName(_ name: String) -> Bool {
let normalized = name.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
return normalized == "mpv"
}
private func geometryFromAXWindow(_ axWindow: AXUIElement) -> WindowGeometry? {
var positionRef: CFTypeRef?
var sizeRef: CFTypeRef?
let positionStatus = AXUIElementCopyAttributeValue(axWindow, kAXPositionAttribute as CFString, &positionRef)
let sizeStatus = AXUIElementCopyAttributeValue(axWindow, kAXSizeAttribute as CFString, &sizeRef)
guard positionStatus == .success,
sizeStatus == .success,
let positionRaw = positionRef,
let sizeRaw = sizeRef,
CFGetTypeID(positionRaw) == AXValueGetTypeID(),
CFGetTypeID(sizeRaw) == AXValueGetTypeID() else {
return nil
}
let positionValue = positionRaw as! AXValue
let sizeValue = sizeRaw as! AXValue
guard AXValueGetType(positionValue) == .cgPoint,
AXValueGetType(sizeValue) == .cgSize else {
return nil
}
var position = CGPoint.zero
var size = CGSize.zero
guard AXValueGetValue(positionValue, .cgPoint, &position),
AXValueGetValue(sizeValue, .cgSize, &size) else {
return nil
}
let geometry = geometryFromRect(
x: position.x,
y: position.y,
width: size.width,
height: size.height
)
guard geometry.width >= 100, geometry.height >= 100 else {
return nil
}
return geometry
}
private func geometryFromAccessibilityAPI() -> WindowGeometry? {
let runningApps = NSWorkspace.shared.runningApplications.filter { app in
guard let name = app.localizedName else {
return false
}
return normalizedMpvName(name)
}
for app in runningApps {
let appElement = AXUIElementCreateApplication(app.processIdentifier)
if !windowHasTargetSocket(app.processIdentifier) {
continue
}
var windowsRef: CFTypeRef?
let status = AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &windowsRef)
guard status == .success, let windows = windowsRef as? [AXUIElement], !windows.isEmpty else {
continue
}
for window in windows {
var minimizedRef: CFTypeRef?
let minimizedStatus = AXUIElementCopyAttributeValue(window, kAXMinimizedAttribute as CFString, &minimizedRef)
if minimizedStatus == .success, let minimized = minimizedRef as? Bool, minimized {
continue
}
var windowPid: pid_t = 0
if AXUIElementGetPid(window, &windowPid) != .success {
continue
}
if !windowHasTargetSocket(windowPid) {
continue
}
if let geometry = geometryFromAXWindow(window) {
return geometry
}
}
}
return nil
}
private func geometryFromCoreGraphics() -> WindowGeometry? {
// Keep the CG fallback for environments without Accessibility permissions.
// Use on-screen layer-0 windows to avoid off-screen helpers/shadows.
let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements]
let windowList = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] ?? []
for window in windowList {
guard let ownerName = window[kCGWindowOwnerName as String] as? String,
normalizedMpvName(ownerName) else {
continue
}
guard let ownerPid = window[kCGWindowOwnerPID as String] as? pid_t else {
continue
}
if !windowHasTargetSocket(ownerPid) {
continue
}
if let layer = window[kCGWindowLayer as String] as? Int, layer != 0 {
continue
}
if let alpha = window[kCGWindowAlpha as String] as? Double, alpha <= 0.01 {
continue
}
if let onScreen = window[kCGWindowIsOnscreen as String] as? Int, onScreen == 0 {
continue
}
guard let bounds = window[kCGWindowBounds as String] as? [String: CGFloat] else {
continue
}
let geometry = geometryFromRect(
x: bounds["X"] ?? 0,
y: bounds["Y"] ?? 0,
width: bounds["Width"] ?? 0,
height: bounds["Height"] ?? 0
)
guard geometry.width >= 100, geometry.height >= 100 else {
continue
}
return geometry
}
return nil
}
if let window = geometryFromAccessibilityAPI() ?? geometryFromCoreGraphics() {
print("\(window.x),\(window.y),\(window.width),\(window.height)")
} else {
print("not-found")
}

903
scripts/get_frequency.ts Normal file
View File

@@ -0,0 +1,903 @@
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
import { createFrequencyDictionaryLookup } from '../src/core/services/frequency-dictionary.js';
import { MecabTokenizer } from '../src/mecab-tokenizer.js';
import type { MergedToken, FrequencyDictionaryLookup } from '../src/types.js';
interface CliOptions {
input: string;
dictionaryPath: string;
emitPretty: boolean;
emitDiagnostics: boolean;
mecabCommand?: string;
mecabDictionaryPath?: string;
forceMecabOnly?: boolean;
yomitanExtensionPath?: string;
yomitanUserDataPath?: string;
emitColoredLine: boolean;
colorMode: 'single' | 'banded';
colorTopX: number;
colorSingle: string;
colorBand1: string;
colorBand2: string;
colorBand3: string;
colorBand4: string;
colorBand5: string;
colorKnown: string;
colorNPlusOne: string;
}
function parseCliArgs(argv: string[]): CliOptions {
const args = [...argv];
let inputParts: string[] = [];
let dictionaryPath = path.join(process.cwd(), 'vendor', 'jiten_freq_global');
let emitPretty = false;
let emitDiagnostics = false;
let mecabCommand: string | undefined;
let mecabDictionaryPath: string | undefined;
let forceMecabOnly = false;
let yomitanExtensionPath: string | undefined;
let yomitanUserDataPath: string | undefined;
let emitColoredLine = false;
let colorMode: 'single' | 'banded' = 'single';
let colorTopX = 1000;
let colorSingle = '#f5a97f';
let colorBand1 = '#ed8796';
let colorBand2 = '#f5a97f';
let colorBand3 = '#f9e2af';
let colorBand4 = '#a6e3a1';
let colorBand5 = '#8aadf4';
let colorKnown = '#a6da95';
let colorNPlusOne = '#c6a0f6';
while (args.length > 0) {
const arg = args.shift();
if (!arg) break;
if (arg === '--help' || arg === '-h') {
printUsage();
process.exit(0);
}
if (arg === '--dictionary') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --dictionary');
}
dictionaryPath = path.resolve(next);
continue;
}
if (arg === '--mecab-command') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --mecab-command');
}
mecabCommand = next;
continue;
}
if (arg === '--mecab-dictionary') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --mecab-dictionary');
}
mecabDictionaryPath = next;
continue;
}
if (arg === '--yomitan-extension') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --yomitan-extension');
}
yomitanExtensionPath = path.resolve(next);
continue;
}
if (arg === '--yomitan-user-data') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --yomitan-user-data');
}
yomitanUserDataPath = path.resolve(next);
continue;
}
if (arg === '--colorized-line') {
emitColoredLine = true;
continue;
}
if (arg === '--color-mode') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-mode');
}
if (next !== 'single' && next !== 'banded') {
throw new Error("--color-mode must be 'single' or 'banded'");
}
colorMode = next;
continue;
}
if (arg === '--color-top-x') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-top-x');
}
const parsed = Number.parseInt(next, 10);
if (!Number.isFinite(parsed) || parsed <= 0) {
throw new Error('--color-top-x must be a positive integer');
}
colorTopX = parsed;
continue;
}
if (arg === '--color-single') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-single');
}
colorSingle = next;
continue;
}
if (arg === '--color-band-1') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-band-1');
}
colorBand1 = next;
continue;
}
if (arg === '--color-band-2') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-band-2');
}
colorBand2 = next;
continue;
}
if (arg === '--color-band-3') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-band-3');
}
colorBand3 = next;
continue;
}
if (arg === '--color-band-4') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-band-4');
}
colorBand4 = next;
continue;
}
if (arg === '--color-band-5') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-band-5');
}
colorBand5 = next;
continue;
}
if (arg === '--color-known') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-known');
}
colorKnown = next;
continue;
}
if (arg === '--color-n-plus-one') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --color-n-plus-one');
}
colorNPlusOne = next;
continue;
}
if (arg.startsWith('--dictionary=')) {
dictionaryPath = path.resolve(arg.slice('--dictionary='.length));
continue;
}
if (arg.startsWith('--mecab-command=')) {
mecabCommand = arg.slice('--mecab-command='.length);
continue;
}
if (arg.startsWith('--mecab-dictionary=')) {
mecabDictionaryPath = arg.slice('--mecab-dictionary='.length);
continue;
}
if (arg.startsWith('--yomitan-extension=')) {
yomitanExtensionPath = path.resolve(arg.slice('--yomitan-extension='.length));
continue;
}
if (arg.startsWith('--yomitan-user-data=')) {
yomitanUserDataPath = path.resolve(arg.slice('--yomitan-user-data='.length));
continue;
}
if (arg.startsWith('--colorized-line')) {
emitColoredLine = true;
continue;
}
if (arg.startsWith('--color-mode=')) {
const value = arg.slice('--color-mode='.length);
if (value !== 'single' && value !== 'banded') {
throw new Error("--color-mode must be 'single' or 'banded'");
}
colorMode = value;
continue;
}
if (arg.startsWith('--color-top-x=')) {
const value = arg.slice('--color-top-x='.length);
const parsed = Number.parseInt(value, 10);
if (!Number.isFinite(parsed) || parsed <= 0) {
throw new Error('--color-top-x must be a positive integer');
}
colorTopX = parsed;
continue;
}
if (arg.startsWith('--color-single=')) {
colorSingle = arg.slice('--color-single='.length);
continue;
}
if (arg.startsWith('--color-band-1=')) {
colorBand1 = arg.slice('--color-band-1='.length);
continue;
}
if (arg.startsWith('--color-band-2=')) {
colorBand2 = arg.slice('--color-band-2='.length);
continue;
}
if (arg.startsWith('--color-band-3=')) {
colorBand3 = arg.slice('--color-band-3='.length);
continue;
}
if (arg.startsWith('--color-band-4=')) {
colorBand4 = arg.slice('--color-band-4='.length);
continue;
}
if (arg.startsWith('--color-band-5=')) {
colorBand5 = arg.slice('--color-band-5='.length);
continue;
}
if (arg.startsWith('--color-known=')) {
colorKnown = arg.slice('--color-known='.length);
continue;
}
if (arg.startsWith('--color-n-plus-one=')) {
colorNPlusOne = arg.slice('--color-n-plus-one='.length);
continue;
}
if (arg === '--pretty') {
emitPretty = true;
continue;
}
if (arg === '--diagnostics') {
emitDiagnostics = true;
continue;
}
if (arg === '--force-mecab') {
forceMecabOnly = true;
continue;
}
if (arg.startsWith('-')) {
throw new Error(`Unknown flag: ${arg}`);
}
inputParts.push(arg);
}
const input = inputParts.join(' ').trim();
if (!input) {
const stdin = fs.readFileSync(0, 'utf8').trim();
if (!stdin) {
throw new Error('Please provide input text as arguments or via stdin.');
}
return {
input: stdin,
dictionaryPath,
emitPretty,
emitDiagnostics,
forceMecabOnly,
yomitanExtensionPath,
yomitanUserDataPath,
emitColoredLine,
colorMode,
colorTopX,
colorSingle,
colorBand1,
colorBand2,
colorBand3,
colorBand4,
colorBand5,
colorKnown,
colorNPlusOne,
mecabCommand,
mecabDictionaryPath,
};
}
return {
input,
dictionaryPath,
emitPretty,
emitDiagnostics,
forceMecabOnly,
yomitanExtensionPath,
yomitanUserDataPath,
emitColoredLine,
colorMode,
colorTopX,
colorSingle,
colorBand1,
colorBand2,
colorBand3,
colorBand4,
colorBand5,
colorKnown,
colorNPlusOne,
mecabCommand,
mecabDictionaryPath,
};
}
function printUsage(): void {
process.stdout.write(`Usage:
bun run get-frequency [--pretty] [--diagnostics] [--dictionary <path>] [--mecab-command <path>] [--mecab-dictionary <path>] <text>
--pretty Pretty-print JSON output.
--diagnostics Include merged-frequency lookup-term details.
--force-mecab Skip Yomitan parser initialization and force MeCab fallback.
--yomitan-extension <path> Optional path to a Yomitan extension directory.
--yomitan-user-data <path> Optional Electron userData directory for Yomitan state.
--colorized-line Output a terminal-colorized line based on token classification.
--color-mode <single|banded> Frequency coloring mode (default: single).
--color-top-x <n> Frequency color applies when rank <= n (default: 1000).
--color-single <#hex> Frequency single-mode color (default: #f5a97f).
--color-band-1 <#hex> Frequency band-1 color.
--color-band-2 <#hex> Frequency band-2 color.
--color-band-3 <#hex> Frequency band-3 color.
--color-band-4 <#hex> Frequency band-4 color.
--color-band-5 <#hex> Frequency band-5 color.
--color-known <#hex> Known-word color (default: #a6da95).
--color-n-plus-one <#hex> N+1 target color (default: #c6a0f6).
--dictionary <path> Frequency dictionary root path (default: ./vendor/jiten_freq_global)
--mecab-command <path> Optional MeCab binary path (default: mecab)
--mecab-dictionary <path> Optional MeCab dictionary directory (default: system default)
-h, --help Show usage.
\n`);
}
type FrequencyCandidate = {
term: string;
rank: number;
};
function getFrequencyLookupTextCandidates(token: MergedToken): string[] {
const lookupText = token.headword?.trim() || token.reading?.trim() || token.surface.trim();
return lookupText ? [lookupText] : [];
}
function getBestFrequencyLookupCandidate(
token: MergedToken,
getFrequencyRank: FrequencyDictionaryLookup,
): FrequencyCandidate | null {
const lookupTexts = getFrequencyLookupTextCandidates(token);
let best: FrequencyCandidate | null = null;
for (const term of lookupTexts) {
const rank = getFrequencyRank(term);
if (typeof rank !== 'number' || !Number.isFinite(rank) || rank <= 0) {
continue;
}
if (!best || rank < best.rank) {
best = { term, rank };
}
}
return best;
}
function simplifyToken(token: MergedToken): Record<string, unknown> {
return {
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
partOfSpeech: token.partOfSpeech,
isMerged: token.isMerged,
isKnown: token.isKnown,
isNPlusOneTarget: token.isNPlusOneTarget,
frequencyRank: token.frequencyRank,
jlptLevel: token.jlptLevel,
};
}
function simplifyTokenWithVerbose(
token: MergedToken,
getFrequencyRank: FrequencyDictionaryLookup,
): Record<string, unknown> {
const candidates = getFrequencyLookupTextCandidates(token)
.map((term) => ({
term,
rank: getFrequencyRank(term),
}))
.filter(
(candidate) =>
typeof candidate.rank === 'number' && Number.isFinite(candidate.rank) && candidate.rank > 0,
);
const bestCandidate = getBestFrequencyLookupCandidate(token, getFrequencyRank);
return {
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
partOfSpeech: token.partOfSpeech,
isMerged: token.isMerged,
isKnown: token.isKnown,
isNPlusOneTarget: token.isNPlusOneTarget,
frequencyRank: token.frequencyRank,
jlptLevel: token.jlptLevel,
frequencyCandidates: candidates,
frequencyBestLookupTerm: bestCandidate?.term ?? null,
frequencyBestLookupRank: bestCandidate?.rank ?? null,
};
}
interface YomitanRuntimeState {
yomitanExt: unknown | null;
parserWindow: unknown | null;
parserReadyPromise: Promise<void> | null;
parserInitPromise: Promise<boolean> | null;
available: boolean;
note?: string;
}
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
promise
.then((value) => {
clearTimeout(timer);
resolve(value);
})
.catch((error) => {
clearTimeout(timer);
reject(error);
});
});
}
function destroyUnknownParserWindow(window: unknown): void {
if (!window || typeof window !== 'object') {
return;
}
const candidate = window as {
isDestroyed?: () => boolean;
destroy?: () => void;
};
if (typeof candidate.isDestroyed !== 'function') {
return;
}
if (typeof candidate.destroy !== 'function') {
return;
}
if (!candidate.isDestroyed()) {
candidate.destroy();
}
}
async function createYomitanRuntimeState(userDataPath: string): Promise<YomitanRuntimeState> {
const state: YomitanRuntimeState = {
yomitanExt: null,
parserWindow: null,
parserReadyPromise: null,
parserInitPromise: null,
available: false,
};
const electronImport = await import('electron').catch((error) => {
state.note = error instanceof Error ? error.message : 'unknown error';
return null;
});
if (!electronImport || !electronImport.app || !electronImport.app.whenReady) {
state.note = 'electron runtime not available in this process';
return state;
}
try {
await electronImport.app.whenReady();
const loadYomitanExtension = (await import('../src/core/services/yomitan-extension-loader.js'))
.loadYomitanExtension as (options: {
userDataPath: string;
getYomitanParserWindow: () => unknown;
setYomitanParserWindow: (window: unknown) => void;
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
setYomitanExtension: (extension: unknown) => void;
}) => Promise<unknown>;
const extension = await loadYomitanExtension({
userDataPath,
getYomitanParserWindow: () => state.parserWindow,
setYomitanParserWindow: (window) => {
state.parserWindow = window;
},
setYomitanParserReadyPromise: (promise) => {
state.parserReadyPromise = promise;
},
setYomitanParserInitPromise: (promise) => {
state.parserInitPromise = promise;
},
setYomitanExtension: (extension) => {
state.yomitanExt = extension;
},
});
if (!extension) {
state.note = 'yomitan extension is not available';
return state;
}
state.yomitanExt = extension;
state.available = true;
return state;
} catch (error) {
state.note = error instanceof Error ? error.message : 'failed to initialize yomitan extension';
return state;
}
}
async function createYomitanRuntimeStateWithSearch(
userDataPath: string,
extensionPath?: string,
): Promise<YomitanRuntimeState> {
const preferredPath = extensionPath ? path.resolve(extensionPath) : undefined;
const defaultVendorPath = path.resolve(process.cwd(), 'vendor', 'yomitan');
const candidates = [...(preferredPath ? [preferredPath] : []), defaultVendorPath];
for (const candidate of candidates) {
if (!candidate) {
continue;
}
try {
if (fs.existsSync(path.join(candidate, 'manifest.json'))) {
const state = await createYomitanRuntimeState(userDataPath);
if (state.available) {
return state;
}
if (!state.note) {
state.note = `Failed to load yomitan extension at ${candidate}`;
}
return state;
}
} catch {
continue;
}
}
return createYomitanRuntimeState(userDataPath);
}
async function getFrequencyLookup(dictionaryPath: string): Promise<FrequencyDictionaryLookup> {
return createFrequencyDictionaryLookup({
searchPaths: [dictionaryPath],
log: (message) => {
// Keep script output pure JSON by default
if (process.env.DEBUG_FREQUENCY === '1') {
console.error(message);
}
},
});
}
const ANSI_RESET = '\u001b[0m';
const ANSI_FG_PREFIX = '\u001b[38;2';
const HEX_COLOR_PATTERN = /^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/;
function parseHexRgb(input: string): [number, number, number] | null {
const normalized = input.trim().replace(/^#/, '');
if (!HEX_COLOR_PATTERN.test(`#${normalized}`)) {
return null;
}
const expanded =
normalized.length === 3
? normalized
.split('')
.map((char) => `${char}${char}`)
.join('')
: normalized;
const r = Number.parseInt(expanded.substring(0, 2), 16);
const g = Number.parseInt(expanded.substring(2, 4), 16);
const b = Number.parseInt(expanded.substring(4, 6), 16);
if (!Number.isFinite(r) || !Number.isFinite(g) || !Number.isFinite(b)) {
return null;
}
return [r, g, b];
}
function wrapWithForeground(text: string, color: string): string {
const rgb = parseHexRgb(color);
if (!rgb) {
return text;
}
return `${ANSI_FG_PREFIX};${rgb[0]};${rgb[1]};${rgb[2]}m${text}${ANSI_RESET}`;
}
function getBandColor(
rank: number,
colorTopX: number,
colorMode: 'single' | 'banded',
colorSingle: string,
bandedColors: [string, string, string, string, string],
): string {
const topX = Math.max(1, Math.floor(colorTopX));
const safeRank = Math.max(1, Math.floor(rank));
if (safeRank > topX) {
return '';
}
if (colorMode === 'single') {
return colorSingle;
}
const normalizedBand = Math.ceil((safeRank / topX) * bandedColors.length);
const band = Math.min(bandedColors.length, Math.max(1, normalizedBand));
return bandedColors[band - 1];
}
function getTokenColor(token: MergedToken, args: CliOptions): string {
if (token.isNPlusOneTarget) {
return args.colorNPlusOne;
}
if (token.isKnown) {
return args.colorKnown;
}
if (typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)) {
return getBandColor(token.frequencyRank, args.colorTopX, args.colorMode, args.colorSingle, [
args.colorBand1,
args.colorBand2,
args.colorBand3,
args.colorBand4,
args.colorBand5,
]);
}
return '';
}
function renderColoredLine(text: string, tokens: MergedToken[], args: CliOptions): string {
if (!args.emitColoredLine) {
return text;
}
if (tokens.length === 0) {
return text;
}
const ordered = [...tokens].sort((a, b) => {
const aStart = a.startPos ?? 0;
const bStart = b.startPos ?? 0;
if (aStart !== bStart) {
return aStart - bStart;
}
return (a.endPos ?? a.surface.length) - (b.endPos ?? b.surface.length);
});
let cursor = 0;
let output = '';
for (const token of ordered) {
const start = token.startPos ?? 0;
const end =
token.endPos ??
(token.startPos ? token.startPos + token.surface.length : token.surface.length);
if (start < 0 || end < 0 || end < start) {
continue;
}
const safeStart = Math.min(Math.max(0, start), text.length);
const safeEnd = Math.min(Math.max(safeStart, end), text.length);
if (safeStart > cursor) {
output += text.slice(cursor, safeStart);
}
const tokenText = text.slice(safeStart, safeEnd);
const color = getTokenColor(token, args);
output += color ? wrapWithForeground(tokenText, color) : tokenText;
cursor = safeEnd;
}
if (cursor < text.length) {
output += text.slice(cursor);
}
return output;
}
async function main(): Promise<void> {
let electronModule: typeof import('electron') | null = null;
let yomitanState: YomitanRuntimeState | null = null;
try {
const args = parseCliArgs(process.argv.slice(2));
const getFrequencyRank = await getFrequencyLookup(args.dictionaryPath);
const mecabTokenizer = new MecabTokenizer({
mecabCommand: args.mecabCommand,
dictionaryPath: args.mecabDictionaryPath,
});
const isMecabAvailable = await mecabTokenizer.checkAvailability();
if (!isMecabAvailable) {
throw new Error(
'MeCab is not available on this system. Install/run environment with MeCab to tokenize input.',
);
}
electronModule = await import('electron').catch(() => null);
if (electronModule && args.yomitanUserDataPath) {
electronModule.app.setPath('userData', args.yomitanUserDataPath);
}
yomitanState = !args.forceMecabOnly
? await createYomitanRuntimeStateWithSearch(
electronModule?.app?.getPath ? electronModule.app.getPath('userData') : process.cwd(),
args.yomitanExtensionPath,
)
: null;
const hasYomitan = Boolean(yomitanState?.available && yomitanState?.yomitanExt);
let useYomitan = hasYomitan;
const deps = createTokenizerDepsRuntime({
getYomitanExt: () => (useYomitan ? yomitanState!.yomitanExt : null) as never,
getYomitanParserWindow: () => (useYomitan ? yomitanState!.parserWindow : null) as never,
setYomitanParserWindow: (window) => {
if (!useYomitan) {
return;
}
yomitanState!.parserWindow = window;
},
getYomitanParserReadyPromise: () =>
(useYomitan ? yomitanState!.parserReadyPromise : null) as never,
setYomitanParserReadyPromise: (promise) => {
if (!useYomitan) {
return;
}
yomitanState!.parserReadyPromise = promise;
},
getYomitanParserInitPromise: () =>
(useYomitan ? yomitanState!.parserInitPromise : null) as never,
setYomitanParserInitPromise: (promise) => {
if (!useYomitan) {
return;
}
yomitanState!.parserInitPromise = promise;
},
isKnownWord: () => false,
getKnownWordMatchMode: () => 'headword',
getJlptLevel: () => null,
getFrequencyDictionaryEnabled: () => true,
getFrequencyRank,
getMecabTokenizer: () => ({
tokenize: (text: string) => mecabTokenizer.tokenize(text),
}),
});
let subtitleData;
if (useYomitan) {
try {
subtitleData = await withTimeout(
tokenizeSubtitle(args.input, deps),
8000,
'Yomitan tokenizer',
);
} catch (error) {
useYomitan = false;
destroyUnknownParserWindow(yomitanState?.parserWindow ?? null);
if (yomitanState) {
yomitanState.parserWindow = null;
yomitanState.parserReadyPromise = null;
yomitanState.parserInitPromise = null;
const fallbackNote =
error instanceof Error ? error.message : 'Yomitan tokenizer timed out';
yomitanState.note = yomitanState.note
? `${yomitanState.note}; ${fallbackNote}`
: fallbackNote;
}
subtitleData = await tokenizeSubtitle(args.input, deps);
}
} else {
subtitleData = await tokenizeSubtitle(args.input, deps);
}
const tokenCount = subtitleData.tokens?.length ?? 0;
const mergedCount = subtitleData.tokens?.filter((token) => token.isMerged).length ?? 0;
const tokens =
subtitleData.tokens?.map((token) =>
args.emitDiagnostics
? simplifyTokenWithVerbose(token, getFrequencyRank)
: simplifyToken(token),
) ?? null;
const diagnostics = {
yomitan: {
available: Boolean(yomitanState?.available),
loaded: useYomitan,
forceMecabOnly: args.forceMecabOnly,
note: yomitanState?.note ?? null,
},
mecab: {
command: args.mecabCommand ?? 'mecab',
dictionaryPath: args.mecabDictionaryPath ?? null,
available: isMecabAvailable,
},
tokenizer: {
sourceHint: tokenCount === 0 ? 'none' : useYomitan ? 'yomitan-merged' : 'mecab-merge',
mergedTokenCount: mergedCount,
totalTokenCount: tokenCount,
},
};
if (tokens === null) {
diagnostics.mecab['status'] = 'no-tokens';
diagnostics.mecab['note'] =
'MeCab returned no parseable tokens. This is often caused by a missing/invalid MeCab dictionary path.';
} else {
diagnostics.mecab['status'] = 'ok';
}
const output = {
input: args.input,
tokenizerText: subtitleData.text,
tokens,
diagnostics,
};
const json = JSON.stringify(output, null, args.emitPretty ? 2 : undefined);
process.stdout.write(`${json}\n`);
if (args.emitColoredLine && subtitleData.tokens) {
const coloredLine = renderColoredLine(subtitleData.text, subtitleData.tokens, args);
process.stdout.write(`${coloredLine}\n`);
}
} finally {
destroyUnknownParserWindow(yomitanState?.parserWindow ?? null);
if (electronModule?.app) {
electronModule.app.quit();
}
}
}
main()
.then(() => {
process.exit(0);
})
.catch((error) => {
console.error(`Error: ${(error as Error).message}`);
process.exit(1);
});

168
scripts/mkv-to-readme-video.sh Executable file
View File

@@ -0,0 +1,168 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat << 'USAGE'
Usage:
scripts/mkv-to-readme-video.sh [--force] <input.mkv>
Description:
Generates two browser-friendly files next to the input file:
- <name>.mp4 (H.264 + AAC, prefers NVIDIA GPU if available)
- <name>.webm (AV1/VP9 + Opus, prefers NVIDIA GPU if available)
- <name>.gif (palette-optimised, 15 fps)
Options:
-f, --force Overwrite existing output files
Encoding profile:
- Crop: 1920x1080 at x=760 y=180
- MP4: H.264 + AAC
- WebM: AV1/VP9 + Opus at 30 fps
USAGE
}
force=0
input=""
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help)
usage
exit 0
;;
-f|--force)
force=1
;;
-*)
echo "Error: unknown option: $1" >&2
usage
exit 1
;;
*)
if [[ -n "$input" ]]; then
echo "Error: expected exactly one input file." >&2
usage
exit 1
fi
input="$1"
;;
esac
shift
done
if [[ -z "$input" ]]; then
usage
exit 1
fi
if ! command -v ffmpeg > /dev/null 2>&1; then
echo "Error: ffmpeg is not installed or not in PATH." >&2
exit 1
fi
if [[ ! -f "$input" ]]; then
echo "Error: input file not found: $input" >&2
exit 1
fi
dir="$(dirname "$input")"
filename="$(basename "$input")"
base="${filename%.*}"
mp4_out="$dir/$base.mp4"
webm_out="$dir/$base.webm"
gif_out="$dir/$base.gif"
overwrite_flag="-n"
if [[ "$force" -eq 1 ]]; then
overwrite_flag="-y"
fi
if [[ "$force" -eq 0 ]]; then
for output in "$mp4_out" "$webm_out" "$gif_out"; do
if [[ -e "$output" ]]; then
echo "Error: output exists: $output (use --force to overwrite)" >&2
exit 1
fi
done
fi
has_encoder() {
local encoder="$1"
ffmpeg -hide_banner -encoders 2> /dev/null | grep -qE "[[:space:]]${encoder}[[:space:]]"
}
crop_vf="crop=1920:1080:760:180"
webm_vf="${crop_vf},fps=30"
gif_vf="${crop_vf},fps=15,scale=960:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=128[p];[s1][p]paletteuse=dither=bayer:bayer_scale=3"
echo "Generating MP4: $mp4_out"
if has_encoder "h264_nvenc"; then
echo "Trying GPU encoder for MP4: h264_nvenc"
if ffmpeg "$overwrite_flag" -i "$input" \
-vf "$crop_vf" \
-c:v h264_nvenc -preset p6 -rc:v vbr -cq:v 20 -b:v 0 \
-pix_fmt yuv420p -movflags +faststart \
-c:a aac -b:a 160k \
"$mp4_out"; then
:
else
echo "GPU MP4 encode failed; retrying with CPU encoder: libx264"
ffmpeg "$overwrite_flag" -i "$input" \
-vf "$crop_vf" \
-c:v libx264 -preset slow -crf 20 \
-profile:v high -level 4.1 -pix_fmt yuv420p \
-movflags +faststart \
-c:a aac -b:a 160k \
"$mp4_out"
fi
else
echo "Using CPU encoder for MP4: libx264"
ffmpeg "$overwrite_flag" -i "$input" \
-vf "$crop_vf" \
-c:v libx264 -preset slow -crf 20 \
-profile:v high -level 4.1 -pix_fmt yuv420p \
-movflags +faststart \
-c:a aac -b:a 160k \
"$mp4_out"
fi
echo "Generating WebM: $webm_out"
if has_encoder "av1_nvenc"; then
echo "Trying GPU encoder for WebM: av1_nvenc"
if ffmpeg "$overwrite_flag" -i "$input" \
-vf "$webm_vf" \
-c:v av1_nvenc -preset p6 -cq:v 34 -b:v 0 \
-c:a libopus -b:a 96k \
"$webm_out"; then
:
else
echo "GPU WebM encode failed; retrying with CPU encoder: libvpx-vp9"
ffmpeg "$overwrite_flag" -i "$input" \
-vf "$webm_vf" \
-c:v libvpx-vp9 -crf 34 -b:v 0 \
-row-mt 1 -threads 8 \
-c:a libopus -b:a 96k \
"$webm_out"
fi
else
echo "Using CPU encoder for WebM: libvpx-vp9"
ffmpeg "$overwrite_flag" -i "$input" \
-vf "$webm_vf" \
-c:v libvpx-vp9 -crf 34 -b:v 0 \
-row-mt 1 -threads 8 \
-c:a libopus -b:a 96k \
"$webm_out"
fi
echo "Generating GIF: $gif_out"
ffmpeg "$overwrite_flag" -i "$input" \
-vf "$gif_vf" \
"$gif_out"
echo "Done."
echo "MP4: $mp4_out"
echo "WebM: $webm_out"
echo "GIF: $gif_out"

236
scripts/patch-texthooker.sh Executable file
View File

@@ -0,0 +1,236 @@
#!/bin/bash
#
# SubMiner - All-in-one sentence mining overlay
# Copyright (C) 2024 sudacode
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# patch-texthooker.sh - Apply patches to texthooker-ui
#
# This script patches texthooker-ui to:
# 1) Handle empty sentences from mpv.
# 2) Apply SubMiner default texthooker styling.
#
# Usage: ./patch-texthooker.sh [texthooker_dir]
# texthooker_dir: Path to the texthooker-ui directory (default: vendor/texthooker-ui)
#
set -e
sed_in_place() {
local script="$1"
local file="$2"
if sed -i '' "$script" "$file" 2>/dev/null; then
return 0
fi
sed -i "$script" "$file"
}
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEXTHOOKER_DIR="${1:-$SCRIPT_DIR/../vendor/texthooker-ui}"
if [ ! -d "$TEXTHOOKER_DIR" ]; then
echo "Error: texthooker-ui directory not found: $TEXTHOOKER_DIR"
exit 1
fi
echo "Patching texthooker-ui in: $TEXTHOOKER_DIR"
SOCKET_TS="$TEXTHOOKER_DIR/src/socket.ts"
APP_CSS="$TEXTHOOKER_DIR/src/app.css"
STORES_TS="$TEXTHOOKER_DIR/src/stores/stores.ts"
if [ ! -f "$SOCKET_TS" ]; then
echo "Error: socket.ts not found at $SOCKET_TS"
exit 1
fi
if [ ! -f "$APP_CSS" ]; then
echo "Error: app.css not found at $APP_CSS"
exit 1
fi
if [ ! -f "$STORES_TS" ]; then
echo "Error: stores.ts not found at $STORES_TS"
exit 1
fi
echo "Patching socket.ts..."
# Patch 1: Change || to ?? (nullish coalescing)
# This ensures empty string is kept instead of falling back to raw JSON
if grep -q '\.sentence ?? event\.data' "$SOCKET_TS"; then
echo " - Nullish coalescing already patched, skipping"
else
sed_in_place 's/\.sentence || event\.data/.sentence ?? event.data/' "$SOCKET_TS"
echo " - Changed || to ?? (nullish coalescing)"
fi
# Patch 2: Skip emitting empty lines
# This prevents empty sentences from being added to the UI
if grep -q "if (line)" "$SOCKET_TS"; then
echo " - Empty line check already patched, skipping"
else
sed_in_place 's/\t\tnewLine\$\.next(\[line, LineType\.SOCKET\]);/\t\tif (line) {\n\t\t\tnewLine$.next([line, LineType.SOCKET]);\n\t\t}/' "$SOCKET_TS"
echo " - Added empty line check"
fi
echo "Patching app.css..."
# Patch 3: Apply SubMiner default texthooker-ui styling
if grep -q "SUBMINER_DEFAULT_STYLE_START" "$APP_CSS"; then
echo " - Default SubMiner CSS already patched, skipping"
else
cat >> "$APP_CSS" << 'EOF'
/* SUBMINER_DEFAULT_STYLE_START */
:root {
--sm-bg: #1a1b2e;
--sm-surface: #222436;
--sm-border: rgba(255, 255, 255, 0.05);
--sm-text: #c8d3f5;
--sm-text-muted: #636da6;
--sm-hover-bg: rgba(130, 170, 255, 0.06);
--sm-scrollbar: rgba(255, 255, 255, 0.08);
--sm-scrollbar-hover: rgba(255, 255, 255, 0.15);
}
html,
body {
margin: 0;
display: flex;
flex: 1;
min-height: 100%;
overflow: auto;
background: var(--sm-bg);
color: var(--sm-text);
}
body[data-theme] {
background: var(--sm-bg);
color: var(--sm-text);
}
main,
header,
#pip-container {
background: transparent;
color: var(--sm-text);
}
header.bg-base-100,
header.bg-base-200 {
background: transparent;
}
main,
main.flex {
font-family: 'Noto Sans CJK JP', 'Hiragino Sans', system-ui, sans-serif;
padding: 0.5rem min(4vw, 2rem);
line-height: 1.7;
gap: 0;
}
p,
p.cursor-pointer {
font-family: 'Noto Sans CJK JP', 'Hiragino Sans', system-ui, sans-serif;
font-size: clamp(18px, 2vw, 26px);
letter-spacing: 0.04em;
line-height: 1.65;
white-space: normal;
margin: 0;
padding: 0.65rem 1rem;
border: none;
border-bottom: 1px solid var(--sm-border);
border-radius: 0;
background: transparent;
transition: background 0.2s ease;
}
p:hover,
p.cursor-pointer:hover {
background: var(--sm-hover-bg);
}
p:last-child,
p.cursor-pointer:last-child {
border-bottom: none;
}
p.cursor-pointer.whitespace-pre-wrap {
white-space: normal;
}
p.cursor-pointer.my-2 {
margin: 0;
}
p.cursor-pointer.py-4,
p.cursor-pointer.py-2,
p.cursor-pointer.px-2,
p.cursor-pointer.px-4 {
padding: 0.65rem 1rem;
}
*::-webkit-scrollbar {
width: 4px;
}
*::-webkit-scrollbar-track {
background: transparent;
}
*::-webkit-scrollbar-thumb {
background: var(--sm-scrollbar);
border-radius: 2px;
}
*::-webkit-scrollbar-thumb:hover {
background: var(--sm-scrollbar-hover);
}
/* SUBMINER_DEFAULT_STYLE_END */
EOF
echo " - Added default SubMiner CSS block"
fi
echo "Patching stores.ts defaults..."
# Patch 4: Change default settings for title/whitespace/animation/reconnect
if grep -q "preserveWhitespace\\$: false" "$STORES_TS" && \
grep -q "removeAllWhitespace\\$: true" "$STORES_TS" && \
grep -q "enableLineAnimation\\$: true" "$STORES_TS" && \
grep -q "continuousReconnect\\$: true" "$STORES_TS" && \
grep -q "windowTitle\\$: 'SubMiner Texthooker'" "$STORES_TS"; then
echo " - Default settings already patched, skipping"
else
sed_in_place "s/windowTitle\\$: '',/windowTitle\\$: 'SubMiner Texthooker',/" "$STORES_TS"
sed_in_place 's/preserveWhitespace\$: true,/preserveWhitespace\$: false,/' "$STORES_TS"
sed_in_place 's/removeAllWhitespace\$: false,/removeAllWhitespace\$: true,/' "$STORES_TS"
sed_in_place 's/enableLineAnimation\$: false,/enableLineAnimation\$: true,/' "$STORES_TS"
sed_in_place 's/continuousReconnect\$: false,/continuousReconnect\$: true,/' "$STORES_TS"
echo " - Updated default settings (title/whitespace/animation/reconnect)"
fi
echo ""
echo "texthooker-ui patching complete!"
echo ""
echo "Changes applied:"
echo " 1. socket.ts: Use ?? instead of || to preserve empty strings"
echo " 2. socket.ts: Skip emitting empty sentences"
echo " 3. app.css: Apply SubMiner default styling (without !important)"
echo " 4. stores.ts: Update default settings for title/whitespace/animation/reconnect"
echo ""
echo "To rebuild: cd vendor/texthooker-ui && pnpm run build"

261
scripts/patch-yomitan.sh Executable file
View File

@@ -0,0 +1,261 @@
#!/bin/bash
#
# SubMiner - All-in-one sentence mining overlay
# Copyright (C) 2024 sudacode
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# patch-yomitan.sh - Apply Electron compatibility patches to Yomitan
#
# This script applies the necessary patches to make Yomitan work in Electron
# after upgrading to a new version. Run this after extracting a fresh Yomitan release.
#
# Usage: ./patch-yomitan.sh [yomitan_dir]
# yomitan_dir: Path to the Yomitan directory (default: vendor/yomitan)
#
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
YOMITAN_DIR="${1:-$SCRIPT_DIR/../vendor/yomitan}"
if [ ! -d "$YOMITAN_DIR" ]; then
echo "Error: Yomitan directory not found: $YOMITAN_DIR"
exit 1
fi
echo "Patching Yomitan in: $YOMITAN_DIR"
PERMISSIONS_UTIL="$YOMITAN_DIR/js/data/permissions-util.js"
if [ ! -f "$PERMISSIONS_UTIL" ]; then
echo "Error: permissions-util.js not found at $PERMISSIONS_UTIL"
exit 1
fi
echo "Patching permissions-util.js..."
if grep -q "Electron workaround" "$PERMISSIONS_UTIL"; then
echo " - Already patched, skipping"
else
cat > "$PERMISSIONS_UTIL.tmp" << 'PATCH_EOF'
/*
* Copyright (C) 2023-2025 Yomitan Authors
* Copyright (C) 2021-2022 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {getFieldMarkers} from './anki-util.js';
/**
* This function returns whether an Anki field marker might require clipboard permissions.
* This is speculative and may not guarantee that the field marker actually does require the permission,
* as the custom handlebars template is not deeply inspected.
* @param {string} marker
* @returns {boolean}
*/
function ankiFieldMarkerMayUseClipboard(marker) {
switch (marker) {
case 'clipboard-image':
case 'clipboard-text':
return true;
default:
return false;
}
}
/**
* @param {chrome.permissions.Permissions} permissions
* @returns {Promise<boolean>}
*/
export function hasPermissions(permissions) {
return new Promise((resolve, reject) => {
chrome.permissions.contains(permissions, (result) => {
const e = chrome.runtime.lastError;
if (e) {
reject(new Error(e.message));
} else {
resolve(result);
}
});
});
}
/**
* @param {chrome.permissions.Permissions} permissions
* @param {boolean} shouldHave
* @returns {Promise<boolean>}
*/
export function setPermissionsGranted(permissions, shouldHave) {
return (
shouldHave ?
new Promise((resolve, reject) => {
chrome.permissions.request(permissions, (result) => {
const e = chrome.runtime.lastError;
if (e) {
reject(new Error(e.message));
} else {
resolve(result);
}
});
}) :
new Promise((resolve, reject) => {
chrome.permissions.remove(permissions, (result) => {
const e = chrome.runtime.lastError;
if (e) {
reject(new Error(e.message));
} else {
resolve(!result);
}
});
})
);
}
/**
* @returns {Promise<chrome.permissions.Permissions>}
*/
export function getAllPermissions() {
// Electron workaround - chrome.permissions.getAll() not available
return Promise.resolve({
origins: ["<all_urls>"],
permissions: ["clipboardWrite", "storage", "unlimitedStorage", "scripting", "contextMenus"]
});
}
/**
* @param {string} fieldValue
* @returns {string[]}
*/
export function getRequiredPermissionsForAnkiFieldValue(fieldValue) {
const markers = getFieldMarkers(fieldValue);
for (const marker of markers) {
if (ankiFieldMarkerMayUseClipboard(marker)) {
return ['clipboardRead'];
}
}
return [];
}
/**
* @param {chrome.permissions.Permissions} permissions
* @param {import('settings').ProfileOptions} options
* @returns {boolean}
*/
export function hasRequiredPermissionsForOptions(permissions, options) {
const permissionsSet = new Set(permissions.permissions);
if (!permissionsSet.has('nativeMessaging') && (options.parsing.enableMecabParser || options.general.enableYomitanApi)) {
return false;
}
if (!permissionsSet.has('clipboardRead')) {
if (options.clipboard.enableBackgroundMonitor || options.clipboard.enableSearchPageMonitor) {
return false;
}
const fieldsList = options.anki.cardFormats.map((cardFormat) => cardFormat.fields);
for (const fields of fieldsList) {
for (const {value: fieldValue} of Object.values(fields)) {
const markers = getFieldMarkers(fieldValue);
for (const marker of markers) {
if (ankiFieldMarkerMayUseClipboard(marker)) {
return false;
}
}
}
}
}
return true;
}
PATCH_EOF
mv "$PERMISSIONS_UTIL.tmp" "$PERMISSIONS_UTIL"
echo " - Patched successfully"
fi
OPTIONS_SCHEMA="$YOMITAN_DIR/data/schemas/options-schema.json"
if [ ! -f "$OPTIONS_SCHEMA" ]; then
echo "Error: options-schema.json not found at $OPTIONS_SCHEMA"
exit 1
fi
echo "Patching options-schema.json..."
if grep -q '"selectText".*"default": true' "$OPTIONS_SCHEMA"; then
sed -i '/"selectText": {/,/"default":/{s/"default": true/"default": false/}' "$OPTIONS_SCHEMA"
echo " - Changed selectText default to false"
elif grep -q '"selectText".*"default": false' "$OPTIONS_SCHEMA"; then
echo " - selectText already set to false, skipping"
else
echo " - Warning: Could not find selectText setting"
fi
if grep -q '"layoutAwareScan".*"default": true' "$OPTIONS_SCHEMA"; then
sed -i '/"layoutAwareScan": {/,/"default":/{s/"default": true/"default": false/}' "$OPTIONS_SCHEMA"
echo " - Changed layoutAwareScan default to false"
elif grep -q '"layoutAwareScan".*"default": false' "$OPTIONS_SCHEMA"; then
echo " - layoutAwareScan already set to false, skipping"
else
echo " - Warning: Could not find layoutAwareScan setting"
fi
POPUP_JS="$YOMITAN_DIR/js/app/popup.js"
if [ ! -f "$POPUP_JS" ]; then
echo "Error: popup.js not found at $POPUP_JS"
exit 1
fi
echo "Patching popup.js..."
if grep -q "yomitan-popup-shown" "$POPUP_JS"; then
echo " - Already patched, skipping"
else
# Add the visibility event dispatch after the existing _onVisibleChange code
# We need to add it after: void this._invokeSafe('displayVisibilityChanged', {value});
sed -i "/void this._invokeSafe('displayVisibilityChanged', {value});/a\\
\\
// Dispatch custom events for popup visibility (Electron integration)\\
if (value) {\\
window.dispatchEvent(new CustomEvent('yomitan-popup-shown'));\\
} else {\\
window.dispatchEvent(new CustomEvent('yomitan-popup-hidden'));\\
}" "$POPUP_JS"
echo " - Added visibility events"
fi
echo ""
echo "Yomitan patching complete!"
echo ""
echo "Changes applied:"
echo " 1. permissions-util.js: Hardcoded permissions (Electron workaround)"
echo " 2. options-schema.json: selectText=false, layoutAwareScan=false"
echo " 3. popup.js: Added yomitan-popup-shown/hidden events"
echo ""
echo "To verify: Run 'bun run dev' and check for 'Yomitan extension loaded successfully'"

193
scripts/test-plugin-start-gate.lua Normal file
View File

@@ -0,0 +1,193 @@
local function run_plugin_scenario(config)
config = config or {}
local recorded = {
async_calls = {},
script_messages = {},
osd = {},
logs = {},
}
local function make_mp_stub()
local mp = {}
function mp.get_property(name)
if name == "platform" then
return config.platform or "linux"
end
if name == "filename/no-ext" then
return config.filename_no_ext or ""
end
if name == "filename" then
return config.filename or ""
end
if name == "path" then
return config.path or ""
end
if name == "media-title" then
return config.media_title or ""
end
return ""
end
function mp.get_property_native(_name)
return config.chapter_list or {}
end
function mp.command_native(command)
local args = command.args or {}
if args[1] == "ps" then
return {
status = 0,
stdout = config.process_list or "",
stderr = "",
}
end
if args[1] == "curl" then
return { status = 0, stdout = "{}", stderr = "" }
end
return { status = 0, stdout = "", stderr = "" }
end
function mp.command_native_async(command, callback)
recorded.async_calls[#recorded.async_calls + 1] = command
if callback then
callback(true, { status = 0, stdout = "", stderr = "" }, nil)
end
end
function mp.add_timeout(_seconds, callback)
if callback then
callback()
end
end
function mp.register_script_message(name, fn)
recorded.script_messages[name] = fn
end
function mp.add_key_binding(_keys, _name, _fn) end
function mp.register_event(_name, _fn) end
function mp.add_hook(_name, _prio, _fn) end
function mp.observe_property(_name, _kind, _fn) end
function mp.osd_message(message, _duration)
recorded.osd[#recorded.osd + 1] = message
end
function mp.get_time()
return 0
end
function mp.commandv(...) end
function mp.set_property_native(...) end
function mp.get_script_name()
return "subminer"
end
return mp
end
local mp = make_mp_stub()
local options = {}
local utils = {}
function options.read_options(target, _name)
if config.socket_path then
target.socket_path = config.socket_path
end
end
function utils.file_info(path)
local exists = config.files and config.files[path]
if exists then
return { is_dir = false }
end
return nil
end
function utils.join_path(...)
local parts = { ... }
return table.concat(parts, "/")
end
function utils.parse_json(_json)
return {}, nil
end
package.loaded["mp"] = nil
package.loaded["mp.input"] = nil
package.loaded["mp.msg"] = nil
package.loaded["mp.options"] = nil
package.loaded["mp.utils"] = nil
package.preload["mp"] = function()
return mp
end
package.preload["mp.input"] = function()
return {
select = function(_) end,
}
end
package.preload["mp.msg"] = function()
return {
info = function(line)
recorded.logs[#recorded.logs + 1] = line
end,
warn = function(line)
recorded.logs[#recorded.logs + 1] = line
end,
error = function(line)
recorded.logs[#recorded.logs + 1] = line
end,
debug = function(line)
recorded.logs[#recorded.logs + 1] = line
end,
}
end
package.preload["mp.options"] = function()
return options
end
package.preload["mp.utils"] = function()
return utils
end
local ok, err = pcall(dofile, "plugin/subminer.lua")
if not ok then
return nil, err, recorded
end
return recorded, nil, recorded
end
local function assert_true(condition, message)
if condition then
return
end
error(message)
end
local function find_start_call(async_calls)
for _, call in ipairs(async_calls) do
local args = call.args or {}
for i = 1, #args do
if args[i] == "--start" then
return true
end
end
end
return false
end
local binary_path = "/tmp/subminer-binary"
do
local recorded, err = run_plugin_scenario({
process_list = "",
files = {
[binary_path] = true,
},
})
assert_true(recorded ~= nil, "plugin failed to load for cold-start scenario: " .. tostring(err))
assert_true(recorded.script_messages["subminer-start"] ~= nil, "subminer-start script message not registered")
recorded.script_messages["subminer-start"]("texthooker=no")
assert_true(find_start_call(recorded.async_calls), "expected cold-start to invoke --start command when process is absent")
end
print("plugin start gate regression tests: OK")

638
scripts/test-yomitan-parser.ts Normal file
View File

@@ -0,0 +1,638 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
import { MecabTokenizer } from '../src/mecab-tokenizer.js';
import type { MergedToken } from '../src/types.js';
interface CliOptions {
input: string;
emitPretty: boolean;
emitJson: boolean;
forceMecabOnly: boolean;
yomitanExtensionPath?: string;
yomitanUserDataPath?: string;
mecabCommand?: string;
mecabDictionaryPath?: string;
}
interface YomitanParseHeadword {
term?: unknown;
}
interface YomitanParseSegment {
text?: unknown;
reading?: unknown;
headwords?: unknown;
}
interface YomitanParseResultItem {
source?: unknown;
index?: unknown;
content?: unknown;
}
interface ParsedCandidate {
source: string;
index: number;
tokens: Array<{
surface: string;
reading: string;
headword: string;
startPos: number;
endPos: number;
}>;
}
interface YomitanRuntimeState {
available: boolean;
note: string | null;
extension: Electron.Extension | null;
parserWindow: Electron.BrowserWindow | null;
parserReadyPromise: Promise<void> | null;
parserInitPromise: Promise<boolean> | null;
}
const DEFAULT_YOMITAN_USER_DATA_PATH = path.join(os.homedir(), '.config', 'SubMiner');
function destroyParserWindow(window: Electron.BrowserWindow | null): void {
if (!window || window.isDestroyed()) {
return;
}
window.destroy();
}
async function shutdownYomitanRuntime(yomitan: YomitanRuntimeState): Promise<void> {
destroyParserWindow(yomitan.parserWindow);
const electronModule = await import('electron').catch(() => null);
if (electronModule?.app) {
electronModule.app.quit();
}
}
function parseCliArgs(argv: string[]): CliOptions {
const args = [...argv];
const inputParts: string[] = [];
let emitPretty = true;
let emitJson = false;
let forceMecabOnly = false;
let yomitanExtensionPath: string | undefined;
let yomitanUserDataPath: string | undefined = DEFAULT_YOMITAN_USER_DATA_PATH;
let mecabCommand: string | undefined;
let mecabDictionaryPath: string | undefined;
while (args.length > 0) {
const arg = args.shift();
if (!arg) break;
if (arg === '--help' || arg === '-h') {
printUsage();
process.exit(0);
}
if (arg === '--pretty') {
emitPretty = true;
continue;
}
if (arg === '--json') {
emitJson = true;
continue;
}
if (arg === '--force-mecab') {
forceMecabOnly = true;
continue;
}
if (arg === '--yomitan-extension') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --yomitan-extension');
}
yomitanExtensionPath = path.resolve(next);
continue;
}
if (arg.startsWith('--yomitan-extension=')) {
yomitanExtensionPath = path.resolve(arg.slice('--yomitan-extension='.length));
continue;
}
if (arg === '--yomitan-user-data') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --yomitan-user-data');
}
yomitanUserDataPath = path.resolve(next);
continue;
}
if (arg.startsWith('--yomitan-user-data=')) {
yomitanUserDataPath = path.resolve(arg.slice('--yomitan-user-data='.length));
continue;
}
if (arg === '--mecab-command') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --mecab-command');
}
mecabCommand = next;
continue;
}
if (arg.startsWith('--mecab-command=')) {
mecabCommand = arg.slice('--mecab-command='.length);
continue;
}
if (arg === '--mecab-dictionary') {
const next = args.shift();
if (!next) {
throw new Error('Missing value for --mecab-dictionary');
}
mecabDictionaryPath = next;
continue;
}
if (arg.startsWith('--mecab-dictionary=')) {
mecabDictionaryPath = arg.slice('--mecab-dictionary='.length);
continue;
}
if (arg.startsWith('-')) {
throw new Error(`Unknown flag: ${arg}`);
}
inputParts.push(arg);
}
const input = inputParts.join(' ').trim();
if (input.length > 0) {
return {
input,
emitPretty,
emitJson,
forceMecabOnly,
yomitanExtensionPath,
yomitanUserDataPath,
mecabCommand,
mecabDictionaryPath,
};
}
const stdin = fs.readFileSync(0, 'utf8').trim();
if (!stdin) {
throw new Error('Please provide input text as arguments or via stdin.');
}
return {
input: stdin,
emitPretty,
emitJson,
forceMecabOnly,
yomitanExtensionPath,
yomitanUserDataPath,
mecabCommand,
mecabDictionaryPath,
};
}
function printUsage(): void {
process.stdout.write(`Usage:
bun run test-yomitan-parser:electron -- [--pretty] [--json] [--yomitan-extension <path>] [--yomitan-user-data <path>] [--mecab-command <path>] [--mecab-dictionary <path>] <text>
--pretty Pretty-print JSON output.
--json Emit machine-readable JSON output.
--force-mecab Skip Yomitan parser setup and test MeCab fallback only.
--yomitan-extension <path> Optional path to Yomitan extension directory.
--yomitan-user-data <path> Optional Electron userData directory (default: ~/.config/SubMiner).
--mecab-command <path> Optional MeCab binary path (default: mecab).
--mecab-dictionary <path> Optional MeCab dictionary directory.
-h, --help Show usage.
`);
}
function normalizeDisplayText(text: string): string {
return text.replace(/\r\n/g, '\n').replace(/\\N/g, '\n').replace(/\\n/g, '\n').trim();
}
function normalizeTokenizerText(text: string): string {
return normalizeDisplayText(text).replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
}
function isObject(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === 'object');
}
function isHeadwordRows(value: unknown): value is YomitanParseHeadword[][] {
return (
Array.isArray(value) &&
value.every(
(row) =>
Array.isArray(row) &&
row.every((entry) => isObject(entry) && typeof entry.term === 'string'),
)
);
}
function extractHeadwordTerms(segment: YomitanParseSegment): string[] {
if (!isHeadwordRows(segment.headwords)) {
return [];
}
const terms: string[] = [];
const seen = new Set<string>();
for (const row of segment.headwords) {
for (const entry of row) {
const term = (entry.term as string).trim();
if (!term || seen.has(term)) {
continue;
}
seen.add(term);
terms.push(term);
}
}
return terms;
}
function mapParseResultsToCandidates(parseResults: unknown): ParsedCandidate[] {
if (!Array.isArray(parseResults)) {
return [];
}
const candidates: ParsedCandidate[] = [];
for (const item of parseResults) {
if (!isObject(item)) {
continue;
}
const parseItem = item as YomitanParseResultItem;
if (!Array.isArray(parseItem.content) || typeof parseItem.source !== 'string') {
continue;
}
const candidateTokens: ParsedCandidate['tokens'] = [];
let charOffset = 0;
let validLineCount = 0;
for (const line of parseItem.content) {
if (!Array.isArray(line)) {
continue;
}
const lineSegments = line as YomitanParseSegment[];
if (lineSegments.some((segment) => typeof segment.text !== 'string')) {
continue;
}
validLineCount += 1;
for (const segment of lineSegments) {
const surface = (segment.text as string) ?? '';
if (!surface) {
continue;
}
const startPos = charOffset;
const endPos = startPos + surface.length;
charOffset = endPos;
const headwordTerms = extractHeadwordTerms(segment);
candidateTokens.push({
surface,
reading: typeof segment.reading === 'string' ? segment.reading : '',
headword: headwordTerms[0] ?? surface,
startPos,
endPos,
});
}
}
if (validLineCount === 0 || candidateTokens.length === 0) {
continue;
}
candidates.push({
source: parseItem.source,
index:
typeof parseItem.index === 'number' && Number.isInteger(parseItem.index)
? parseItem.index
: 0,
tokens: candidateTokens,
});
}
return candidates;
}
function candidateTokenSignature(token: {
surface: string;
reading: string;
headword: string;
startPos: number;
endPos: number;
}): string {
return `${token.surface}\u001f${token.reading}\u001f${token.headword}\u001f${token.startPos}\u001f${token.endPos}`;
}
function mergedTokenSignature(token: MergedToken): string {
return `${token.surface}\u001f${token.reading}\u001f${token.headword}\u001f${token.startPos}\u001f${token.endPos}`;
}
function findSelectedCandidateIndexes(
candidates: ParsedCandidate[],
mergedTokens: MergedToken[] | null,
): number[] {
if (!mergedTokens || mergedTokens.length === 0) {
return [];
}
const mergedSignatures = mergedTokens.map(mergedTokenSignature);
const selected: number[] = [];
for (let i = 0; i < candidates.length; i += 1) {
const candidateSignatures = candidates[i].tokens.map(candidateTokenSignature);
if (candidateSignatures.length !== mergedSignatures.length) {
continue;
}
let allMatch = true;
for (let j = 0; j < candidateSignatures.length; j += 1) {
if (candidateSignatures[j] !== mergedSignatures[j]) {
allMatch = false;
break;
}
}
if (allMatch) {
selected.push(i);
}
}
return selected;
}
function resolveYomitanExtensionPath(explicitPath?: string): string | null {
const candidates = [
explicitPath ? path.resolve(explicitPath) : null,
path.resolve(process.cwd(), 'vendor', 'yomitan'),
];
for (const candidate of candidates) {
if (!candidate) {
continue;
}
if (fs.existsSync(path.join(candidate, 'manifest.json'))) {
return candidate;
}
}
return null;
}
async function setupYomitanRuntime(options: CliOptions): Promise<YomitanRuntimeState> {
const state: YomitanRuntimeState = {
available: false,
note: null,
extension: null,
parserWindow: null,
parserReadyPromise: null,
parserInitPromise: null,
};
if (options.forceMecabOnly) {
state.note = 'force-mecab enabled';
return state;
}
const electronModule = await import('electron').catch((error) => {
state.note = error instanceof Error ? error.message : 'electron import failed';
return null;
});
if (!electronModule?.app || !electronModule?.session) {
state.note = 'electron runtime not available in this process';
return state;
}
if (options.yomitanUserDataPath) {
electronModule.app.setPath('userData', options.yomitanUserDataPath);
}
await electronModule.app.whenReady();
const extensionPath = resolveYomitanExtensionPath(options.yomitanExtensionPath);
if (!extensionPath) {
state.note = 'no Yomitan extension directory found';
return state;
}
try {
state.extension = await electronModule.session.defaultSession.loadExtension(extensionPath, {
allowFileAccess: true,
});
state.available = true;
return state;
} catch (error) {
state.note = error instanceof Error ? error.message : 'failed to load Yomitan extension';
state.available = false;
return state;
}
}
async function fetchRawParseResults(
parserWindow: Electron.BrowserWindow,
text: string,
): Promise<unknown> {
const script = `
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
}
if (!response || typeof response !== "object") {
reject(new Error("Invalid response from Yomitan backend"));
return;
}
if (response.error) {
reject(new Error(response.error.message || "Yomitan backend error"));
return;
}
resolve(response.result);
});
});
const optionsFull = await invoke("optionsGetFull", undefined);
const profileIndex = optionsFull.profileCurrent;
const scanLength =
optionsFull.profiles?.[profileIndex]?.options?.scanning?.length ?? 40;
return await invoke("parseText", {
text: ${JSON.stringify(text)},
optionsContext: { index: profileIndex },
scanLength,
useInternalParser: true,
useMecabParser: true
});
})();
`;
return parserWindow.webContents.executeJavaScript(script, true);
}
function renderTextOutput(payload: Record<string, unknown>): void {
process.stdout.write(`Input: ${String(payload.input)}\n`);
process.stdout.write(`Tokenizer text: ${String(payload.tokenizerText)}\n`);
process.stdout.write(`Yomitan available: ${String(payload.yomitanAvailable)}\n`);
process.stdout.write(`Yomitan note: ${String(payload.yomitanNote ?? '')}\n`);
process.stdout.write(
`Selected candidate indexes: ${JSON.stringify(payload.selectedCandidateIndexes)}\n`,
);
process.stdout.write('\nFinal selected tokens:\n');
const finalTokens = payload.finalTokens as Array<Record<string, unknown>> | null;
if (!finalTokens || finalTokens.length === 0) {
process.stdout.write(' (none)\n');
} else {
for (let i = 0; i < finalTokens.length; i += 1) {
const token = finalTokens[i];
process.stdout.write(
` [${i}] ${token.surface} -> ${token.headword} (${token.reading}) [${token.startPos}, ${token.endPos})\n`,
);
}
}
process.stdout.write('\nYomitan parse candidates:\n');
const candidates = payload.candidates as Array<Record<string, unknown>>;
if (!candidates || candidates.length === 0) {
process.stdout.write(' (none)\n');
return;
}
for (let i = 0; i < candidates.length; i += 1) {
const candidate = candidates[i];
process.stdout.write(
` [${i}] source=${String(candidate.source)} index=${String(candidate.index)} selectedByTokenizer=${String(candidate.selectedByTokenizer)} tokenCount=${String(candidate.tokenCount)}\n`,
);
const tokens = candidate.tokens as Array<Record<string, unknown>> | undefined;
if (!tokens || tokens.length === 0) {
continue;
}
for (let j = 0; j < tokens.length; j += 1) {
const token = tokens[j];
process.stdout.write(
` - ${token.surface} -> ${token.headword} (${token.reading}) [${token.startPos}, ${token.endPos})\n`,
);
}
}
}
async function main(): Promise<void> {
const args = parseCliArgs(process.argv.slice(2));
const yomitan: YomitanRuntimeState = {
available: false,
note: null,
extension: null,
parserWindow: null,
parserReadyPromise: null,
parserInitPromise: null,
};
try {
const mecabTokenizer = new MecabTokenizer({
mecabCommand: args.mecabCommand,
dictionaryPath: args.mecabDictionaryPath,
});
const isMecabAvailable = await mecabTokenizer.checkAvailability();
if (!isMecabAvailable) {
throw new Error('MeCab is not available on this system.');
}
const runtime = await setupYomitanRuntime(args);
yomitan.available = runtime.available;
yomitan.note = runtime.note;
yomitan.extension = runtime.extension;
yomitan.parserWindow = runtime.parserWindow;
yomitan.parserReadyPromise = runtime.parserReadyPromise;
yomitan.parserInitPromise = runtime.parserInitPromise;
const deps = createTokenizerDepsRuntime({
getYomitanExt: () => yomitan.extension,
getYomitanParserWindow: () => yomitan.parserWindow,
setYomitanParserWindow: (window) => {
yomitan.parserWindow = window;
},
getYomitanParserReadyPromise: () => yomitan.parserReadyPromise,
setYomitanParserReadyPromise: (promise) => {
yomitan.parserReadyPromise = promise;
},
getYomitanParserInitPromise: () => yomitan.parserInitPromise,
setYomitanParserInitPromise: (promise) => {
yomitan.parserInitPromise = promise;
},
isKnownWord: () => false,
getKnownWordMatchMode: () => 'headword',
getJlptLevel: () => null,
getMecabTokenizer: () => ({
tokenize: (text: string) => mecabTokenizer.tokenize(text),
}),
});
const subtitleData = await tokenizeSubtitle(args.input, deps);
const tokenizeText = normalizeTokenizerText(args.input);
let rawParseResults: unknown = null;
if (
yomitan.available &&
yomitan.parserWindow &&
!yomitan.parserWindow.isDestroyed() &&
tokenizeText
) {
rawParseResults = await fetchRawParseResults(yomitan.parserWindow, tokenizeText);
}
const parsedCandidates = mapParseResultsToCandidates(rawParseResults);
const selectedCandidateIndexes = findSelectedCandidateIndexes(
parsedCandidates,
subtitleData.tokens,
);
const selectedIndexSet = new Set<number>(selectedCandidateIndexes);
const payload = {
input: args.input,
tokenizerText: subtitleData.text,
yomitanAvailable: yomitan.available,
yomitanNote: yomitan.note,
selectedCandidateIndexes,
finalTokens:
subtitleData.tokens?.map((token) => ({
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
pos1: token.pos1,
partOfSpeech: token.partOfSpeech,
isKnown: token.isKnown,
isNPlusOneTarget: token.isNPlusOneTarget,
})) ?? null,
candidates: parsedCandidates.map((candidate, idx) => ({
source: candidate.source,
index: candidate.index,
selectedByTokenizer: selectedIndexSet.has(idx),
tokenCount: candidate.tokens.length,
tokens: candidate.tokens,
})),
};
if (args.emitJson) {
process.stdout.write(`${JSON.stringify(payload, null, args.emitPretty ? 2 : undefined)}\n`);
} else {
renderTextOutput(payload);
}
} finally {
await shutdownYomitanRuntime(yomitan);
}
}
main()
.then(() => {
process.exit(0);
})
.catch((error) => {
console.error(`Error: ${(error as Error).message}`);
process.exit(1);
});

42
scripts/verify-generated-launcher.sh Executable file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LAUNCHER_OUT="$REPO_ROOT/dist/launcher/subminer"
if [[ ! -f "$REPO_ROOT/launcher/main.ts" ]]; then
echo "[FAIL] launcher source missing: launcher/main.ts"
exit 1
fi
if ! grep -Fn -- "--outfile=\"\$(LAUNCHER_OUT)\"" "$REPO_ROOT/Makefile" >/dev/null; then
echo "[FAIL] Makefile build-launcher target is not writing to dist/launcher/subminer"
exit 1
fi
if [[ ! -f "$LAUNCHER_OUT" ]]; then
echo "[FAIL] generated launcher not found at dist/launcher/subminer"
echo " run: make build-launcher"
exit 1
fi
if [[ ! -x "$LAUNCHER_OUT" ]]; then
echo "[FAIL] generated launcher is not executable: dist/launcher/subminer"
exit 1
fi
if [[ -f "$REPO_ROOT/subminer" ]]; then
echo "[FAIL] stale repo-root launcher artifact found: ./subminer"
echo " expected generated location: dist/launcher/subminer"
echo " remove stale artifact and use: make build-launcher"
exit 1
fi
if git -C "$REPO_ROOT" ls-files --error-unmatch dist/launcher/subminer >/dev/null 2>&1; then
echo "[FAIL] dist/launcher/subminer is tracked by git; generated artifacts must remain untracked"
exit 1
fi
echo "[OK] launcher workflow verified"
echo " source: launcher/*.ts"
echo " generated artifact: dist/launcher/subminer"