diff --git a/changes/log-export-local-date.md b/changes/log-export-local-date.md new file mode 100644 index 00000000..ed988076 --- /dev/null +++ b/changes/log-export-local-date.md @@ -0,0 +1,5 @@ +type: fixed +area: logs + +- Fixed log filenames to use the user's local date so log export includes the current logs instead of stale prior-day logs around UTC midnight. +- Expanded log export redaction to mask common PII and secrets, including IP addresses, emails, auth/cookie headers, yt-dlp cookie arguments, URL credentials, compound token/key/password fields, and signed YouTube media URL query strings. diff --git a/docs-site/configuration.md b/docs-site/configuration.md index b2a8f99f..6d6cf0ea 100644 --- a/docs-site/configuration.md +++ b/docs-site/configuration.md @@ -188,6 +188,9 @@ Control the minimum log level for runtime output: | `files.launcher` | boolean | Write launcher command logs (default: `true`) | | `files.mpv` | boolean | Write mpv player logs. Enable temporarily for mpv/plugin debugging. | +Log filenames use the local calendar date, for example `app-YYYY-MM-DD.log`, `launcher-YYYY-MM-DD.log`, and `mpv-YYYY-MM-DD.log`. +Log export creates a sanitized copy of those files; it does not rewrite the original log files on disk. + ### Updates Configure automatic update checks and update notifications: diff --git a/docs-site/launcher-script.md b/docs-site/launcher-script.md index fd5db4e7..d86bd11b 100644 --- a/docs-site/launcher-script.md +++ b/docs-site/launcher-script.md @@ -83,7 +83,7 @@ subminer stats -b # start background stats daemon | `subminer stats cleanup` | Backfill vocabulary metadata and prune stale rows | | `subminer doctor` | Dependency + config + socket diagnostics | | `subminer settings` | Open the SubMiner settings window | -| `subminer logs -e` | Export a sanitized log ZIP and print its path | +| `subminer logs -e` | Export a sanitized local-date log ZIP and print its path | | `subminer config path` | Print active config file path | | `subminer config show` | Print active config contents | | `subminer mpv status` | Check mpv socket readiness | diff --git a/docs-site/usage.md b/docs-site/usage.md index 4ddf0d22..35196448 100644 --- a/docs-site/usage.md +++ b/docs-site/usage.md @@ -148,7 +148,7 @@ SubMiner.AppImage --dictionary-select --dictionary-anilist-id 21355 # Pin corre SubMiner.AppImage --help # Show all options ``` -The tray menu includes `Export Logs`, which creates the same sanitized log ZIP as `subminer logs -e` and shows the archive path when complete. +The tray menu includes `Export Logs`, which creates the same sanitized local-date log ZIP as `subminer logs -e` and shows the archive path when complete. Export sanitization masks common PII and secrets, including home-directory usernames, IP addresses, emails, auth/cookie headers, yt-dlp cookie arguments, URL credentials, token/key/password fields, and signed YouTube media URL query strings. The exported copy is sanitized; source log files remain unredacted on disk. Once Jellyfin is configured, the tray menu includes `Jellyfin Discovery` for starting or stopping cast discovery in the current app session without changing config. @@ -191,7 +191,7 @@ This flow requires `mpv.exe` to be discoverable. Leave `mpv.executablePath` blan - `subminer jellyfin` / `subminer jf`: Jellyfin-focused workflow aliases. - `subminer doctor`: health checks for core dependencies and runtime paths. - `subminer settings`: open the SubMiner settings window (also `subminer --settings`). -- `subminer logs -e`: export a sanitized ZIP of today's logs, or the most recent logs when no current-day log exists. +- `subminer logs -e`: export a sanitized ZIP of today's local-date logs, or the most recent logs when no current-day log exists. The exported copy masks common PII and secrets; on-disk logs are unchanged. - `subminer config`: config file helpers (`path`, `show`). - `subminer mpv`: mpv helpers (`status`, `socket`, `idle`). - `subminer dictionary `: generates a Yomitan-importable character dictionary ZIP from a file/directory target. diff --git a/launcher/log.test.ts b/launcher/log.test.ts index fa898d15..aabd2a44 100644 --- a/launcher/log.test.ts +++ b/launcher/log.test.ts @@ -2,9 +2,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; import { getDefaultLauncherLogFile, getDefaultMpvLogFile } from './types.js'; +import { localDateKey } from '../src/shared/log-files.js'; test('getDefaultMpvLogFile uses APPDATA on windows', () => { - const today = new Date().toISOString().slice(0, 10); + const today = localDateKey(new Date()); const resolved = getDefaultMpvLogFile({ platform: 'win32', homeDir: 'C:\\Users\\tester', @@ -20,7 +21,7 @@ test('getDefaultMpvLogFile uses APPDATA on windows', () => { }); test('getDefaultLauncherLogFile uses launcher prefix', () => { - const today = new Date().toISOString().slice(0, 10); + const today = localDateKey(new Date()); const resolved = getDefaultLauncherLogFile({ platform: 'linux', homeDir: '/home/tester', diff --git a/src/logger.test.ts b/src/logger.test.ts index cda4f694..bce6dd73 100644 --- a/src/logger.test.ts +++ b/src/logger.test.ts @@ -2,9 +2,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; import { resolveDefaultLogFilePath, setLogRotation } from './logger'; +import { localDateKey } from './shared/log-files'; test('resolveDefaultLogFilePath uses APPDATA on windows', () => { - const today = new Date().toISOString().slice(0, 10); + const today = localDateKey(new Date()); const resolved = resolveDefaultLogFilePath({ platform: 'win32', homeDir: 'C:\\Users\\tester', @@ -20,7 +21,7 @@ test('resolveDefaultLogFilePath uses APPDATA on windows', () => { }); test('resolveDefaultLogFilePath uses .config on linux', () => { - const today = new Date().toISOString().slice(0, 10); + const today = localDateKey(new Date()); const resolved = resolveDefaultLogFilePath({ platform: 'linux', homeDir: '/home/tester', @@ -34,7 +35,7 @@ test('resolveDefaultLogFilePath uses .config on linux', () => { test('setLogRotation accepts numeric retention days', () => { const previous = process.env.SUBMINER_LOG_ROTATION; - const today = new Date().toISOString().slice(0, 10); + const today = localDateKey(new Date()); setLogRotation(14); try { const resolved = resolveDefaultLogFilePath({ diff --git a/src/main/runtime/log-export.test.ts b/src/main/runtime/log-export.test.ts index 854b9456..25cec0f5 100644 --- a/src/main/runtime/log-export.test.ts +++ b/src/main/runtime/log-export.test.ts @@ -14,6 +14,20 @@ function cleanupDir(dirPath: string): void { fs.rmSync(dirPath, { recursive: true, force: true }); } +function withTimeZone(timeZone: string, run: () => T): T { + const previous = process.env.TZ; + process.env.TZ = timeZone; + try { + return run(); + } finally { + if (previous === undefined) { + delete process.env.TZ; + } else { + process.env.TZ = previous; + } + } +} + function writeLog(logsDir: string, name: string, content: string, mtime: string): string { const logPath = path.join(logsDir, name); fs.writeFileSync(logPath, content, 'utf8'); @@ -67,6 +81,80 @@ test('maskUsernamesInLogText redacts linux macOS and Windows home paths', () => assert.doesNotMatch(masked, /kyle/); }); +test('maskUsernamesInLogText redacts IP addresses and emails', () => { + const masked = maskUsernamesInLogText( + [ + 'ffmpeg failed after request from public ip 203.0.113.42', + 'connect tcp 192.168.1.25:443: i/o timeout', + 'remote addr [2001:db8::1234]:443', + 'support email kyle@example.test', + ].join('\n'), + ); + + assert.match(masked, /public ip /); + assert.match(masked, /tcp :443/); + assert.match(masked, /remote addr \[\]:443/); + assert.match(masked, /support email /); + assert.doesNotMatch(masked, /203\.0\.113\.42/); + assert.doesNotMatch(masked, /192\.168\.1\.25/); + assert.doesNotMatch(masked, /2001:db8::1234/); + assert.doesNotMatch(masked, /kyle@example\.test/); +}); + +test('maskUsernamesInLogText redacts headers and yt-dlp cookie arguments', () => { + const masked = maskUsernamesInLogText( + [ + 'Authorization: Bearer ya29.secret-token', + 'Cookie: SID=session-value; HSID=history-value', + 'Set-Cookie: VISITOR_INFO1_LIVE=visitor; Path=/', + 'x-goog-visitor-id: Cgt2aXNpdG9y', + 'warn yt-dlp header Authorization: Bearer inline-token', + 'yt-dlp --cookies /Users/kyle/cookies.txt --cookies-from-browser chrome:Profile 1', + 'yt-dlp --cookies=/tmp/ytdlp-cookies.txt --cookies-from-browser=firefox:default', + ].join('\n'), + ); + + assert.match(masked, /Authorization: /); + assert.match(masked, /Cookie: /); + assert.match(masked, /Set-Cookie: /); + assert.match(masked, /x-goog-visitor-id: /i); + assert.match(masked, /--cookies /); + assert.match(masked, /--cookies=/); + assert.match(masked, /--cookies-from-browser /); + assert.match(masked, /--cookies-from-browser=/); + assert.doesNotMatch(masked, /ya29/); + assert.doesNotMatch(masked, /inline-token/); + assert.doesNotMatch(masked, /session-value/); + assert.doesNotMatch(masked, /cookies\.txt/); + assert.doesNotMatch(masked, /firefox:default/); +}); + +test('maskUsernamesInLogText redacts URL credentials and sensitive query values', () => { + const masked = maskUsernamesInLogText( + [ + 'GET https://alice:secret@example.test/watch?v=abc&access_token=tok123&api_key=key456', + 'stream https://video.example.test/file.m3u8?signature=sig789&expire=1777777777', + 'callback subminer://anilist-setup?access_token=ani-token&state=ok', + 'json {"password":"hunter2","refreshToken":"refresh-token","client_secret":"client-secret"}', + ].join('\n'), + ); + + assert.match(masked, /https:\/\/@example\.test/); + assert.match(masked, /access_token=/); + assert.match(masked, /api_key=/); + assert.match(masked, /signature=/); + assert.match(masked, /"password":""/); + assert.match(masked, /"refreshToken":""/); + assert.match(masked, /"client_secret":""/); + assert.doesNotMatch(masked, /alice:secret/); + assert.doesNotMatch(masked, /tok123/); + assert.doesNotMatch(masked, /key456/); + assert.doesNotMatch(masked, /sig789/); + assert.doesNotMatch(masked, /ani-token/); + assert.doesNotMatch(masked, /hunter2/); + assert.match(masked, /state=ok/); +}); + test('exportLogsArchive exports current-day logs and masks usernames', () => { const root = makeTempDir(); const logsDir = path.join(root, 'logs'); @@ -152,6 +240,43 @@ test('exportLogsArchive ignores older dated logs when current-day dated logs exi } }); +test('exportLogsArchive ranks dated fallback logs by local day freshness', () => { + withTimeZone('America/Los_Angeles', () => { + const root = makeTempDir(); + const logsDir = path.join(root, 'logs'); + fs.mkdirSync(logsDir, { recursive: true }); + + try { + const datedLog = writeLog( + logsDir, + 'app-2026-05-25.log', + 'dated local-day log\n', + '2026-05-25T12:00:00Z', + ); + writeLog( + logsDir, + 'app-2026-05-undated.log', + 'undated touched before local midnight\n', + '2026-05-26T01:00:00Z', + ); + + const result = exportLogsArchive({ + logsDir, + outputDir: root, + now: new Date('2026-05-27T16:00:00.000Z'), + }); + + assert.equal(result.mode, 'most-recent'); + assert.deepEqual(result.exportedFiles, [datedLog]); + + const entries = readStoredZipEntries(result.zipPath); + assert.deepEqual([...entries.keys()], ['logs/app-2026-05-25.log']); + } finally { + cleanupDir(root); + } + }); +}); + test('exportLogsArchive falls back to newest log per kind', () => { const root = makeTempDir(); const logsDir = path.join(root, 'logs'); diff --git a/src/main/runtime/log-export.ts b/src/main/runtime/log-export.ts index 302857d7..25fca4af 100644 --- a/src/main/runtime/log-export.ts +++ b/src/main/runtime/log-export.ts @@ -1,8 +1,9 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { resolveLogBaseDir } from '../../shared/log-files'; +import { localDateKey, resolveLogBaseDir } from '../../shared/log-files'; import { writeStoredZip } from '../../shared/stored-zip'; +import { redactLogExportText } from './log-redaction'; type LogCandidate = { path: string; @@ -29,16 +30,10 @@ export type ExportLogsOptions = { now?: Date; }; -const REDACTED_USER = ''; - function pad(value: number): string { return String(value).padStart(2, '0'); } -function localDateKey(date: Date): string { - return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; -} - function localWeekKey(date: Date): string { const startOfYear = new Date(date.getFullYear(), 0, 1); const dayOfYear = @@ -126,17 +121,41 @@ function selectMostRecentPerKind(candidates: LogCandidate[]): LogCandidate[] { return [...byKind.values()].sort((left, right) => left.name.localeCompare(right.name)); } +function localDateEndMs(dateKey: string): number | null { + const match = dateKey.match(/^(\d{4})-(\d{2})-(\d{2})$/); + if (!match) return null; + + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const date = new Date(year, month - 1, day, 23, 59, 59, 999); + if ( + Number.isNaN(date.getTime()) || + date.getFullYear() !== year || + date.getMonth() !== month - 1 || + date.getDate() !== day + ) { + return null; + } + return date.getTime(); +} + +function localWeekEndMs(weekKey: string): number | null { + const match = weekKey.match(/^(\d{4})-W(\d{2})$/); + if (!match) return null; + + const year = Number(match[1]); + const week = Number(match[2]); + const date = new Date(year, 0, week * 7, 23, 59, 59, 999); + return Number.isNaN(date.getTime()) ? null : date.getTime(); +} + function candidateFreshnessMs(candidate: LogCandidate): number { if (candidate.fileDateKey) { - return Date.parse(`${candidate.fileDateKey}T23:59:59.999Z`); + return localDateEndMs(candidate.fileDateKey) ?? candidate.mtimeMs; } if (candidate.fileWeekKey) { - const match = candidate.fileWeekKey.match(/^(\d{4})-W(\d{2})$/); - if (match) { - const year = Number(match[1]); - const week = Number(match[2]); - return Date.UTC(year, 0, week * 7, 23, 59, 59, 999); - } + return localWeekEndMs(candidate.fileWeekKey) ?? candidate.mtimeMs; } return candidate.mtimeMs; } @@ -167,9 +186,7 @@ function selectLogCandidates( } export function maskUsernamesInLogText(text: string): string { - return text - .replace(/(\/(?:home|Users)\/)([^/\r\n]+)(?=\/|$)/g, `$1${REDACTED_USER}`) - .replace(/([A-Za-z]:[\\/]+Users[\\/]+)([^\\/:\r\n]+)(?=[\\/]|$)/g, `$1${REDACTED_USER}`); + return redactLogExportText(text); } export function exportLogsArchive(options: ExportLogsOptions = {}): ExportLogsResult { diff --git a/src/main/runtime/log-redaction.test.ts b/src/main/runtime/log-redaction.test.ts new file mode 100644 index 00000000..bd71a387 --- /dev/null +++ b/src/main/runtime/log-redaction.test.ts @@ -0,0 +1,137 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { LOG_EXPORT_REDACTION_RULES, redactLogExportText } from './log-redaction'; + +test('log export redaction rules expose auditable fixtures', () => { + assert.deepEqual( + LOG_EXPORT_REDACTION_RULES.map((rule) => rule.name), + [ + 'signed-youtube-media-url-query', + 'url-sensitive-components', + 'sensitive-headers', + 'yt-dlp-cookie-args', + 'json-secret-values', + 'generic-sensitive-key-values', + 'home-path-usernames', + 'email-addresses', + 'ip-addresses', + ], + ); + + const names = new Set(); + for (const rule of LOG_EXPORT_REDACTION_RULES) { + assert.equal(names.has(rule.name), false, `duplicate rule name: ${rule.name}`); + names.add(rule.name); + assert.ok(rule.pattern.length > 0, `${rule.name} pattern`); + assert.ok(rule.replacement.length > 0, `${rule.name} replacement`); + assert.ok(rule.risk.length > 0, `${rule.name} risk`); + assert.ok(rule.examples.length > 0, `${rule.name} examples`); + + for (const example of rule.examples) { + assert.equal(rule.redact(example.input), example.expected, `${rule.name} direct`); + assert.equal(redactLogExportText(example.input), example.expected, rule.name); + } + } +}); + +test('redactLogExportText redacts parsed URL query values without swallowing punctuation', () => { + const masked = redactLogExportText( + [ + 'failed URL (https://example.test/watch?access_token=tok123).', + 'callback subminer://anilist-setup?access_token=ani-token&state=ok', + 'encoded https://example.test/watch?client%5Fsecret=secret-value&plain=ok!', + ].join('\n'), + ); + + assert.match(masked, /\(https:\/\/example\.test\/watch\?access_token=\)\./); + assert.match(masked, /subminer:\/\/anilist-setup\?access_token=&state=ok/); + assert.match(masked, /client%5Fsecret=&plain=ok!/); + assert.doesNotMatch(masked, /tok123/); + assert.doesNotMatch(masked, /ani-token/); + assert.doesNotMatch(masked, /secret-value/); +}); + +test('redactLogExportText redacts URL credentials containing at signs', () => { + const masked = redactLogExportText('GET https://alice:p@ss@example.test/watch?state=ok'); + + assert.equal(masked, 'GET https://@example.test/watch?state=ok'); + assert.doesNotMatch(masked, /alice/); + assert.doesNotMatch(masked, /p@ss/); +}); + +test('redactLogExportText redacts signed YouTube media URL query strings', () => { + const masked = redactLogExportText( + [ + 'ffmpeg failed for (https://rr1---sn-a5mekn6r.googlevideo.com/videoplayback?expire=1777777777&ei=EI_VALUE&id=o-SECRETID&itag=251&ip=203.0.113.42&source=youtube&requiressl=yes&n=NSECRET&sparams=expire,ei,ip,id,itag,source,requiressl&sig=SIGSECRET&lsig=LSIGSECRET).', + 'manifest https://manifest.googlevideo.com/videoplayback?expire=1777777777&signature=SIG2#frag', + ].join('\n'), + ); + + assert.match( + masked, + /\(https:\/\/rr1---sn-a5mekn6r\.googlevideo\.com\/videoplayback\?\)\./, + ); + assert.match(masked, /https:\/\/manifest\.googlevideo\.com\/videoplayback\?#frag/); + assert.doesNotMatch(masked, /1777777777/); + assert.doesNotMatch(masked, /EI_VALUE/); + assert.doesNotMatch(masked, /o-SECRETID/); + assert.doesNotMatch(masked, /203\.0\.113\.42/); + assert.doesNotMatch(masked, /NSECRET/); + assert.doesNotMatch(masked, /SIGSECRET/); + assert.doesNotMatch(masked, /LSIGSECRET/); +}); + +test('redactLogExportText redacts nested JSON secret values', () => { + const masked = redactLogExportText( + 'json {"nested":{"token":123,"safe":"ok"},"items":[{"password":true},{"client_secret":null}]}', + ); + + assert.match(masked, /"token":""/); + assert.match(masked, /"safe":"ok"/); + assert.match(masked, /"password":""/); + assert.match(masked, /"client_secret":""/); + assert.doesNotMatch(masked, /123/); + assert.doesNotMatch(masked, /true/); + assert.doesNotMatch(masked, /null/); +}); + +test('redactLogExportText redacts compound secret keys', () => { + const masked = redactLogExportText( + [ + 'json {"openaiApiKey":"sk-secret","google_api_key":"AIza-secret","userPassword":"hunter2","safe":"ok"}', + 'openaiApiKey=sk-inline google_api_key=AIza-inline userPassword=hunter-inline', + ].join('\n'), + ); + + assert.match(masked, /"openaiApiKey":""/); + assert.match(masked, /"google_api_key":""/); + assert.match(masked, /"userPassword":""/); + assert.match(masked, /"safe":"ok"/); + assert.match(masked, /openaiApiKey=/); + assert.match(masked, /google_api_key=/); + assert.match(masked, /userPassword=/); + assert.doesNotMatch(masked, /sk-secret|AIza-secret|hunter2/); + assert.doesNotMatch(masked, /sk-inline|AIza-inline|hunter-inline/); +}); + +test('redactLogExportText redacts quoted secret values containing the opposite quote', () => { + const masked = redactLogExportText(`"password":"pa'ss" 'token':'to"ken' "safe":"pa'ss"`); + + assert.equal(masked, `"password":"" 'token':'' "safe":"pa'ss"`); +}); + +test('redactLogExportText redacts IPv6 addresses with zone identifiers', () => { + const masked = redactLogExportText('connected [fe80::1%en0]:443 and fe80::2%eth0'); + + assert.equal(masked, 'connected []:443 and '); +}); + +test('redactLogExportText keeps malformed JSON scans bounded', () => { + const malformedLine = `${'{'.repeat(70 * 1024)} token=secret`; + const masked = redactLogExportText(`${malformedLine}\njson {"token":"secret","safe":"ok"}`); + + assert.match(masked, /token=/); + assert.match(masked, /json {"token":"","safe":"ok"}/); + assert.doesNotMatch(masked, /token=secret/); + assert.doesNotMatch(masked, /"token":"secret"/); +}); diff --git a/src/main/runtime/log-redaction.ts b/src/main/runtime/log-redaction.ts new file mode 100644 index 00000000..190cc16c --- /dev/null +++ b/src/main/runtime/log-redaction.ts @@ -0,0 +1,522 @@ +import * as net from 'net'; + +type LogRedactionExample = { + input: string; + expected: string; +}; + +export type LogRedactionRule = { + name: string; + pattern: string; + replacement: string; + risk: 'pii' | 'secret' | 'pii-or-secret'; + examples: readonly LogRedactionExample[]; + redact: (text: string) => string; +}; + +const REDACTED_USER = ''; +const REDACTED_VALUE = ''; +const REDACTED_CREDENTIALS = ''; +const REDACTED_EMAIL = ''; +const REDACTED_IP = ''; +const MAX_JSON_CANDIDATE_SCAN_CHARS = 64 * 1024; + +const SENSITIVE_HEADER_NAMES = [ + 'api-key', + 'authorization', + 'cookie', + 'proxy-authorization', + 'set-cookie', + 'x-api-key', + 'x-emby-token', + 'x-goog-visitor-id', + 'x-mediabrowser-token', +].join('|'); + +const SENSITIVE_VALUE_KEY_PATTERNS = [ + 'access[_-]?token', + 'api[_-]?key', + 'apikey', + 'authorization', + 'client[_-]?secret', + 'cookie', + 'cookies', + 'id[_-]?token', + 'password', + 'passwd', + 'pwd', + 'refresh[_-]?token', + 'secret', + 'session', + 'sid', + 'sig', + 'signature', + 'token', +].join('|'); + +const SENSITIVE_VALUE_KEY_SEQUENCES: readonly (readonly string[])[] = [ + ['access', 'token'], + ['api', 'key'], + ['apikey'], + ['authorization'], + ['client', 'secret'], + ['cookie'], + ['cookies'], + ['id', 'token'], + ['password'], + ['passwd'], + ['pwd'], + ['refresh', 'token'], + ['secret'], + ['session'], + ['sid'], + ['sig'], + ['signature'], + ['token'], +]; +const SENSITIVE_NORMALIZED_KEY_SUFFIXES = [ + 'accesstoken', + 'apikey', + 'authorization', + 'clientsecret', + 'cookie', + 'cookies', + 'idtoken', + 'password', + 'passwd', + 'pwd', + 'refreshtoken', + 'secret', + 'session', + 'signature', + 'token', +] as const; +const SENSITIVE_HEADER_RE = new RegExp( + `(^|\\r?\\n)(\\s*(?:${SENSITIVE_HEADER_NAMES})\\s*:\\s*)[^\\r\\n]*`, + 'gi', +); +const SENSITIVE_INLINE_HEADER_RE = new RegExp( + `\\b((?:${SENSITIVE_HEADER_NAMES})\\s*:\\s*)[^\\r\\n]*`, + 'gi', +); +const KEY_VALUE_NAME_PATTERN = '[A-Za-z0-9][A-Za-z0-9_.%-]*'; +const SENSITIVE_QUOTED_VALUE_RE = new RegExp( + `(["'])(${KEY_VALUE_NAME_PATTERN})\\1(\\s*[:=]\\s*)(["'])((?:(?!\\4)[^\\r\\n])*)\\4`, + 'g', +); +const SENSITIVE_UNQUOTED_VALUE_RE = new RegExp( + `\\b(${KEY_VALUE_NAME_PATTERN})(\\s*[:=]\\s*)(?:"[^"\\r\\n]*"|'[^'\\r\\n]*'|[^\\s,}\\]\\)&<>"']+)`, + 'g', +); +const YTDLP_COOKIE_EQUALS_RE = + /(--(?:cookies|cookies-from-browser)=)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s\r\n]+)/gi; +const YTDLP_COOKIE_SPACE_RE = + /(--(?:cookies|cookies-from-browser)\s+)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\r\n]*?)(?=\s+--[A-Za-z0-9][A-Za-z0-9-]*|\r?\n|$)/gi; +const URL_TOKEN_RE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/gi; +const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)([^/?#\s"'<>]*@)/gi; +const URL_QUERY_PAIR_RE = /([?&;]|&)([^=&#;]+)=([^&#;]*)/gi; +const TRAILING_URL_PUNCTUATION_RE = /[)\].,;:!?]+$/; +const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const BRACKETED_IP_RE = /\[([0-9A-F:.]+(?:%[A-Z0-9_.~-]+)?)\]/gi; +const IPV4_RE = /(^|[^A-Za-z0-9_.-])((?:\d{1,3}\.){3}\d{1,3})(?![A-Za-z0-9_.-])/g; +const IPV6_RE = + /(^|[^A-Za-z0-9_.-])([0-9A-F]{0,4}:[0-9A-F:.]*(?:%[A-Z0-9_.~-]+)?)(?![A-Za-z0-9_.-])/gi; + +function safeDecodeFormComponent(value: string): string { + try { + return decodeURIComponent(value.replace(/\+/g, ' ')); + } catch { + return value; + } +} + +function tokenizeValueKey(key: string): string[] { + return ( + key + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .match(/[a-z0-9]+/g) ?? [] + ); +} + +function containsTokenSequence(tokens: readonly string[], sequence: readonly string[]): boolean { + for (let index = 0; index <= tokens.length - sequence.length; index += 1) { + if (sequence.every((token, offset) => tokens[index + offset] === token)) { + return true; + } + } + return false; +} + +function isSensitiveValueKey(key: string): boolean { + const decoded = safeDecodeFormComponent(key); + const tokens = tokenizeValueKey(decoded); + if (SENSITIVE_VALUE_KEY_SEQUENCES.some((sequence) => containsTokenSequence(tokens, sequence))) { + return true; + } + + const normalized = tokens.join(''); + return SENSITIVE_NORMALIZED_KEY_SUFFIXES.some((suffix) => normalized.endsWith(suffix)); +} + +function splitTrailingUrlPunctuation(token: string): { core: string; trailing: string } { + const match = token.match(TRAILING_URL_PUNCTUATION_RE); + if (!match) return { core: token, trailing: '' }; + return { + core: token.slice(0, -match[0].length), + trailing: match[0], + }; +} + +function redactSensitiveUrlQueryPairs(rawUrl: string): string { + const queryStart = rawUrl.indexOf('?'); + if (queryStart === -1) return rawUrl; + + const hashStart = rawUrl.indexOf('#', queryStart); + const queryEnd = hashStart === -1 ? rawUrl.length : hashStart; + const query = rawUrl.slice(queryStart, queryEnd); + const redactedQuery = query.replace(URL_QUERY_PAIR_RE, (match, prefix: string, rawKey: string) => + isSensitiveValueKey(rawKey) ? `${prefix}${rawKey}=${REDACTED_VALUE}` : match, + ); + + return `${rawUrl.slice(0, queryStart)}${redactedQuery}${rawUrl.slice(queryEnd)}`; +} + +function redactUrlToken(token: string): string { + const { core, trailing } = splitTrailingUrlPunctuation(token); + try { + new URL(core); + } catch { + return token; + } + + const withoutCredentials = core.replace(URL_CREDENTIALS_RE, `$1${REDACTED_CREDENTIALS}@`); + return `${redactSensitiveUrlQueryPairs(withoutCredentials)}${trailing}`; +} + +function redactUrlSensitiveComponents(text: string): string { + return text.replace(URL_TOKEN_RE, (token: string) => redactUrlToken(token)); +} + +function isGoogleVideoHost(hostname: string): boolean { + const normalized = hostname.toLowerCase(); + return normalized === 'googlevideo.com' || normalized.endsWith('.googlevideo.com'); +} + +function isSignedYouTubeMediaUrl(url: URL): boolean { + return isGoogleVideoHost(url.hostname) && url.pathname === '/videoplayback' && url.search !== ''; +} + +function redactSignedYouTubeMediaUrlToken(token: string): string { + const { core, trailing } = splitTrailingUrlPunctuation(token); + let parsed: URL; + try { + parsed = new URL(core); + } catch { + return token; + } + + if (!isSignedYouTubeMediaUrl(parsed)) return token; + + const queryStart = core.indexOf('?'); + const hashStart = core.indexOf('#', queryStart); + if (queryStart === -1) return token; + + const hash = hashStart === -1 ? '' : core.slice(hashStart); + return `${core.slice(0, queryStart)}?${REDACTED_VALUE}${hash}${trailing}`; +} + +function redactSignedYouTubeMediaUrlQueries(text: string): string { + return text.replace(URL_TOKEN_RE, (token: string) => redactSignedYouTubeMediaUrlToken(token)); +} + +function redactSensitiveHeaders(text: string): string { + return text + .replace( + SENSITIVE_HEADER_RE, + (_match, lineStart: string, headerPrefix: string) => + `${lineStart}${headerPrefix}${REDACTED_VALUE}`, + ) + .replace( + SENSITIVE_INLINE_HEADER_RE, + (_match, headerPrefix: string) => `${headerPrefix}${REDACTED_VALUE}`, + ); +} + +function redactYtDlpCookieArgs(text: string): string { + return text + .replace(YTDLP_COOKIE_EQUALS_RE, `$1${REDACTED_VALUE}`) + .replace(YTDLP_COOKIE_SPACE_RE, `$1${REDACTED_VALUE}`); +} + +function findJsonScanEnd(text: string, start: number): number { + const boundedEnd = Math.min(text.length, start + MAX_JSON_CANDIDATE_SCAN_CHARS); + const newline = text.indexOf('\n', start); + if (newline !== -1 && newline < boundedEnd) return newline; + return boundedEnd; +} + +function findJsonEnd(text: string, start: number, endExclusive: number): number { + const stack: string[] = []; + let inString = false; + let quote = ''; + let escaped = false; + + for (let index = start; index < endExclusive; index += 1) { + const char = text[index]; + + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === quote) { + inString = false; + } + continue; + } + + if (char === '"' || char === "'") { + inString = true; + quote = char; + continue; + } + + if (char === '{') { + stack.push('}'); + } else if (char === '[') { + stack.push(']'); + } else if (char === '}' || char === ']') { + if (stack.pop() !== char) return -1; + if (stack.length === 0) return index; + } + } + + return -1; +} + +function redactJsonValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((entry) => redactJsonValue(entry)); + } + + if (value && typeof value === 'object') { + const redacted: Record = {}; + for (const [key, entry] of Object.entries(value)) { + redacted[key] = isSensitiveValueKey(key) ? REDACTED_VALUE : redactJsonValue(entry); + } + return redacted; + } + + return value; +} + +function redactJsonPayloads(text: string): string { + let output = ''; + let cursor = 0; + + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + if (char !== '{' && char !== '[') continue; + + const scanEnd = findJsonScanEnd(text, index); + const end = findJsonEnd(text, index, scanEnd); + if (end === -1) { + index = Math.max(index, scanEnd - 1); + continue; + } + + const candidate = text.slice(index, end + 1); + try { + const parsed = JSON.parse(candidate); + output += text.slice(cursor, index); + output += JSON.stringify(redactJsonValue(parsed)); + cursor = end + 1; + index = end; + } catch { + // Non-JSON brace pairs remain available to the fallback key-value rules. + } + } + + return `${output}${text.slice(cursor)}`; +} + +function redactSensitiveKeyValues(text: string): string { + return text + .replace( + SENSITIVE_QUOTED_VALUE_RE, + (match, keyQuote: string, key: string, separator: string, valueQuote: string) => + isSensitiveValueKey(key) + ? `${keyQuote}${key}${keyQuote}${separator}${valueQuote}${REDACTED_VALUE}${valueQuote}` + : match, + ) + .replace(SENSITIVE_UNQUOTED_VALUE_RE, (match, key: string, separator: string) => { + if (!isSensitiveValueKey(key)) return match; + const value = match.slice(key.length + separator.length); + const quote = value[0]; + if (quote === '"' || quote === "'") { + return `${key}${separator}${quote}${REDACTED_VALUE}${quote}`; + } + return `${key}${separator}${REDACTED_VALUE}`; + }); +} + +function redactHomePathUsernames(text: string): string { + return text + .replace(/(\/(?:home|Users)\/)([^/\r\n]+)(?=\/|$)/g, `$1${REDACTED_USER}`) + .replace(/([A-Za-z]:[\\/]+Users[\\/]+)([^\\/:\r\n]+)(?=[\\/]|$)/g, `$1${REDACTED_USER}`); +} + +function normalizeIpCandidate(candidate: string): string { + return candidate.split('%', 1)[0] ?? candidate; +} + +function isIpAddress(candidate: string): boolean { + return net.isIP(normalizeIpCandidate(candidate)) !== 0; +} + +function redactIpAddresses(text: string): string { + return text + .replace(BRACKETED_IP_RE, (match, candidate: string) => + isIpAddress(candidate) ? `[${REDACTED_IP}]` : match, + ) + .replace(IPV4_RE, (match, prefix: string, candidate: string) => + net.isIP(candidate) === 4 ? `${prefix}${REDACTED_IP}` : match, + ) + .replace(IPV6_RE, (match, prefix: string, candidate: string) => + net.isIP(normalizeIpCandidate(candidate)) === 6 ? `${prefix}${REDACTED_IP}` : match, + ); +} + +export const LOG_EXPORT_REDACTION_RULES: readonly LogRedactionRule[] = [ + { + name: 'signed-youtube-media-url-query', + pattern: '*.googlevideo.com/videoplayback query strings', + replacement: REDACTED_VALUE, + risk: 'pii-or-secret', + examples: [ + { + input: 'https://rr1---sn.example.googlevideo.com/videoplayback?expire=1&sig=secret', + expected: `https://rr1---sn.example.googlevideo.com/videoplayback?${REDACTED_VALUE}`, + }, + ], + redact: redactSignedYouTubeMediaUrlQueries, + }, + { + name: 'url-sensitive-components', + pattern: 'URL tokens with credentials or sensitive query params', + replacement: `${REDACTED_CREDENTIALS}, ${REDACTED_VALUE}`, + risk: 'pii-or-secret', + examples: [ + { + input: 'GET https://alice:secret@example.test/watch?access_token=tok123&state=ok', + expected: `GET https://${REDACTED_CREDENTIALS}@example.test/watch?access_token=${REDACTED_VALUE}&state=ok`, + }, + ], + redact: redactUrlSensitiveComponents, + }, + { + name: 'sensitive-headers', + pattern: `headers: ${SENSITIVE_HEADER_NAMES}`, + replacement: REDACTED_VALUE, + risk: 'secret', + examples: [ + { + input: 'Authorization: Bearer token', + expected: `Authorization: ${REDACTED_VALUE}`, + }, + ], + redact: redactSensitiveHeaders, + }, + { + name: 'yt-dlp-cookie-args', + pattern: '--cookies and --cookies-from-browser arguments', + replacement: REDACTED_VALUE, + risk: 'secret', + examples: [ + { + input: 'yt-dlp --cookies /Users/kyle/cookies.txt --verbose', + expected: `yt-dlp --cookies ${REDACTED_VALUE} --verbose`, + }, + ], + redact: redactYtDlpCookieArgs, + }, + { + name: 'json-secret-values', + pattern: `JSON object keys: ${SENSITIVE_VALUE_KEY_PATTERNS}`, + replacement: REDACTED_VALUE, + risk: 'secret', + examples: [ + { + input: 'json {"token":123,"safe":"ok"}', + expected: `json {"token":"${REDACTED_VALUE}","safe":"ok"}`, + }, + { + input: 'json {"openaiApiKey":"sk-secret","safe":"ok"}', + expected: `json {"openaiApiKey":"${REDACTED_VALUE}","safe":"ok"}`, + }, + ], + redact: redactJsonPayloads, + }, + { + name: 'generic-sensitive-key-values', + pattern: `key-value pairs: ${SENSITIVE_VALUE_KEY_PATTERNS}`, + replacement: REDACTED_VALUE, + risk: 'secret', + examples: [ + { + input: 'refreshToken=abc123 state=ok', + expected: `refreshToken=${REDACTED_VALUE} state=ok`, + }, + { + input: 'google_api_key=AIza-secret userPassword=hunter2', + expected: `google_api_key=${REDACTED_VALUE} userPassword=${REDACTED_VALUE}`, + }, + ], + redact: redactSensitiveKeyValues, + }, + { + name: 'home-path-usernames', + pattern: 'Linux, macOS, and Windows home paths', + replacement: REDACTED_USER, + risk: 'pii', + examples: [ + { + input: '/Users/kyle/Library/Application Support/SubMiner', + expected: `/Users/${REDACTED_USER}/Library/Application Support/SubMiner`, + }, + ], + redact: redactHomePathUsernames, + }, + { + name: 'email-addresses', + pattern: 'email-like addresses', + replacement: REDACTED_EMAIL, + risk: 'pii', + examples: [ + { + input: 'support kyle@example.test', + expected: `support ${REDACTED_EMAIL}`, + }, + ], + redact: (text) => text.replace(EMAIL_RE, REDACTED_EMAIL), + }, + { + name: 'ip-addresses', + pattern: 'IPv4, bracketed IPv6, and bare IPv6 candidates validated with net.isIP', + replacement: REDACTED_IP, + risk: 'pii', + examples: [ + { + input: 'remote addr [2001:db8::1234]:443 and 203.0.113.42', + expected: `remote addr [${REDACTED_IP}]:443 and ${REDACTED_IP}`, + }, + ], + redact: redactIpAddresses, + }, +]; + +export function redactLogExportText(text: string): string { + return LOG_EXPORT_REDACTION_RULES.reduce((redacted, rule) => rule.redact(redacted), text); +} diff --git a/src/shared/log-files.test.ts b/src/shared/log-files.test.ts index 1441ae2a..1e5ee2e7 100644 --- a/src/shared/log-files.test.ts +++ b/src/shared/log-files.test.ts @@ -12,6 +12,45 @@ import { resolveDefaultLogFilePath, } from './log-files'; +function withTimeZone(timeZone: string, run: () => T): T { + const previous = process.env.TZ; + process.env.TZ = timeZone; + try { + return run(); + } finally { + if (previous === undefined) { + delete process.env.TZ; + } else { + process.env.TZ = previous; + } + } +} + +function withMockedNow(isoDate: string, run: () => T): T { + const RealDate = Date; + const fixedMs = RealDate.parse(isoDate); + class FixedDate extends RealDate { + constructor(value?: string | number | Date) { + if (arguments.length === 0) { + super(fixedMs); + } else { + super(value as string); + } + } + + static override now(): number { + return fixedMs; + } + } + + globalThis.Date = FixedDate as DateConstructor; + try { + return run(); + } finally { + globalThis.Date = RealDate; + } +} + test('resolveDefaultLogFilePath uses app prefix by default', () => { const now = new Date('2026-03-22T12:00:00.000Z'); const resolved = resolveDefaultLogFilePath('app', { @@ -26,6 +65,36 @@ test('resolveDefaultLogFilePath uses app prefix by default', () => { ); }); +test('resolveDefaultLogFilePath uses the local date when UTC has advanced', () => { + withTimeZone('America/Los_Angeles', () => { + const now = new Date('2026-05-26T02:00:00.000Z'); + assert.equal(now.toISOString().slice(0, 10), '2026-05-26'); + + const resolved = resolveDefaultLogFilePath('app', { + platform: 'linux', + homeDir: '/home/tester', + now, + }); + + assert.equal( + resolved, + path.join('/home/tester', '.config', 'SubMiner', 'logs', 'app-2026-05-25.log'), + ); + }); +}); + +test('resolveDefaultLogFilePath rejects invalid dates', () => { + assert.throws( + () => + resolveDefaultLogFilePath('app', { + platform: 'linux', + homeDir: '/home/tester', + now: new Date('invalid'), + }), + /Invalid time value/, + ); +}); + test('resolveDefaultLogFilePath uses daily filenames for mpv logs', () => { const now = new Date('2026-03-22T12:00:00.000Z'); const resolved = resolveDefaultLogFilePath('mpv', { @@ -91,6 +160,34 @@ test('pruneLogFiles removes logs older than retention window', () => { } }); +test('pruneLogDirectoryForPath deduplicates by local day across UTC midnight', () => { + withTimeZone('America/Los_Angeles', () => { + const logsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-log-prune-local-day-')); + const logPath = path.join(logsDir, 'app-2026-05-25.log'); + const firstStalePath = path.join(logsDir, 'app-first-stale.log'); + const secondStalePath = path.join(logsDir, 'app-second-stale.log'); + const staleDate = new Date('2026-05-01T12:00:00.000Z'); + + try { + fs.writeFileSync(firstStalePath, 'stale\n', 'utf8'); + fs.utimesSync(firstStalePath, staleDate, staleDate); + + withMockedNow('2026-05-25T23:30:00.000Z', () => pruneLogDirectoryForPath(logPath, 7)); + + assert.equal(fs.existsSync(firstStalePath), false); + + fs.writeFileSync(secondStalePath, 'stale\n', 'utf8'); + fs.utimesSync(secondStalePath, staleDate, staleDate); + + withMockedNow('2026-05-26T00:30:00.000Z', () => pruneLogDirectoryForPath(logPath, 7)); + + assert.equal(fs.existsSync(secondStalePath), true); + } finally { + fs.rmSync(logsDir, { recursive: true, force: true }); + } + }); +}); + test('appendLogLine trims oversized logs to newest bytes', () => { const logsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-log-trim-')); const logPath = path.join(logsDir, 'app.log'); diff --git a/src/shared/log-files.ts b/src/shared/log-files.ts index 19b7f10f..5ca9d848 100644 --- a/src/shared/log-files.ts +++ b/src/shared/log-files.ts @@ -36,6 +36,17 @@ function floorDiv(left: number, right: number): number { return Math.floor(left / right); } +function padDatePart(value: number): string { + return String(value).padStart(2, '0'); +} + +export function localDateKey(date: Date): string { + if (Number.isNaN(date.getTime())) { + throw new RangeError('Invalid time value'); + } + return `${date.getFullYear()}-${padDatePart(date.getMonth() + 1)}-${padDatePart(date.getDate())}`; +} + function daysFromCivil(year: number, month: number, day: number): bigint { const adjustedYear = year - (month <= 2 ? 1 : 0); const era = floorDiv(adjustedYear >= 0 ? adjustedYear : adjustedYear - 399, 400); @@ -78,7 +89,7 @@ export function resolveDefaultLogFilePath( }, ): string { const now = options?.now ?? new Date(); - const suffix = now.toISOString().slice(0, 10); + const suffix = localDateKey(now); return path.join(resolveLogBaseDir(options), 'logs', `${kind}-${suffix}.log`); } @@ -167,7 +178,7 @@ export function pruneLogFiles( function maybePruneLogDirectory(logPath: string, retentionDays: number): void { const logsDir = path.dirname(logPath); - const key = `${logsDir}:${new Date().toISOString().slice(0, 10)}:${retentionDays}`; + const key = `${logsDir}:${localDateKey(new Date())}:${retentionDays}`; if (prunedDirectories.has(key)) return; pruneLogFiles(logsDir, { retentionDays }); prunedDirectories.add(key);