sudacode/dotfiles
1
0
Fork 0
mirror of https://github.com/ksyasuda/dotfiles.git synced 2026-07-23 16:49:46 -07:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-09 23:26:11 -07:00
parent bf8420e4ab
commit 54d97bb493
32 changed files with 11768 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/skills/html-plan/references/html-effectiveness/12-incident-report.html
+597
View File
@@ -0,0 +1,597 @@
<!-- Copyright 2026 Anthropic PBC · SPDX-License-Identifier: Apache-2.0 -->
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>INC-2025-0412 — Elevated 502s on task sync</title>
<style>
:root {
--ivory: #FAF9F5;
--slate: #141413;
--clay: #D97757;
--oat: #E3DACC;
--olive: #788C5D;
--rust: #B04A3F;
--gray-100: #F0EEE6;
--gray-300: #D1CFC5;
--gray-500: #87867F;
--gray-700: #3D3D3A;
--white: #FFFFFF;
--serif: ui-serif, Georgia, serif;
--sans: system-ui, -apple-system, sans-serif;
--mono: ui-monospace, 'SF Mono', Menlo, monospace;
--radius-panel: 12px;
--radius-row: 8px;
--border: 1.5px solid var(--gray-300);
}
* { box-sizing: border-box; }
html { scroll-behavior: smooth; }
body {
margin: 0;
padding: 56px 24px 120px;
background: var(--ivory);
color: var(--slate);
font-family: var(--sans);
font-size: 15px;
line-height: 1.6;
-webkit-font-smoothing: antialiased;
}
.page {
max-width: 820px;
margin: 0 auto;
}
/* ---------- Header ---------- */
header {
margin-bottom: 32px;
}
.inc-id {
font-family: var(--mono);
font-size: 13px;
color: var(--gray-500);
letter-spacing: 0.02em;
margin-bottom: 8px;
}
h1 {
font-family: var(--serif);
font-weight: 500;
font-size: 36px;
letter-spacing: -0.01em;
margin: 0 0 18px;
line-height: 1.2;
}
.meta-row {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 10px 12px;
}
.pill {
display: inline-flex;
align-items: baseline;
gap: 6px;
font-family: var(--sans);
font-size: 12px;
font-weight: 600;
border-radius: 999px;
padding: 5px 12px;
line-height: 1;
}
.pill .k {
font-weight: 400;
opacity: 0.75;
}
.pill.sev {
background: var(--clay);
color: var(--white);
letter-spacing: 0.03em;
}
.pill.resolved {
background: var(--olive);
color: var(--white);
}
.pill.neutral {
background: var(--gray-100);
color: var(--gray-700);
border: var(--border);
}
.pill.neutral .v {
font-family: var(--mono);
}
/* ---------- TL;DR ---------- */
.tldr {
background: var(--slate);
color: var(--ivory);
border-radius: var(--radius-panel);
padding: 22px 26px;
margin-bottom: 48px;
}
.tldr-label {
font-family: var(--mono);
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.1em;
color: var(--oat);
margin-bottom: 10px;
}
.tldr p {
margin: 0;
font-size: 15.5px;
line-height: 1.65;
}
.tldr code {
font-family: var(--mono);
font-size: 13.5px;
background: rgba(250, 249, 245, 0.12);
padding: 1px 5px;
border-radius: 4px;
}
/* ---------- Sections ---------- */
section {
margin-bottom: 52px;
scroll-margin-top: 24px;
}
h2 {
font-family: var(--serif);
font-weight: 500;
font-size: 24px;
letter-spacing: -0.01em;
margin: 0 0 6px;
}
hr.rule {
border: none;
border-top: 1px solid var(--gray-300);
margin: 0 0 22px;
}
p { margin: 0 0 14px; }
code.inline {
font-family: var(--mono);
font-size: 13px;
background: var(--gray-100);
padding: 1.5px 5px;
border-radius: 4px;
}
/* ---------- Timeline ---------- */
.timeline {
position: relative;
padding: 4px 0 4px 16px;
}
.timeline::before {
content: "";
position: absolute;
left: 16px;
top: 8px;
bottom: 8px;
width: 2px;
background: var(--gray-300);
}
.tl-entry {
position: relative;
padding: 0 0 22px 28px;
}
.tl-entry:last-child {
padding-bottom: 0;
}
.tl-dot {
position: absolute;
left: -5px;
top: 6px;
width: 12px;
height: 12px;
border-radius: 50%;
background: var(--gray-500);
border: 2px solid var(--ivory);
box-sizing: content-box;
}
.tl-dot.impact { background: var(--clay); }
.tl-dot.mitigated { background: var(--olive); }
.tl-time {
display: inline-block;
font-family: var(--mono);
font-size: 12px;
color: var(--gray-700);
background: var(--gray-100);
border: 1px solid var(--gray-300);
border-radius: 6px;
padding: 2px 8px;
margin-bottom: 6px;
}
.tl-body {
font-size: 14px;
color: var(--gray-700);
}
.tl-body strong {
color: var(--slate);
font-weight: 600;
}
/* ---------- Code diff ---------- */
.code-panel {
background: var(--slate);
color: var(--gray-100);
border-radius: var(--radius-panel);
padding: 18px 20px;
font-family: var(--mono);
font-size: 13px;
line-height: 1.7;
overflow-x: auto;
margin: 8px 0 4px;
}
.code-panel .path {
color: var(--gray-500);
font-size: 12px;
margin-bottom: 10px;
display: block;
}
.diff-line { white-space: pre; }
.diff-line.ctx { color: var(--gray-300); }
.diff-line.del { color: #E0897A; }
.diff-line.add { color: #A3B88A; }
/* ---------- Impact table ---------- */
table.impact {
width: 100%;
max-width: 460px;
border-collapse: separate;
border-spacing: 0;
background: var(--white);
border: var(--border);
border-radius: var(--radius-panel);
overflow: hidden;
}
table.impact th,
table.impact td {
text-align: left;
padding: 12px 18px;
font-size: 14px;
border-bottom: 1px solid var(--gray-100);
}
table.impact tr:last-child th,
table.impact tr:last-child td {
border-bottom: none;
}
table.impact th {
font-weight: 400;
color: var(--gray-500);
width: 55%;
}
table.impact td {
font-family: var(--mono);
color: var(--slate);
}
/* ---------- Action items ---------- */
.actions {
background: var(--white);
border: var(--border);
border-radius: var(--radius-panel);
overflow: hidden;
}
.ai-row {
display: grid;
grid-template-columns: 36px 36px 1fr 96px;
align-items: center;
gap: 14px;
padding: 14px 18px;
border-bottom: 1px solid var(--gray-100);
}
.ai-row:last-child { border-bottom: none; }
.ai-check {
width: 18px;
height: 18px;
border: 1.5px solid var(--gray-300);
border-radius: 5px;
background: var(--white);
position: relative;
}
.ai-row.done .ai-check {
background: var(--olive);
border-color: var(--olive);
}
.ai-row.done .ai-check::after {
content: "";
position: absolute;
left: 4px;
top: 1px;
width: 5px;
height: 9px;
border: solid var(--white);
border-width: 0 2px 2px 0;
transform: rotate(40deg);
}
.ai-row.done .ai-desc {
color: var(--gray-500);
text-decoration: line-through;
text-decoration-color: var(--gray-300);
}
.ai-avatar {
width: 30px;
height: 30px;
border-radius: 50%;
background: var(--oat);
color: var(--gray-700);
font-size: 11px;
font-weight: 600;
display: flex;
align-items: center;
justify-content: center;
letter-spacing: 0.02em;
}
.ai-desc {
font-size: 14px;
color: var(--slate);
}
.ai-due {
font-family: var(--mono);
font-size: 12px;
color: var(--gray-500);
text-align: right;
}
/* ---------- Fixed TOC ---------- */
.toc {
display: none;
}
@media (min-width: 1100px) {
.toc {
display: block;
position: fixed;
top: 72px;
right: max(24px, calc(50vw - 410px - 200px));
width: 170px;
font-size: 13px;
}
.toc-title {
font-family: var(--mono);
font-size: 10px;
text-transform: uppercase;
letter-spacing: 0.1em;
color: var(--gray-500);
margin-bottom: 12px;
}
.toc a {
display: block;
color: var(--gray-500);
text-decoration: none;
padding: 5px 0 5px 12px;
border-left: 1.5px solid var(--gray-300);
line-height: 1.3;
}
.toc a:hover {
color: var(--slate);
border-left-color: var(--clay);
}
}
footer {
margin-top: 64px;
padding-top: 20px;
border-top: 1px solid var(--gray-300);
font-family: var(--mono);
font-size: 12px;
color: var(--gray-500);
}
</style>
</head>
<body>
<nav class="toc">
<div class="toc-title">On this page</div>
<a href="#tldr">TL;DR</a>
<a href="#timeline">Timeline</a>
<a href="#root-cause">Root cause</a>
<a href="#impact">Impact</a>
<a href="#actions">Action items</a>
</nav>
<div class="page">
<header>
<div class="inc-id">INC-2025-0412</div>
<h1>Elevated 502s on task sync</h1>
<div class="meta-row">
<span class="pill sev">SEV-2</span>
<span class="pill resolved">Resolved</span>
<span class="pill neutral"><span class="k">Duration</span> <span class="v">47 min</span></span>
<span class="pill neutral"><span class="k">Detected</span> <span class="v">Apr 12 &middot; 14:07</span></span>
<span class="pill neutral"><span class="k">Owner</span> <span class="v">Devon Park</span></span>
</div>
</header>
<div class="tldr" id="tldr">
<div class="tldr-label">TL;DR</div>
<p>
A config rollout lowered the database connection-pool limit on the
<code>sync-worker</code> tier from 64 to 8, exhausting connections under
normal afternoon load. The sync API returned 502s for roughly 21% of
requests over 47 minutes. We mitigated by reverting the config and
cycling the worker fleet; no data was lost.
</p>
</div>
<!-- Timeline -->
<section id="timeline">
<h2>Timeline</h2>
<hr class="rule">
<div class="timeline">
<div class="tl-entry">
<span class="tl-dot"></span>
<span class="tl-time">14:02</span>
<div class="tl-body">
Config change <code class="inline">cfg-9a12</code> promoted to
production via the standard rollout pipeline.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot impact"></span>
<span class="tl-time">14:06</span>
<div class="tl-body">
<strong>Impact starts.</strong> Sync workers begin queueing on pool
checkout; p95 latency climbs past 4s and the load balancer starts
returning 502s.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot"></span>
<span class="tl-time">14:07</span>
<div class="tl-body">
Alert fires: <code class="inline">sync_5xx_rate &gt; 2%</code> for 60s.
On-call (Devon) acknowledges.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot"></span>
<span class="tl-time">14:18</span>
<div class="tl-body">
Initial hypothesis is a bad deploy of the API service; last two
application deploys are rolled back with no effect.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot"></span>
<span class="tl-time">14:31</span>
<div class="tl-body">
Mira joins and notices pool-wait saturation in the worker dashboard.
Investigation pivots to infra config rather than app code.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot mitigated"></span>
<span class="tl-time">14:44</span>
<div class="tl-body">
<strong>Mitigated.</strong> <code class="inline">cfg-9a12</code>
reverted; worker fleet cycled. 5xx rate drops below 0.2% within
three minutes.
</div>
</div>
<div class="tl-entry">
<span class="tl-dot"></span>
<span class="tl-time">14:49</span>
<div class="tl-body">
Monitors green for 5 minutes. Incident declared resolved; status
page updated.
</div>
</div>
</div>
</section>
<!-- Root cause -->
<section id="root-cause">
<h2>Root cause</h2>
<hr class="rule">
<p>
PR <code class="inline">#4888</code> made connection-pool limits
configurable per worker tier. The default for the new
<code class="inline">sync-worker</code> key was meant to inherit the
global value (64) but was hard-coded to 8 during a local test and
committed. The config linter only validates type, not magnitude, so the
change passed CI.
</p>
<p>
Because config rollouts and code deploys go through separate pipelines,
the on-call's first instinct &mdash; rolling back the most recent
application deploys &mdash; had no effect and cost roughly 13 minutes of
diagnosis time.
</p>
<div class="code-panel">
<span class="path">infra/config/workers.yaml</span>
<div class="diff-line ctx"> pool:</div>
<div class="diff-line ctx"> global_max_connections: 64</div>
<div class="diff-line ctx"> tiers:</div>
<div class="diff-line del">- sync-worker: { max_connections: 64 }</div>
<div class="diff-line add">+ sync-worker: { max_connections: 8 } # debug value, do not ship</div>
<div class="diff-line ctx"> webhook-worker: { max_connections: 32 }</div>
</div>
</section>
<!-- Impact -->
<section id="impact">
<h2>Impact</h2>
<hr class="rule">
<table class="impact">
<tr>
<th>Requests failed (502)</th>
<td>~41,200</td>
</tr>
<tr>
<th>Peak error rate</th>
<td>21.4%</td>
</tr>
<tr>
<th>Users affected</th>
<td>~2,300 workspaces</td>
</tr>
<tr>
<th>Data loss</th>
<td>None &mdash; clients retried</td>
</tr>
<tr>
<th>SLA breach</th>
<td>No (within monthly budget)</td>
</tr>
</table>
</section>
<!-- Action items -->
<section id="actions">
<h2>Action items</h2>
<hr class="rule">
<div class="actions">
<div class="ai-row done">
<span class="ai-check"></span>
<span class="ai-avatar" title="Devon Park">DP</span>
<span class="ai-desc">Revert cfg-9a12 and restore pool limit to 64</span>
<span class="ai-due">Apr 12</span>
</div>
<div class="ai-row">
<span class="ai-check"></span>
<span class="ai-avatar" title="Mira Okafor">MO</span>
<span class="ai-desc">Add config-linter range check for <code class="inline">max_connections</code> (warn &lt; 32)</span>
<span class="ai-due">Apr 18</span>
</div>
<div class="ai-row">
<span class="ai-check"></span>
<span class="ai-avatar" title="Sam Reyes">SR</span>
<span class="ai-desc">Surface &ldquo;recent config rollouts&rdquo; alongside deploys in the on-call dashboard</span>
<span class="ai-due">Apr 25</span>
</div>
<div class="ai-row">
<span class="ai-check"></span>
<span class="ai-avatar" title="Priya Anand">PA</span>
<span class="ai-desc">Canary config changes to one worker AZ for 10 min before fleet-wide promote</span>
<span class="ai-due">May 02</span>
</div>
</div>
</section>
<footer>
Authored from on-call notes + alert history &middot; reviewed by Devon Park, Mira Okafor
</footer>
</div>
</body>
</html>