diff --git a/docker-compose.yml b/docker-compose.yml index 0a2a523..52c1300 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -183,6 +183,46 @@ services: retries: 3 start_period: 360s + # api-watchdog (2026-09-12, see docker/watchdog/README.md in the + # argus-appliance repo): closes a real production gap -- `restart: + # unless-stopped` above only ever reacts to the api container actually + # EXITING. A real incident (CT 101, 2026-09-11/12) had it hang/wedge + # internally for ~11 hours while `docker ps` still reported it `Up`; the + # HEALTHCHECK above correctly kept detecting the hang the entire time, + # but nothing was watching that status and acting on it. This polls the + # same already-scoped docker-socket-proxy (zero new grants) and restarts + # argus-api after several consecutive unhealthy checks. + # + # Built locally rather than pulled/pinned/cosign-verified like api/ui/ + # nginx above: there is not yet a released, signed argus-api-watchdog + # image in the registry (tracked as a follow-up to extend ADR-0051's + # verification set to a 4th component). The build context ships in this + # repo so it stays fully reproducible from source, not a hand-built + # local image nobody can regenerate. + api-watchdog: + build: + context: ./watchdog + container_name: argus-api-watchdog + restart: unless-stopped + security_opt: + - "no-new-privileges:true" + cap_drop: + - ALL + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + environment: + WATCHDOG_TARGET_CONTAINER: argus-api + WATCHDOG_INTERVAL_SECONDS: "30" + WATCHDOG_UNHEALTHY_THRESHOLD: "5" + depends_on: + docker-socket-proxy: + condition: service_started + api: + condition: service_started + # React UI (Admin Panel), served over HTTPS ui: # See the api service's own comment above -- same ADR-0051 pinning. diff --git a/watchdog/Dockerfile b/watchdog/Dockerfile new file mode 100644 index 0000000..7840d72 --- /dev/null +++ b/watchdog/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:3.20 + +RUN apk add --no-cache curl jq + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +USER nobody +ENTRYPOINT ["/entrypoint.sh"] diff --git a/watchdog/README.md b/watchdog/README.md new file mode 100644 index 0000000..b594351 --- /dev/null +++ b/watchdog/README.md @@ -0,0 +1,45 @@ +# api-watchdog + +Recovers from a real production incident class (2026-09-11/12, CT 101): the `api` container's +process hung/wedged internally while `docker ps` still reported it `Up` -- it never actually +exited, so Docker's own `restart: unless-stopped` policy (which only reacts to a container +exiting) never fired. The container's `HEALTHCHECK` (`wget --spider http://localhost:8080/health`) +correctly kept detecting the hang the entire time -- `docker inspect`'s `.State.Health.Status` +genuinely flipped to `unhealthy` -- but nothing was watching that status and acting on it, so the +outage lasted roughly 11 hours until a human happened to notice and ran `docker restart` by hand. + +## What it does + +Polls `docker-socket-proxy` (the same already-existing, scoped Docker API proxy the `api` service +itself uses -- see `docker/docker-socket-proxy/README.md`) for `argus-api`'s health status every +`WATCHDOG_INTERVAL_SECONDS` (default 30s). After `WATCHDOG_UNHEALTHY_THRESHOLD` (default 5) +*consecutive* `unhealthy` readings (~2.5 minutes sustained, not one blip), it issues +`POST /containers/argus-api/restart` through the same proxy and resets its counter. A `healthy` +reading at any point resets the counter immediately. + +Deliberately talks to `docker-socket-proxy`, never a raw `/var/run/docker.sock` mount -- zero new +grants needed (`CONTAINERS=1`/`ALLOW_RESTARTS=1` are already on for the `api` service's own +diagnostics recovery actions), and this container never gets docker.sock access at all. + +## Why not just shorten `retries`/rely on `restart: unless-stopped` alone + +`restart: unless-stopped` restarts a container Docker itself observes as exited -- it has no +concept of "unhealthy," and this incident's whole failure mode was a container that never +exited. Docker Engine does not ship a built-in "restart on unhealthy" action; some setups solve +this with the third-party `willfarrell/autoheal` image, but that requires mounting the real +docker.sock into a new container. Reusing the already-present, already-audited +`docker-socket-proxy` instead keeps this fix inside ARGUS's existing security boundary. + +## Tuning + +- `WATCHDOG_TARGET_CONTAINER` (default `argus-api`) +- `WATCHDOG_INTERVAL_SECONDS` (default `30`, matches the api healthcheck's own interval) +- `WATCHDOG_UNHEALTHY_THRESHOLD` (default `5`) -- consecutive unhealthy checks before restarting + +## Verification + +Sandbox-tested (`docker-compose.local-sandbox.yml`) by temporarily forcing the `api` service's +healthcheck to always fail (`test: ["CMD", "false"]`), confirming: the watchdog logs 5 consecutive +`unhealthy` checks, then a real `POST .../restart` through the proxy, and `argus-api`'s container +start time visibly advances. Reverted the forced-failure healthcheck override afterward -- this +watchdog is not itself part of that sandbox scenario's cleanup. diff --git a/watchdog/entrypoint.sh b/watchdog/entrypoint.sh new file mode 100644 index 0000000..72dbc56 --- /dev/null +++ b/watchdog/entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# api-watchdog: closes a real gap found during a 2026-09-11/12 production +# incident (see docker/watchdog/README.md) -- Docker's own `restart: +# unless-stopped` policy only ever acts on a container actually EXITING; it +# does nothing for a container that hangs/wedges while `docker ps` still +# reports it "Up". The api service's HEALTHCHECK (wget against /health) +# correctly kept detecting the hang for the incident's entire ~11-hour +# duration -- `docker inspect`'s `.State.Health.Status` genuinely flips to +# "unhealthy" -- but nothing was ever watching that status and acting on it, +# so the outage went unrecovered until a human happened to notice and ran +# `docker restart` by hand. +# +# Talks to the EXISTING docker-socket-proxy (already scoped with +# CONTAINERS=1 + ALLOW_RESTARTS=1 for the api service's own diagnostics +# recovery actions -- see docker/docker-socket-proxy/README.md) rather than +# mounting the raw host docker.sock into a new container: zero new grants +# needed, no new attack surface, same security boundary the api service +# itself already relies on. +set -eu + +DOCKER_API="${WATCHDOG_DOCKER_API:-http://docker-socket-proxy:2375}" +TARGET="${WATCHDOG_TARGET_CONTAINER:-argus-api}" +INTERVAL="${WATCHDOG_INTERVAL_SECONDS:-30}" +THRESHOLD="${WATCHDOG_UNHEALTHY_THRESHOLD:-5}" + +log() { + printf '[watchdog] %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +log "starting -- target=$TARGET interval=${INTERVAL}s threshold=$THRESHOLD checks (~$((INTERVAL * THRESHOLD))s sustained unhealthy before acting)" + +consecutive=0 +while true; do + sleep "$INTERVAL" + + body="$(curl -sf --max-time 10 "$DOCKER_API/containers/$TARGET/json" 2>/dev/null || true)" + if [ -z "$body" ]; then + log "WARNING: could not reach docker-socket-proxy or inspect $TARGET -- leaving consecutive count unchanged" + continue + fi + status="$(printf '%s' "$body" | jq -r '.State.Health.Status // "none"' 2>/dev/null || echo "parse_error")" + + case "$status" in + healthy) + if [ "$consecutive" -gt 0 ]; then + log "recovered on its own after $consecutive unhealthy check(s) -- resetting counter" + fi + consecutive=0 + ;; + unhealthy) + consecutive=$((consecutive + 1)) + log "unhealthy check $consecutive/$THRESHOLD" + if [ "$consecutive" -ge "$THRESHOLD" ]; then + log "sustained unhealthy for $consecutive consecutive checks -- restarting $TARGET" + if curl -sf --max-time 30 -X POST "$DOCKER_API/containers/$TARGET/restart?t=30" >/dev/null 2>&1; then + log "restart request accepted" + else + log "WARNING: restart request failed -- will keep retrying every $INTERVAL" + fi + consecutive=0 + fi + ;; + starting | none) + # "starting" = still inside start_period (normal on boot/redeploy); + # "none" = the target has no HEALTHCHECK at all -- either way, not + # this watchdog's concern. + consecutive=0 + ;; + *) + log "unexpected/unparseable health status ($status) -- leaving consecutive count unchanged" + ;; + esac +done