root
72db2b9718
The 2026-09-12 CT 101 502 incident fix (api-watchdog: detects the API container hanging while docker still reports it healthy, and restarts it) only ever landed in argus-appliance's docker/docker-compose.yml, the source-build dev compose. It never reached this repo, which is what update.sh/install.sh on every real Docker-based install actually use -- so the mitigation never shipped to a real install. Ported the service (built locally from a bundled Dockerfile, since there is no released, cosign-verified argus-api-watchdog image yet) plus its build context.
74 lines
2.9 KiB
Bash
74 lines
2.9 KiB
Bash
#!/bin/sh
|
|
# api-watchdog: closes a real gap found during a 2026-09-11/12 production
|
|
# incident (see docker/watchdog/README.md) -- Docker's own `restart:
|
|
# unless-stopped` policy only ever acts on a container actually EXITING; it
|
|
# does nothing for a container that hangs/wedges while `docker ps` still
|
|
# reports it "Up". The api service's HEALTHCHECK (wget against /health)
|
|
# correctly kept detecting the hang for the incident's entire ~11-hour
|
|
# duration -- `docker inspect`'s `.State.Health.Status` genuinely flips to
|
|
# "unhealthy" -- but nothing was ever watching that status and acting on it,
|
|
# so the outage went unrecovered until a human happened to notice and ran
|
|
# `docker restart` by hand.
|
|
#
|
|
# Talks to the EXISTING docker-socket-proxy (already scoped with
|
|
# CONTAINERS=1 + ALLOW_RESTARTS=1 for the api service's own diagnostics
|
|
# recovery actions -- see docker/docker-socket-proxy/README.md) rather than
|
|
# mounting the raw host docker.sock into a new container: zero new grants
|
|
# needed, no new attack surface, same security boundary the api service
|
|
# itself already relies on.
|
|
set -eu
|
|
|
|
DOCKER_API="${WATCHDOG_DOCKER_API:-http://docker-socket-proxy:2375}"
|
|
TARGET="${WATCHDOG_TARGET_CONTAINER:-argus-api}"
|
|
INTERVAL="${WATCHDOG_INTERVAL_SECONDS:-30}"
|
|
THRESHOLD="${WATCHDOG_UNHEALTHY_THRESHOLD:-5}"
|
|
|
|
log() {
|
|
printf '[watchdog] %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
|
|
}
|
|
|
|
log "starting -- target=$TARGET interval=${INTERVAL}s threshold=$THRESHOLD checks (~$((INTERVAL * THRESHOLD))s sustained unhealthy before acting)"
|
|
|
|
consecutive=0
|
|
while true; do
|
|
sleep "$INTERVAL"
|
|
|
|
body="$(curl -sf --max-time 10 "$DOCKER_API/containers/$TARGET/json" 2>/dev/null || true)"
|
|
if [ -z "$body" ]; then
|
|
log "WARNING: could not reach docker-socket-proxy or inspect $TARGET -- leaving consecutive count unchanged"
|
|
continue
|
|
fi
|
|
status="$(printf '%s' "$body" | jq -r '.State.Health.Status // "none"' 2>/dev/null || echo "parse_error")"
|
|
|
|
case "$status" in
|
|
healthy)
|
|
if [ "$consecutive" -gt 0 ]; then
|
|
log "recovered on its own after $consecutive unhealthy check(s) -- resetting counter"
|
|
fi
|
|
consecutive=0
|
|
;;
|
|
unhealthy)
|
|
consecutive=$((consecutive + 1))
|
|
log "unhealthy check $consecutive/$THRESHOLD"
|
|
if [ "$consecutive" -ge "$THRESHOLD" ]; then
|
|
log "sustained unhealthy for $consecutive consecutive checks -- restarting $TARGET"
|
|
if curl -sf --max-time 30 -X POST "$DOCKER_API/containers/$TARGET/restart?t=30" >/dev/null 2>&1; then
|
|
log "restart request accepted"
|
|
else
|
|
log "WARNING: restart request failed -- will keep retrying every $INTERVAL"
|
|
fi
|
|
consecutive=0
|
|
fi
|
|
;;
|
|
starting | none)
|
|
# "starting" = still inside start_period (normal on boot/redeploy);
|
|
# "none" = the target has no HEALTHCHECK at all -- either way, not
|
|
# this watchdog's concern.
|
|
consecutive=0
|
|
;;
|
|
*)
|
|
log "unexpected/unparseable health status ($status) -- leaving consecutive count unchanged"
|
|
;;
|
|
esac
|
|
done
|