fix: port api-watchdog mitigation into the real installer compose
The 2026-09-12 CT 101 502 incident fix (api-watchdog: detects the API container hanging while docker still reports it healthy, and restarts it) only ever landed in argus-appliance's docker/docker-compose.yml, the source-build dev compose. It never reached this repo, which is what update.sh/install.sh on every real Docker-based install actually use -- so the mitigation never shipped to a real install. Ported the service (built locally from a bundled Dockerfile, since there is no released, cosign-verified argus-api-watchdog image yet) plus its build context.
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
# api-watchdog: closes a real gap found during a 2026-09-11/12 production
|
||||
# incident (see docker/watchdog/README.md) -- Docker's own `restart:
|
||||
# unless-stopped` policy only ever acts on a container actually EXITING; it
|
||||
# does nothing for a container that hangs/wedges while `docker ps` still
|
||||
# reports it "Up". The api service's HEALTHCHECK (wget against /health)
|
||||
# correctly kept detecting the hang for the incident's entire ~11-hour
|
||||
# duration -- `docker inspect`'s `.State.Health.Status` genuinely flips to
|
||||
# "unhealthy" -- but nothing was ever watching that status and acting on it,
|
||||
# so the outage went unrecovered until a human happened to notice and ran
|
||||
# `docker restart` by hand.
|
||||
#
|
||||
# Talks to the EXISTING docker-socket-proxy (already scoped with
|
||||
# CONTAINERS=1 + ALLOW_RESTARTS=1 for the api service's own diagnostics
|
||||
# recovery actions -- see docker/docker-socket-proxy/README.md) rather than
|
||||
# mounting the raw host docker.sock into a new container: zero new grants
|
||||
# needed, no new attack surface, same security boundary the api service
|
||||
# itself already relies on.
|
||||
set -eu
|
||||
|
||||
DOCKER_API="${WATCHDOG_DOCKER_API:-http://docker-socket-proxy:2375}"
|
||||
TARGET="${WATCHDOG_TARGET_CONTAINER:-argus-api}"
|
||||
INTERVAL="${WATCHDOG_INTERVAL_SECONDS:-30}"
|
||||
THRESHOLD="${WATCHDOG_UNHEALTHY_THRESHOLD:-5}"
|
||||
|
||||
log() {
|
||||
printf '[watchdog] %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
|
||||
}
|
||||
|
||||
log "starting -- target=$TARGET interval=${INTERVAL}s threshold=$THRESHOLD checks (~$((INTERVAL * THRESHOLD))s sustained unhealthy before acting)"
|
||||
|
||||
consecutive=0
|
||||
while true; do
|
||||
sleep "$INTERVAL"
|
||||
|
||||
body="$(curl -sf --max-time 10 "$DOCKER_API/containers/$TARGET/json" 2>/dev/null || true)"
|
||||
if [ -z "$body" ]; then
|
||||
log "WARNING: could not reach docker-socket-proxy or inspect $TARGET -- leaving consecutive count unchanged"
|
||||
continue
|
||||
fi
|
||||
status="$(printf '%s' "$body" | jq -r '.State.Health.Status // "none"' 2>/dev/null || echo "parse_error")"
|
||||
|
||||
case "$status" in
|
||||
healthy)
|
||||
if [ "$consecutive" -gt 0 ]; then
|
||||
log "recovered on its own after $consecutive unhealthy check(s) -- resetting counter"
|
||||
fi
|
||||
consecutive=0
|
||||
;;
|
||||
unhealthy)
|
||||
consecutive=$((consecutive + 1))
|
||||
log "unhealthy check $consecutive/$THRESHOLD"
|
||||
if [ "$consecutive" -ge "$THRESHOLD" ]; then
|
||||
log "sustained unhealthy for $consecutive consecutive checks -- restarting $TARGET"
|
||||
if curl -sf --max-time 30 -X POST "$DOCKER_API/containers/$TARGET/restart?t=30" >/dev/null 2>&1; then
|
||||
log "restart request accepted"
|
||||
else
|
||||
log "WARNING: restart request failed -- will keep retrying every $INTERVAL"
|
||||
fi
|
||||
consecutive=0
|
||||
fi
|
||||
;;
|
||||
starting | none)
|
||||
# "starting" = still inside start_period (normal on boot/redeploy);
|
||||
# "none" = the target has no HEALTHCHECK at all -- either way, not
|
||||
# this watchdog's concern.
|
||||
consecutive=0
|
||||
;;
|
||||
*)
|
||||
log "unexpected/unparseable health status ($status) -- leaving consecutive count unchanged"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
Reference in New Issue
Block a user