#!/bin/sh
# api-watchdog: closes a real gap found during a 2026-09-11/12 production
# incident (see docker/watchdog/README.md) -- Docker's own `restart:
# unless-stopped` policy only ever acts on a container actually EXITING; it
# does nothing for a container that hangs/wedges while `docker ps` still
# reports it "Up". The api service's HEALTHCHECK (wget against /health)
# correctly kept detecting the hang for the incident's entire ~11-hour
# duration -- `docker inspect`'s `.State.Health.Status` genuinely flips to
# "unhealthy" -- but nothing was ever watching that status and acting on it,
# so the outage went unrecovered until a human happened to notice and ran
# `docker restart` by hand.
#
# Talks to the EXISTING docker-socket-proxy (already scoped with
# CONTAINERS=1 + ALLOW_RESTARTS=1 for the api service's own diagnostics
# recovery actions -- see docker/docker-socket-proxy/README.md) rather than
# mounting the raw host docker.sock into a new container: zero new grants
# needed, no new attack surface, same security boundary the api service
# itself already relies on.
set -eu

DOCKER_API="${WATCHDOG_DOCKER_API:-http://docker-socket-proxy:2375}"
TARGET="${WATCHDOG_TARGET_CONTAINER:-argus-api}"
INTERVAL="${WATCHDOG_INTERVAL_SECONDS:-30}"
THRESHOLD="${WATCHDOG_UNHEALTHY_THRESHOLD:-5}"

log() {
	printf '[watchdog] %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
}

log "starting -- target=$TARGET interval=${INTERVAL}s threshold=$THRESHOLD checks (~$((INTERVAL * THRESHOLD))s sustained unhealthy before acting)"

consecutive=0
while true; do
	sleep "$INTERVAL"

	body="$(curl -sf --max-time 10 "$DOCKER_API/containers/$TARGET/json" 2>/dev/null || true)"
	if [ -z "$body" ]; then
		log "WARNING: could not reach docker-socket-proxy or inspect $TARGET -- leaving consecutive count unchanged"
		continue
	fi
	status="$(printf '%s' "$body" | jq -r '.State.Health.Status // "none"' 2>/dev/null || echo "parse_error")"

	case "$status" in
	healthy)
		if [ "$consecutive" -gt 0 ]; then
			log "recovered on its own after $consecutive unhealthy check(s) -- resetting counter"
		fi
		consecutive=0
		;;
	unhealthy)
		consecutive=$((consecutive + 1))
		log "unhealthy check $consecutive/$THRESHOLD"
		if [ "$consecutive" -ge "$THRESHOLD" ]; then
			log "sustained unhealthy for $consecutive consecutive checks -- restarting $TARGET"
			if curl -sf --max-time 30 -X POST "$DOCKER_API/containers/$TARGET/restart?t=30" >/dev/null 2>&1; then
				log "restart request accepted"
			else
				log "WARNING: restart request failed -- will keep retrying every $INTERVAL"
			fi
			consecutive=0
		fi
		;;
	starting | none)
		# "starting" = still inside start_period (normal on boot/redeploy);
		# "none" = the target has no HEALTHCHECK at all -- either way, not
		# this watchdog's concern.
		consecutive=0
		;;
	*)
		log "unexpected/unparseable health status ($status) -- leaving consecutive count unchanged"
		;;
	esac
done