#!/bin/sh
# ath12k (WCN7850) firmware-wedge watchdog.
#
# Failure mode (observed live on photonicat2, 2026-08): the WCN7850 firmware
# stops answering WMI stats requests. dmesg fills with
#   "ath12k_pci 0000:01:00.0: time out while waiting for fw stats done"
# and every process touching wireless/netlink — rpcd, iwinfo, even
# `ip route` — hangs in unkillable D-state because the dead request holds
# the wiphy/rtnl locks. The web UI stalls, load average climbs with an idle
# CPU, and nothing self-heals.
#
# Recovery (field-proven on the same incident): PCI remove + rescan of the
# card, then `wifi up`. Deliberately NO `wifi down` first — teardown paths
# take the very locks that are wedged and would hang this watchdog too.
# The hot remove releases the stuck waiters, the rescan reprobes the
# firmware from scratch, and the network config layer re-arms the radios
# (the phy index may change across the rescan, which is why everything here
# keys on the PCI address, never on phy names).
#
# This busybox has no `timeout` applet; hangs are detected by backgrounding
# the probe and reaping it by hand.

PCI_ADDR="${PCI_ADDR:-0000:01:00.0}"
CHECK_INTERVAL=60
# Trigger threshold: this many fw-stats timeouts in the recent dmesg tail.
# The wedge spams one every ~3 s, so a real incident hits this within a
# minute; scattered historic lines don't.
SPAM_MIN=3
# At most one recovery attempt per this many seconds, so a card that wedges
# right back doesn't get bounced in a tight loop.
COOLDOWN=600
IWINFO_LIMIT=10

wireless_responsive() {
	iwinfo >/dev/null 2>&1 &
	pid=$!
	i=0
	while [ "$i" -lt "$IWINFO_LIMIT" ]; do
		kill -0 "$pid" 2>/dev/null || { wait "$pid" 2>/dev/null; return 0; }
		sleep 1
		i=$((i + 1))
	done
	kill -9 "$pid" 2>/dev/null
	return 1
}

last_fix=-$COOLDOWN
while sleep "$CHECK_INTERVAL"; do
	spam=$(dmesg | tail -40 | grep -c "time out while waiting for fw stats")
	[ "$spam" -ge "$SPAM_MIN" ] || continue
	wireless_responsive && continue
	now=$(cut -d. -f1 /proc/uptime)
	[ $((now - last_fix)) -ge "$COOLDOWN" ] || continue
	last_fix=$now
	logger -t pcat-wifi-watchdog \
		"ath12k firmware wedged (fw-stats spam=$spam, iwinfo hung); recovering $PCI_ADDR via PCI rescan"
	if [ ! -e "/sys/bus/pci/devices/$PCI_ADDR/remove" ]; then
		logger -t pcat-wifi-watchdog "$PCI_ADDR not present, cannot recover"
		continue
	fi
	echo 1 > "/sys/bus/pci/devices/$PCI_ADDR/remove"
	sleep 3
	echo 1 > /sys/bus/pci/rescan
	sleep 10
	wifi up
	sleep 20
	if wireless_responsive; then
		logger -t pcat-wifi-watchdog "recovery succeeded, wireless responsive again"
	else
		logger -t pcat-wifi-watchdog "recovery attempt finished but wireless still unresponsive"
	fi
done
