From c0623bd320de236898d0f5c257399fa6f953f947 Mon Sep 17 00:00:00 2001
From: Michael Buntarman <michaelboentarman@gmail.com>
Date: Fri, 6 Nov 2026 11:43:28 +0700
Subject: [PATCH 1/5] feat: get a Telegram alert when Postgres runs a server
 program

---
 deployments/alerts/README.md           | 40 ++++++++++++++++
 deployments/alerts/install.sh          | 17 +++++++
 deployments/alerts/pg-program-alert.sh | 64 ++++++++++++++++++++++++++
 deployments/alerts/tn-pg-alert.service |  8 ++++
 deployments/alerts/tn-pg-alert.timer   |  9 ++++
 5 files changed, 138 insertions(+)
 create mode 100644 deployments/alerts/README.md
 create mode 100755 deployments/alerts/install.sh
 create mode 100755 deployments/alerts/pg-program-alert.sh
 create mode 100644 deployments/alerts/tn-pg-alert.service
 create mode 100644 deployments/alerts/tn-pg-alert.timer

diff --git a/deployments/alerts/README.md b/deployments/alerts/README.md
new file mode 100644
index 00000000..1aab9bb1
--- /dev/null
+++ b/deployments/alerts/README.md
@@ -0,0 +1,40 @@
+# Postgres program alert
+
+`tn-pg-alert` sends a Telegram message when a node's Postgres container logs a statement that runs a server program, such as `COPY ... TO PROGRAM`. kwild's database role may not run programs (see `deployments/postgres-roles.sql`), so Postgres refuses the statement and logs it. Any such line means something tried, and it is worth a look.
+
+A systemd timer runs the check every 5 minutes. Each run reads the container's log lines written since the previous run, so nothing is counted twice.
+
+## Install on a node
+
+From a checkout of this repository on the node:
+
+```bash
+cd deployments/alerts
+sudo ./install.sh
+```
+
+The script installs `/usr/local/bin/tn-pg-alert`, the `tn-pg-alert` service and timer, and an empty settings file `/etc/tn-alert/telegram.env` (mode 600). Until the token is set, alerts go to the system journal only (`journalctl -t tn-pg-alert`).
+
+## Set the Telegram token
+
+1. Create a bot with [@BotFather](https://t.me/BotFather) and copy its token.
+2. Add the bot to the chat that should receive alerts and find the chat id, for example from `https://api.telegram.org/bot<token>/getUpdates` after posting a message in the chat.
+3. On each node, put both in `/etc/tn-alert/telegram.env`:
+
+   ```bash
+   TELEGRAM_BOT_TOKEN=123456:ABC...
+   TELEGRAM_CHAT_ID=-1001234567890
+   ```
+
+4. Check that a message arrives: `sudo tn-pg-alert --test`
+
+No restart is needed. The next timer run reads the file.
+
+## Settings
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `TELEGRAM_BOT_TOKEN` | empty | Bot token from @BotFather |
+| `TELEGRAM_CHAT_ID` | empty | Chat that receives the alerts |
+| `PG_CONTAINER` | `tn-postgres` | Name of the node's Postgres container |
+| `NODE_LABEL` | the hostname | Node name shown in the message |
diff --git a/deployments/alerts/install.sh b/deployments/alerts/install.sh
new file mode 100755
index 00000000..54e9f685
--- /dev/null
+++ b/deployments/alerts/install.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Installs tn-pg-alert on a node. Run it as root from this directory:
+#   sudo ./install.sh
+# It is safe to run again and keeps an existing /etc/tn-alert/telegram.env.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+install -m 755 pg-program-alert.sh /usr/local/bin/tn-pg-alert
+install -m 644 tn-pg-alert.service tn-pg-alert.timer /etc/systemd/system/
+install -d -m 700 /etc/tn-alert
+if [ ! -e /etc/tn-alert/telegram.env ]; then
+	install -m 600 /dev/null /etc/tn-alert/telegram.env
+	printf '%s\n' 'TELEGRAM_BOT_TOKEN=' 'TELEGRAM_CHAT_ID=' '# PG_CONTAINER=tn-postgres' '# NODE_LABEL=' >/etc/tn-alert/telegram.env
+fi
+systemctl daemon-reload
+systemctl enable --now tn-pg-alert.timer
+echo "tn-pg-alert installed. Put the bot token and chat id in /etc/tn-alert/telegram.env, then run: sudo tn-pg-alert --test"
diff --git a/deployments/alerts/pg-program-alert.sh b/deployments/alerts/pg-program-alert.sh
new file mode 100755
index 00000000..78c6e8e4
--- /dev/null
+++ b/deployments/alerts/pg-program-alert.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Sends a Telegram message when the node's Postgres container logs a statement
+# that runs a server program (COPY ... TO PROGRAM or FROM PROGRAM). kwild's
+# database role may not run programs, so Postgres refuses such a statement and
+# logs it. Any such line means something tried.
+#
+# Settings come from /etc/tn-alert/telegram.env (see README.md):
+#   TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID  where the message goes
+#   PG_CONTAINER                          Postgres container, default tn-postgres
+#   NODE_LABEL                            node name in the message, default the hostname
+# Without a token and a chat id, the alert goes to the system journal only.
+#
+# Usage: tn-pg-alert          check the logs written since the last run
+#        tn-pg-alert --test   send a test message
+set -euo pipefail
+
+ENV_FILE=${TN_ALERT_ENV:-/etc/tn-alert/telegram.env}
+STATE_DIR=${TN_ALERT_STATE:-/var/lib/tn-alert}
+if [ -r "$ENV_FILE" ]; then
+	# shellcheck disable=SC1090
+	. "$ENV_FILE"
+fi
+PG_CONTAINER=${PG_CONTAINER:-tn-postgres}
+NODE_LABEL=${NODE_LABEL:-$(hostname)}
+
+send() {
+	local text=$1 code
+	logger -t tn-pg-alert -- "$text" || true
+	if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then
+		echo "tn-pg-alert: no Telegram token or chat id set, wrote the alert to the journal only" >&2
+		return 0
+	fi
+	# The URL holds the token, so it goes through stdin, not the command line.
+	code=$(printf 'url = "https://api.telegram.org/bot%s/sendMessage"\n' "$TELEGRAM_BOT_TOKEN" |
+		curl -sS -m 20 -o /dev/null -w '%{http_code}' -K - \
+			--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
+			--data-urlencode "text=${text}") || code=000
+	if [ "$code" != 200 ]; then
+		echo "tn-pg-alert: Telegram answered HTTP $code" >&2
+		return 1
+	fi
+}
+
+if [ "${1:-}" = "--test" ]; then
+	send "${NODE_LABEL}: test message from tn-pg-alert"
+	exit
+fi
+
+mkdir -p "$STATE_DIR"
+now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+since=$(cat "$STATE_DIR/last-run" 2>/dev/null || date -u -d '-10 min' +%Y-%m-%dT%H:%M:%SZ)
+
+if ! logs=$(docker logs --since "$since" --until "$now" "$PG_CONTAINER" 2>&1); then
+	send "${NODE_LABEL}: tn-pg-alert cannot read the logs of ${PG_CONTAINER}"
+	exit 1
+fi
+
+count=$(printf '%s\n' "$logs" | grep -c -i -E '(to|from)[[:space:]]+program' || true)
+if [ "$count" -gt 0 ]; then
+	# The last run time only moves once the message is out, so a failed send
+	# is retried on the next run.
+	send "${NODE_LABEL}: Postgres logged ${count} line(s) with TO PROGRAM between ${since} and ${now}. Read them with: docker logs --since ${since} --until ${now} ${PG_CONTAINER} 2>&1 | grep -i program"
+fi
+echo "$now" >"$STATE_DIR/last-run"
diff --git a/deployments/alerts/tn-pg-alert.service b/deployments/alerts/tn-pg-alert.service
new file mode 100644
index 00000000..9721d479
--- /dev/null
+++ b/deployments/alerts/tn-pg-alert.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=Alert when Postgres logs a statement that runs a server program
+After=docker.service
+Wants=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/tn-pg-alert
diff --git a/deployments/alerts/tn-pg-alert.timer b/deployments/alerts/tn-pg-alert.timer
new file mode 100644
index 00000000..158c5132
--- /dev/null
+++ b/deployments/alerts/tn-pg-alert.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Run tn-pg-alert every 5 minutes
+
+[Timer]
+OnCalendar=*:0/5
+Persistent=true
+
+[Install]
+WantedBy=timers.target

From edbcd148d27b70fe50968a687073e57b64824ba5 Mon Sep 17 00:00:00 2001
From: Michael Buntarman <michaelboentarman@gmail.com>
Date: Fri, 6 Nov 2026 12:25:29 +0700
Subject: [PATCH 2/5] feat(alerts): send to a Telegram topic when one is set

---
 deployments/alerts/README.md           | 3 +++
 deployments/alerts/install.sh          | 2 +-
 deployments/alerts/pg-program-alert.sh | 8 ++++++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/deployments/alerts/README.md b/deployments/alerts/README.md
index 1aab9bb1..6371269b 100644
--- a/deployments/alerts/README.md
+++ b/deployments/alerts/README.md
@@ -24,6 +24,8 @@ The script installs `/usr/local/bin/tn-pg-alert`, the `tn-pg-alert` service and
    ```bash
    TELEGRAM_BOT_TOKEN=123456:ABC...
    TELEGRAM_CHAT_ID=-1001234567890
+   # only for a chat with topics:
+   TELEGRAM_TOPIC_ID=42
    ```
 
 4. Check that a message arrives: `sudo tn-pg-alert --test`
@@ -36,5 +38,6 @@ No restart is needed. The next timer run reads the file.
 |---|---|---|
 | `TELEGRAM_BOT_TOKEN` | empty | Bot token from @BotFather |
 | `TELEGRAM_CHAT_ID` | empty | Chat that receives the alerts |
+| `TELEGRAM_TOPIC_ID` | empty | Topic in a chat with topics, where the alerts go |
 | `PG_CONTAINER` | `tn-postgres` | Name of the node's Postgres container |
 | `NODE_LABEL` | the hostname | Node name shown in the message |
diff --git a/deployments/alerts/install.sh b/deployments/alerts/install.sh
index 54e9f685..3a8b6d49 100755
--- a/deployments/alerts/install.sh
+++ b/deployments/alerts/install.sh
@@ -10,7 +10,7 @@ install -m 644 tn-pg-alert.service tn-pg-alert.timer /etc/systemd/system/
 install -d -m 700 /etc/tn-alert
 if [ ! -e /etc/tn-alert/telegram.env ]; then
 	install -m 600 /dev/null /etc/tn-alert/telegram.env
-	printf '%s\n' 'TELEGRAM_BOT_TOKEN=' 'TELEGRAM_CHAT_ID=' '# PG_CONTAINER=tn-postgres' '# NODE_LABEL=' >/etc/tn-alert/telegram.env
+	printf '%s\n' 'TELEGRAM_BOT_TOKEN=' 'TELEGRAM_CHAT_ID=' '# TELEGRAM_TOPIC_ID=' '# PG_CONTAINER=tn-postgres' '# NODE_LABEL=' >/etc/tn-alert/telegram.env
 fi
 systemctl daemon-reload
 systemctl enable --now tn-pg-alert.timer
diff --git a/deployments/alerts/pg-program-alert.sh b/deployments/alerts/pg-program-alert.sh
index 78c6e8e4..51bb54bf 100755
--- a/deployments/alerts/pg-program-alert.sh
+++ b/deployments/alerts/pg-program-alert.sh
@@ -6,6 +6,7 @@
 #
 # Settings come from /etc/tn-alert/telegram.env (see README.md):
 #   TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID  where the message goes
+#   TELEGRAM_TOPIC_ID                     topic (thread) in a forum chat, optional
 #   PG_CONTAINER                          Postgres container, default tn-postgres
 #   NODE_LABEL                            node name in the message, default the hostname
 # Without a token and a chat id, the alert goes to the system journal only.
@@ -24,16 +25,19 @@ PG_CONTAINER=${PG_CONTAINER:-tn-postgres}
 NODE_LABEL=${NODE_LABEL:-$(hostname)}
 
 send() {
-	local text=$1 code
+	local text=$1 code topic=()
 	logger -t tn-pg-alert -- "$text" || true
 	if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then
 		echo "tn-pg-alert: no Telegram token or chat id set, wrote the alert to the journal only" >&2
 		return 0
 	fi
+	if [ -n "${TELEGRAM_TOPIC_ID:-}" ]; then
+		topic=(--data-urlencode "message_thread_id=${TELEGRAM_TOPIC_ID}")
+	fi
 	# The URL holds the token, so it goes through stdin, not the command line.
 	code=$(printf 'url = "https://api.telegram.org/bot%s/sendMessage"\n' "$TELEGRAM_BOT_TOKEN" |
 		curl -sS -m 20 -o /dev/null -w '%{http_code}' -K - \
-			--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
+			--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" ${topic[@]+"${topic[@]}"} \
 			--data-urlencode "text=${text}") || code=000
 	if [ "$code" != 200 ]; then
 		echo "tn-pg-alert: Telegram answered HTTP $code" >&2

From 6e65540a1a2a08423465bc8dbb46456372b74bb4 Mon Sep 17 00:00:00 2001
From: Michael Buntarman <michaelboentarman@gmail.com>
Date: Fri, 6 Nov 2026 12:38:00 +0700
Subject: [PATCH 3/5] fix(alerts): harden the scan, checkpoint and settings
 file

---
 deployments/alerts/install.sh          | 11 ++++++++---
 deployments/alerts/pg-program-alert.sh | 27 ++++++++++++++++++--------
 deployments/alerts/tn-pg-alert.service |  2 ++
 3 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/deployments/alerts/install.sh b/deployments/alerts/install.sh
index 3a8b6d49..2d133c9a 100755
--- a/deployments/alerts/install.sh
+++ b/deployments/alerts/install.sh
@@ -7,9 +7,14 @@ cd "$(dirname "$0")"
 
 install -m 755 pg-program-alert.sh /usr/local/bin/tn-pg-alert
 install -m 644 tn-pg-alert.service tn-pg-alert.timer /etc/systemd/system/
-install -d -m 700 /etc/tn-alert
-if [ ! -e /etc/tn-alert/telegram.env ]; then
-	install -m 600 /dev/null /etc/tn-alert/telegram.env
+install -d -m 700 -o root -g root /etc/tn-alert
+chown root:root /etc/tn-alert
+chmod 700 /etc/tn-alert
+if [ -e /etc/tn-alert/telegram.env ]; then
+	chown root:root /etc/tn-alert/telegram.env
+	chmod 600 /etc/tn-alert/telegram.env
+else
+	install -m 600 -o root -g root /dev/null /etc/tn-alert/telegram.env
 	printf '%s\n' 'TELEGRAM_BOT_TOKEN=' 'TELEGRAM_CHAT_ID=' '# TELEGRAM_TOPIC_ID=' '# PG_CONTAINER=tn-postgres' '# NODE_LABEL=' >/etc/tn-alert/telegram.env
 fi
 systemctl daemon-reload
diff --git a/deployments/alerts/pg-program-alert.sh b/deployments/alerts/pg-program-alert.sh
index 51bb54bf..f20320c2 100755
--- a/deployments/alerts/pg-program-alert.sh
+++ b/deployments/alerts/pg-program-alert.sh
@@ -25,11 +25,11 @@ PG_CONTAINER=${PG_CONTAINER:-tn-postgres}
 NODE_LABEL=${NODE_LABEL:-$(hostname)}
 
 send() {
-	local text=$1 code topic=()
-	logger -t tn-pg-alert -- "$text" || true
+	local text=$1 code topic=() logged=0
+	logger -t tn-pg-alert -- "$text" || logged=1
 	if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then
 		echo "tn-pg-alert: no Telegram token or chat id set, wrote the alert to the journal only" >&2
-		return 0
+		return "$logged"
 	fi
 	if [ -n "${TELEGRAM_TOPIC_ID:-}" ]; then
 		topic=(--data-urlencode "message_thread_id=${TELEGRAM_TOPIC_ID}")
@@ -52,17 +52,28 @@ fi
 
 mkdir -p "$STATE_DIR"
 now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
-since=$(cat "$STATE_DIR/last-run" 2>/dev/null || date -u -d '-10 min' +%Y-%m-%dT%H:%M:%SZ)
+since=$(cat "$STATE_DIR/last-run" 2>/dev/null || true)
+if ! [[ $since =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
+	since=$(date -u -d '-10 min' +%Y-%m-%dT%H:%M:%SZ)
+fi
 
-if ! logs=$(docker logs --since "$since" --until "$now" "$PG_CONTAINER" 2>&1); then
+# A statement can be split over several log lines, so the scan also matches the
+# fixed message Postgres logs when it refuses one. Only matching lines are kept.
+pattern='(to|from)[[:space:]]+program|external program|pg_execute_server_program'
+if ! matches=$(
+	docker logs --since "$since" --until "$now" "$PG_CONTAINER" 2>&1 | grep -i -E "$pattern"
+	exit "${PIPESTATUS[0]}"
+); then
 	send "${NODE_LABEL}: tn-pg-alert cannot read the logs of ${PG_CONTAINER}"
 	exit 1
 fi
 
-count=$(printf '%s\n' "$logs" | grep -c -i -E '(to|from)[[:space:]]+program' || true)
+count=$(printf '%s' "$matches" | grep -c . || true)
 if [ "$count" -gt 0 ]; then
 	# The last run time only moves once the message is out, so a failed send
 	# is retried on the next run.
-	send "${NODE_LABEL}: Postgres logged ${count} line(s) with TO PROGRAM between ${since} and ${now}. Read them with: docker logs --since ${since} --until ${now} ${PG_CONTAINER} 2>&1 | grep -i program"
+	send "${NODE_LABEL}: Postgres logged ${count} line(s) about running a server program between ${since} and ${now}. Read them with: docker logs --since ${since} --until ${now} ${PG_CONTAINER} 2>&1 | grep -i program"
 fi
-echo "$now" >"$STATE_DIR/last-run"
+printf '%s\n' "$now" >"$STATE_DIR/last-run.tmp"
+sync "$STATE_DIR/last-run.tmp" 2>/dev/null || true
+mv -f "$STATE_DIR/last-run.tmp" "$STATE_DIR/last-run"
diff --git a/deployments/alerts/tn-pg-alert.service b/deployments/alerts/tn-pg-alert.service
index 9721d479..340521f1 100644
--- a/deployments/alerts/tn-pg-alert.service
+++ b/deployments/alerts/tn-pg-alert.service
@@ -6,3 +6,5 @@ Wants=docker.service
 [Service]
 Type=oneshot
 ExecStart=/usr/local/bin/tn-pg-alert
+# A stuck run would block every later one, so it gets a time limit.
+TimeoutStartSec=4min

From 97cd62419d72f4ec96ac9e3c5a81e71880e72a2d Mon Sep 17 00:00:00 2001
From: Michael Buntarman <michaelboentarman@gmail.com>
Date: Fri, 6 Nov 2026 12:48:26 +0700
Subject: [PATCH 4/5] feat(alerts): alert when a node stops making blocks or
 crashes

---
 deployments/alerts/README.md             |  24 ++++-
 deployments/alerts/install.sh            |   9 +-
 deployments/alerts/node-health-alert.sh  | 124 +++++++++++++++++++++++
 deployments/alerts/tn-node-alert.service |  10 ++
 deployments/alerts/tn-node-alert.timer   |   9 ++
 5 files changed, 168 insertions(+), 8 deletions(-)
 create mode 100755 deployments/alerts/node-health-alert.sh
 create mode 100644 deployments/alerts/tn-node-alert.service
 create mode 100644 deployments/alerts/tn-node-alert.timer

diff --git a/deployments/alerts/README.md b/deployments/alerts/README.md
index 6371269b..c1423177 100644
--- a/deployments/alerts/README.md
+++ b/deployments/alerts/README.md
@@ -1,8 +1,20 @@
-# Postgres program alert
+# Node alerts
+
+Two small alerts send Telegram messages from each node. Both read the same settings file and run every 5 minutes from a systemd timer.
+
+## tn-pg-alert: Postgres runs a program
 
 `tn-pg-alert` sends a Telegram message when a node's Postgres container logs a statement that runs a server program, such as `COPY ... TO PROGRAM`. kwild's database role may not run programs (see `deployments/postgres-roles.sql`), so Postgres refuses the statement and logs it. Any such line means something tried, and it is worth a look.
 
-A systemd timer runs the check every 5 minutes. Each run reads the container's log lines written since the previous run, so nothing is counted twice.
+Each run reads the container's log lines written since the previous run, so nothing is counted twice.
+
+## tn-node-alert: the node stops or crashes
+
+`tn-node-alert` sends a message when:
+
+- the node's block height has not changed for 10 minutes (`STALL_MINUTES`), including when its RPC stops answering. The message repeats every hour (`REPEAT_MINUTES`) while the stall lasts, and one more message says when blocks move again.
+- Docker restarted the kwild container on its own, which means it crashed. A manual restart or a recreate does not count.
+- the kwild container is not running.
 
 ## Install on a node
 
@@ -13,7 +25,7 @@ cd deployments/alerts
 sudo ./install.sh
 ```
 
-The script installs `/usr/local/bin/tn-pg-alert`, the `tn-pg-alert` service and timer, and an empty settings file `/etc/tn-alert/telegram.env` (mode 600). Until the token is set, alerts go to the system journal only (`journalctl -t tn-pg-alert`).
+The script installs `/usr/local/bin/tn-pg-alert` and `/usr/local/bin/tn-node-alert`, their services and timers, and an empty settings file `/etc/tn-alert/telegram.env` (mode 600). Until the token is set, alerts go to the system journal only (`journalctl -t tn-pg-alert -t tn-node-alert`).
 
 ## Set the Telegram token
 
@@ -28,7 +40,7 @@ The script installs `/usr/local/bin/tn-pg-alert`, the `tn-pg-alert` service and
    TELEGRAM_TOPIC_ID=42
    ```
 
-4. Check that a message arrives: `sudo tn-pg-alert --test`
+4. Check that messages arrive: `sudo tn-pg-alert --test` and `sudo tn-node-alert --test`
 
 No restart is needed. The next timer run reads the file.
 
@@ -40,4 +52,8 @@ No restart is needed. The next timer run reads the file.
 | `TELEGRAM_CHAT_ID` | empty | Chat that receives the alerts |
 | `TELEGRAM_TOPIC_ID` | empty | Topic in a chat with topics, where the alerts go |
 | `PG_CONTAINER` | `tn-postgres` | Name of the node's Postgres container |
+| `NODE_CONTAINER` | `tn-node` | Name of the node's kwild container |
+| `NODE_RPC` | `http://127.0.0.1:8484/rpc/v1` | The node's JSON-RPC address |
+| `STALL_MINUTES` | `10` | Minutes without a new block before the first alert |
+| `REPEAT_MINUTES` | `60` | Minutes between repeats while a stall lasts |
 | `NODE_LABEL` | the hostname | Node name shown in the message |
diff --git a/deployments/alerts/install.sh b/deployments/alerts/install.sh
index 2d133c9a..4570d6e5 100755
--- a/deployments/alerts/install.sh
+++ b/deployments/alerts/install.sh
@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
-# Installs tn-pg-alert on a node. Run it as root from this directory:
+# Installs tn-pg-alert and tn-node-alert on a node. Run it as root from this directory:
 #   sudo ./install.sh
 # It is safe to run again and keeps an existing /etc/tn-alert/telegram.env.
 set -euo pipefail
 cd "$(dirname "$0")"
 
 install -m 755 pg-program-alert.sh /usr/local/bin/tn-pg-alert
-install -m 644 tn-pg-alert.service tn-pg-alert.timer /etc/systemd/system/
+install -m 755 node-health-alert.sh /usr/local/bin/tn-node-alert
+install -m 644 tn-pg-alert.service tn-pg-alert.timer tn-node-alert.service tn-node-alert.timer /etc/systemd/system/
 install -d -m 700 -o root -g root /etc/tn-alert
 chown root:root /etc/tn-alert
 chmod 700 /etc/tn-alert
@@ -18,5 +19,5 @@ else
 	printf '%s\n' 'TELEGRAM_BOT_TOKEN=' 'TELEGRAM_CHAT_ID=' '# TELEGRAM_TOPIC_ID=' '# PG_CONTAINER=tn-postgres' '# NODE_LABEL=' >/etc/tn-alert/telegram.env
 fi
 systemctl daemon-reload
-systemctl enable --now tn-pg-alert.timer
-echo "tn-pg-alert installed. Put the bot token and chat id in /etc/tn-alert/telegram.env, then run: sudo tn-pg-alert --test"
+systemctl enable --now tn-pg-alert.timer tn-node-alert.timer
+echo "tn-pg-alert and tn-node-alert installed. Put the bot token and chat id in /etc/tn-alert/telegram.env, then run: sudo tn-pg-alert --test && sudo tn-node-alert --test"
diff --git a/deployments/alerts/node-health-alert.sh b/deployments/alerts/node-health-alert.sh
new file mode 100755
index 00000000..689c003a
--- /dev/null
+++ b/deployments/alerts/node-health-alert.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Sends a Telegram message when this node stops moving: its block height has
+# not changed for STALL_MINUTES, or Docker restarted its kwild container on its
+# own (a crash), or the container is not running. A stall alert repeats every
+# REPEAT_MINUTES while it lasts, and one message says when blocks move again.
+#
+# Settings come from /etc/tn-alert/telegram.env, shared with tn-pg-alert:
+#   TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID  where the message goes
+#   TELEGRAM_TOPIC_ID                     topic (thread) in a forum chat, optional
+#   NODE_CONTAINER                        kwild container, default tn-node
+#   NODE_RPC                              JSON-RPC URL, default http://127.0.0.1:8484/rpc/v1
+#   STALL_MINUTES                         default 10
+#   REPEAT_MINUTES                        default 60
+#   NODE_LABEL                            node name in the message, default the hostname
+# Without a token and a chat id, the alert goes to the system journal only.
+#
+# Usage: tn-node-alert          check the node
+#        tn-node-alert --test   send a test message
+set -euo pipefail
+
+ENV_FILE=${TN_ALERT_ENV:-/etc/tn-alert/telegram.env}
+STATE_DIR=${TN_ALERT_STATE:-/var/lib/tn-alert}
+if [ -r "$ENV_FILE" ]; then
+	# shellcheck disable=SC1090
+	. "$ENV_FILE"
+fi
+NODE_CONTAINER=${NODE_CONTAINER:-tn-node}
+NODE_RPC=${NODE_RPC:-http://127.0.0.1:8484/rpc/v1}
+STALL_MINUTES=${STALL_MINUTES:-10}
+REPEAT_MINUTES=${REPEAT_MINUTES:-60}
+NODE_LABEL=${NODE_LABEL:-$(hostname)}
+
+send() {
+	local text=$1 code topic=() logged=0
+	logger -t tn-node-alert -- "$text" || logged=1
+	if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then
+		echo "tn-node-alert: no Telegram token or chat id set, wrote the alert to the journal only" >&2
+		return "$logged"
+	fi
+	if [ -n "${TELEGRAM_TOPIC_ID:-}" ]; then
+		topic=(--data-urlencode "message_thread_id=${TELEGRAM_TOPIC_ID}")
+	fi
+	# The URL holds the token, so it goes through stdin, not the command line.
+	code=$(printf 'url = "https://api.telegram.org/bot%s/sendMessage"\n' "$TELEGRAM_BOT_TOKEN" |
+		curl -sS -m 20 -o /dev/null -w '%{http_code}' -K - \
+			--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" ${topic[@]+"${topic[@]}"} \
+			--data-urlencode "text=${text}") || code=000
+	if [ "$code" != 200 ]; then
+		echo "tn-node-alert: Telegram answered HTTP $code" >&2
+		return 1
+	fi
+}
+
+if [ "${1:-}" = "--test" ]; then
+	send "${NODE_LABEL}: test message from tn-node-alert"
+	exit
+fi
+
+mkdir -p "$STATE_DIR"
+state="$STATE_DIR/node-state"
+now=$(date -u +%s)
+
+# Saved state: height, when it last changed, restart count, when the last stall
+# alert went out (0 when none is open).
+last_height=0 changed_at=$now last_restarts=-1 alerted_at=0
+if [ -r "$state" ]; then
+	read -r last_height changed_at last_restarts alerted_at <"$state" || true
+fi
+for v in last_height changed_at last_restarts alerted_at; do
+	[[ ${!v} =~ ^-?[0-9]+$ ]] || printf -v "$v" '%s' 0
+done
+[ "$changed_at" -gt 0 ] || changed_at=$now
+
+failed=0
+
+# Crashes: Docker counts only the restarts its restart policy made, so a
+# manual restart or a recreate does not alert. A recreate resets the count.
+if info=$(docker inspect -f '{{.RestartCount}} {{.State.Status}} {{.State.ExitCode}}' "$NODE_CONTAINER" 2>/dev/null); then
+	read -r restarts status exit_code <<<"$info"
+	if [ "$status" != running ]; then
+		send "${NODE_LABEL}: ${NODE_CONTAINER} is not running (status ${status}, last exit code ${exit_code})." || failed=1
+	elif [ "$last_restarts" -ge 0 ] && [ "$restarts" -gt "$last_restarts" ]; then
+		send "${NODE_LABEL}: ${NODE_CONTAINER} restarted by itself $((restarts - last_restarts)) time(s) since the last check (${restarts} in total). Check: docker logs --tail 50 ${NODE_CONTAINER}" || failed=1
+	fi
+else
+	restarts=$last_restarts
+	send "${NODE_LABEL}: cannot inspect the container ${NODE_CONTAINER}." || failed=1
+fi
+
+# Height: a node that does not answer keeps its last height, so it counts as
+# stalled too.
+height=$(curl -s -m 10 -X POST "$NODE_RPC" -H 'content-type: application/json' \
+	-d '{"jsonrpc":"2.0","id":1,"method":"user.health","params":{}}' 2>/dev/null |
+	grep -o '"block_height"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') || true
+answering=yes
+if [ -z "$height" ]; then
+	answering=no
+	height=$last_height
+fi
+
+if [ "$height" -gt "$last_height" ]; then
+	if [ "$alerted_at" -gt 0 ]; then
+		send "${NODE_LABEL}: blocks are moving again, height ${height}, after $(((now - changed_at) / 60)) min without a new block." || failed=1
+	fi
+	changed_at=$now
+	alerted_at=0
+else
+	stalled=$(((now - changed_at) / 60))
+	if [ "$stalled" -ge "$STALL_MINUTES" ] &&
+		{ [ "$alerted_at" -eq 0 ] || [ $(((now - alerted_at) / 60)) -ge "$REPEAT_MINUTES" ]; }; then
+		why=""
+		[ "$answering" = no ] && why=" The node RPC is not answering."
+		if send "${NODE_LABEL}: no new block for ${stalled} min, still at height ${height}.${why}"; then
+			alerted_at=$now
+		else
+			failed=1
+		fi
+	fi
+fi
+
+printf '%s %s %s %s\n' "$height" "$changed_at" "${restarts:--1}" "$alerted_at" >"$state.tmp"
+sync "$state.tmp" 2>/dev/null || true
+mv -f "$state.tmp" "$state"
+exit "$failed"
diff --git a/deployments/alerts/tn-node-alert.service b/deployments/alerts/tn-node-alert.service
new file mode 100644
index 00000000..7891b0e6
--- /dev/null
+++ b/deployments/alerts/tn-node-alert.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Alert when the node stops making blocks or crashes
+After=docker.service
+Wants=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/tn-node-alert
+# A stuck run would block every later one, so it gets a time limit.
+TimeoutStartSec=4min
diff --git a/deployments/alerts/tn-node-alert.timer b/deployments/alerts/tn-node-alert.timer
new file mode 100644
index 00000000..510de384
--- /dev/null
+++ b/deployments/alerts/tn-node-alert.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Run tn-node-alert every 5 minutes
+
+[Timer]
+OnCalendar=*:0/5
+Persistent=true
+
+[Install]
+WantedBy=timers.target

From f3c7fd8c25dc008f76f035450be452122b2f2ff8 Mon Sep 17 00:00:00 2001
From: Michael Buntarman <michaelboentarman@gmail.com>
Date: Fri, 6 Nov 2026 13:01:12 +0700
Subject: [PATCH 5/5] fix(alerts): retry a failed recovery message

---
 deployments/alerts/README.md            |  2 +-
 deployments/alerts/node-health-alert.sh | 23 +++++++++++++++++------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/deployments/alerts/README.md b/deployments/alerts/README.md
index c1423177..daeeaa93 100644
--- a/deployments/alerts/README.md
+++ b/deployments/alerts/README.md
@@ -6,7 +6,7 @@ Two small alerts send Telegram messages from each node. Both read the same setti
 
 `tn-pg-alert` sends a Telegram message when a node's Postgres container logs a statement that runs a server program, such as `COPY ... TO PROGRAM`. kwild's database role may not run programs (see `deployments/postgres-roles.sql`), so Postgres refuses the statement and logs it. Any such line means something tried, and it is worth a look.
 
-Each run reads the container's log lines written since the previous run, so nothing is counted twice.
+Each run reads the container's log lines written since the last run that sent its alert. If a send fails, the next run reads the same lines again, so the alert is retried.
 
 ## tn-node-alert: the node stops or crashes
 
diff --git a/deployments/alerts/node-health-alert.sh b/deployments/alerts/node-health-alert.sh
index 689c003a..6b48cd35 100755
--- a/deployments/alerts/node-health-alert.sh
+++ b/deployments/alerts/node-health-alert.sh
@@ -61,12 +61,13 @@ state="$STATE_DIR/node-state"
 now=$(date -u +%s)
 
 # Saved state: height, when it last changed, restart count, when the last stall
-# alert went out (0 when none is open).
-last_height=0 changed_at=$now last_restarts=-1 alerted_at=0
+# alert went out (0 when none is open), and the minutes of a stall whose
+# recovery message has not gone out yet (0 when none).
+last_height=0 changed_at=$now last_restarts=-1 alerted_at=0 recovery=0
 if [ -r "$state" ]; then
-	read -r last_height changed_at last_restarts alerted_at <"$state" || true
+	read -r last_height changed_at last_restarts alerted_at recovery <"$state" || true
 fi
-for v in last_height changed_at last_restarts alerted_at; do
+for v in last_height changed_at last_restarts alerted_at recovery; do
 	[[ ${!v} =~ ^-?[0-9]+$ ]] || printf -v "$v" '%s' 0
 done
 [ "$changed_at" -gt 0 ] || changed_at=$now
@@ -100,7 +101,8 @@ fi
 
 if [ "$height" -gt "$last_height" ]; then
 	if [ "$alerted_at" -gt 0 ]; then
-		send "${NODE_LABEL}: blocks are moving again, height ${height}, after $(((now - changed_at) / 60)) min without a new block." || failed=1
+		recovery=$(((now - changed_at) / 60))
+		[ "$recovery" -gt 0 ] || recovery=1
 	fi
 	changed_at=$now
 	alerted_at=0
@@ -118,7 +120,16 @@ else
 	fi
 fi
 
-printf '%s %s %s %s\n' "$height" "$changed_at" "${restarts:--1}" "$alerted_at" >"$state.tmp"
+# A recovery message that fails is kept and sent again on the next run.
+if [ "$recovery" -gt 0 ]; then
+	if send "${NODE_LABEL}: blocks are moving again, height ${height}, after ${recovery} min without a new block."; then
+		recovery=0
+	else
+		failed=1
+	fi
+fi
+
+printf '%s %s %s %s %s\n' "$height" "$changed_at" "${restarts:--1}" "$alerted_at" "$recovery" >"$state.tmp"
 sync "$state.tmp" 2>/dev/null || true
 mv -f "$state.tmp" "$state"
 exit "$failed"