From 71319a7587833680321f9a56cad648f34bc4cee6 Mon Sep 17 00:00:00 2001
From: Jeena <hello@jeena.net>
Date: Tue, 6 Oct 2026 22:10:10 +0900
Subject: [PATCH] auto-update: Stop treating script noise as success
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both service scripts piped their output through 'tail -5' under
'pipefail', so the exit status came from tail, not the upgrade —
failures looked like success and nothing was sent to the classifier.
For PieFed the problem was worse: deploy.sh starts a service that no
longer exists and pybabel exits 1 on translation errors, so the strict
invocation aborted after stopping the services and left the site down.
Run it plainly (it manages fine without set -e, same as manually) and
judge success by the live service state plus an HTTP check instead of
the exit code.

A Mastodon update that fails now reports its real output and status.
---
 services/mastodon/auto-update.sh | 15 +++++++++++++--
 services/piefed/auto-update.sh   | 23 ++++++++++++++++++++++-
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/services/mastodon/auto-update.sh b/services/mastodon/auto-update.sh
index 8320bee..1dec0eb 100755
--- a/services/mastodon/auto-update.sh
+++ b/services/mastodon/auto-update.sh
@@ -1,4 +1,15 @@
 #!/bin/bash
-set -euo pipefail
+set -uo pipefail
 
-ssh mastodon@abraham.jeena.net "bash -euo pipefail upgrade.sh $2 2>&1 | tail -5"
+out=$(ssh mastodon@abraham.jeena.net "bash -euo pipefail upgrade.sh $2 2>&1 | tail -5; exit \${PIPESTATUS[0]}")
+status=$?
+
+# Report the last lines either way so notifications show what happened.
+if [ -n "$out" ]; then
+    echo "$out"
+elif [ "$status" -ne 0 ]; then
+    echo "Update script exited with status $status"
+fi
+
+# Propagate the upgrade.sh exit status, not tail's.
+exit "$status"
diff --git a/services/piefed/auto-update.sh b/services/piefed/auto-update.sh
index f2a6299..75cec29 100755
--- a/services/piefed/auto-update.sh
+++ b/services/piefed/auto-update.sh
@@ -1,4 +1,25 @@
 #!/bin/bash
 set -euo pipefail
 
-ssh jeena@jeena.net 'cd piefed && bash -euo pipefail deploy.sh 2>&1 | tail -5'
+# deploy.sh comes from upstream and has two quirks that must stay unpatched:
+#   - it starts piefed-notifs.service, which no longer exists (exit 5, harmless
+#     because deploy.sh itself has no "set -e" — the same reason running it
+#     manually always works)
+#   - pybabel compile exits 1 on translation errors, which are expected
+# So run it plainly, do NOT use "bash -euo pipefail deploy.sh" (that aborts
+# after stopping the services and before starting pyfedi -> nginx 504), and
+# judge success by the actual service state and a live HTTP check instead of
+# the exit code.
+ssh -o BatchMode=yes jeena@jeena.net 'cd piefed && ./deploy.sh 2>&1 | tail -5
+systemctl is-active --quiet pyfedi.service celery.service'
+
+# Only report success once the site actually answers again.
+for _ in 1 2 3 4 5 6; do
+    code=$(curl -s -o /dev/null -w '%{http_code}' https://piefed.jeena.net/api/v3/site)
+    if [ "$code" = "200" ]; then
+        exit 0
+    fi
+    sleep 10
+done
+echo "PieFed did not come back up after deploy (last HTTP code: $code)" >&2
+exit 1