auto-update: Stop treating script noise as success
Both service scripts piped their output through 'tail -5' under 'pipefail', so the exit status came from tail, not the upgrade — failures looked like success and nothing was sent to the classifier. For PieFed the problem was worse: deploy.sh starts a service that no longer exists and pybabel exits 1 on translation errors, so the strict invocation aborted after stopping the services and left the site down. Run it plainly (it manages fine without set -e, same as manually) and judge success by the live service state plus an HTTP check instead of the exit code. A Mastodon update that fails now reports its real output and status.
This commit is contained in:
parent
86268107fa
commit
71319a7587
2 changed files with 35 additions and 3 deletions
|
|
@ -1,4 +1,15 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -euo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
ssh mastodon@abraham.jeena.net "bash -euo pipefail upgrade.sh $2 2>&1 | tail -5"
|
out=$(ssh mastodon@abraham.jeena.net "bash -euo pipefail upgrade.sh $2 2>&1 | tail -5; exit \${PIPESTATUS[0]}")
|
||||||
|
status=$?
|
||||||
|
|
||||||
|
# Report the last lines either way so notifications show what happened.
|
||||||
|
if [ -n "$out" ]; then
|
||||||
|
echo "$out"
|
||||||
|
elif [ "$status" -ne 0 ]; then
|
||||||
|
echo "Update script exited with status $status"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Propagate the upgrade.sh exit status, not tail's.
|
||||||
|
exit "$status"
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,25 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ssh jeena@jeena.net 'cd piefed && bash -euo pipefail deploy.sh 2>&1 | tail -5'
|
# deploy.sh comes from upstream and has two quirks that must stay unpatched:
|
||||||
|
# - it starts piefed-notifs.service, which no longer exists (exit 5, harmless
|
||||||
|
# because deploy.sh itself has no "set -e" — the same reason running it
|
||||||
|
# manually always works)
|
||||||
|
# - pybabel compile exits 1 on translation errors, which are expected
|
||||||
|
# So run it plainly, do NOT use "bash -euo pipefail deploy.sh" (that aborts
|
||||||
|
# after stopping the services and before starting pyfedi -> nginx 504), and
|
||||||
|
# judge success by the actual service state and a live HTTP check instead of
|
||||||
|
# the exit code.
|
||||||
|
ssh -o BatchMode=yes jeena@jeena.net 'cd piefed && ./deploy.sh 2>&1 | tail -5
|
||||||
|
systemctl is-active --quiet pyfedi.service celery.service'
|
||||||
|
|
||||||
|
# Only report success once the site actually answers again.
|
||||||
|
for _ in 1 2 3 4 5 6; do
|
||||||
|
code=$(curl -s -o /dev/null -w '%{http_code}' https://piefed.jeena.net/api/v3/site)
|
||||||
|
if [ "$code" = "200" ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
echo "PieFed did not come back up after deploy (last HTTP code: $code)" >&2
|
||||||
|
exit 1
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue