1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#!/bin/bash
#
# Copyright (C) 2026 Danilo M. <danix@danix.xyz>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# notify.sh — push build-chain failures to Gotify.
#
# Two modes, because the chain fails in two ways and only one of them
# has a non-zero exit status:
#
# run <label> <cmd...> run cmd; on failure post the label, the exit
# code and the tail of the log.
# stale post if any tag is older than its budget.
#
# The second mode exists because of how the chain broke in September 2026:
# build-full-image.sh failed for ten nights, but build-sbo-testbuild.sh
# exited 0 every one of them. It saw an unchanged parent digest and skipped,
# which is correct behaviour. Nothing errored, nothing moved, and nothing
# said so. An exit-status alert alone would have caught the first failure
# but not the six days of silence that followed, so `stale` asks the
# registry a different question: not "did it error" but "is it current".
set -euo pipefail
PROJECT_VERSION="1.1.2" # bump via sed across all scripts; see CLAUDE.md Releases
HERE="$(cd "$(dirname "$0")" && pwd)"
source "${HERE}/config"
GOTIFY_URL="${GOTIFY_URL:-http://gotify.noland.dnx}"
GOTIFY_TOKEN_FILE="${GOTIFY_TOKEN_FILE:-/root/.gotify-token}"
LOGFILE="${LOGFILE:-/var/log/sbo-testbuild.log}"
# -current rebuilds nightly; 15.0 is frozen and legitimately sits for weeks.
STALE_DAYS="${STALE_DAYS:-2}"
STALE_DAYS_STABLE="${STALE_DAYS_STABLE:-30}"
STABLE_VARIANT="${STABLE_VARIANT:-15.0}"
# post TITLE MESSAGE PRIORITY
# Best-effort by design: a notifier that fails a build because the notifier
# is down is worse than no notifier. Never exits non-zero.
post() {
local title="$1" message="$2" priority="${3:-8}" token
if [[ ! -r "${GOTIFY_TOKEN_FILE}" ]]; then
echo "notify: no readable token at ${GOTIFY_TOKEN_FILE}; not posting" >&2
return 0
fi
token="$(tr -d '\n' < "${GOTIFY_TOKEN_FILE}")"
[[ -n "${token}" ]] || { echo "notify: empty token; not posting" >&2; return 0; }
curl -sf -m 10 -o /dev/null \
"${GOTIFY_URL}/message?token=${token}" \
-F "title=${title}" \
-F "message=${message}" \
-F "priority=${priority}" \
|| echo "notify: POST to ${GOTIFY_URL} failed (ignored)" >&2
return 0
}
# run LABEL CMD...
# Runs CMD and propagates its exit status, so cron and the caller still see
# the real result; the notification is a side effect, not a substitute.
cmd_run() {
local label="$1"; shift
[[ $# -gt 0 ]] || { echo "notify: run needs a command" >&2; exit 2; }
local rc=0
"$@" || rc=$?
[[ ${rc} -eq 0 ]] && return 0
# The scripts log their own diagnostics; the last lines are usually the
# actual error ("no space left on device" and friends).
local tail_txt=""
[[ -r "${LOGFILE}" ]] && tail_txt="$(tail -n 12 "${LOGFILE}" 2>/dev/null || true)"
post "sbo-testbuild: ${label} FAILED" \
"exit ${rc} at $(date '+%F %T') on $(hostname -s)
${tail_txt}" \
8
return "${rc}"
}
# stale
# Ask the registry how old each tag is. Independent of the build scripts on
# purpose: it catches a wedged chain, a stopped cron and a dead mirror alike,
# none of which produce a failing exit status anywhere.
cmd_stale() {
local now stale_list="" repo tag created age budget
now=$(date +%s)
local unreadable=""
for repo in sbo-base sbo-full sbo-testbuild; do
for tag in "${VARIANTS[@]}"; do
# A tag we cannot read is its own kind of bad news (registry down,
# tag never pushed), so say so rather than skipping quietly.
if ! created=$(tag_created "${repo}" "${tag}") || [[ -z "${created}" ]]; then
unreadable+="${repo}:${tag}"$'\n'
continue
fi
budget="${STALE_DAYS}"
[[ "${tag}" == "${STABLE_VARIANT}" ]] && budget="${STALE_DAYS_STABLE}"
age=$(( (now - created) / 86400 ))
if [[ ${age} -gt ${budget} ]]; then
stale_list+="${repo}:${tag} — ${age}d old (budget ${budget}d)"$'\n'
fi
done
done
if [[ -z "${stale_list}" && -z "${unreadable}" ]]; then
echo "all tags current"
return 0
fi
local body=""
[[ -n "${stale_list}" ]] && body+="Not refreshed:"$'\n'"${stale_list}"$'\n'
[[ -n "${unreadable}" ]] && body+="Could not read from the registry:"$'\n'"${unreadable}"$'\n'
echo "${body}"
post "sbo-testbuild: images going stale" \
"The chain has not refreshed these tags. A stage may be failing, or
skipping because an earlier one did.
${body}Check ${LOGFILE} on $(hostname -s)." \
7
}
# tag_created REPO TAG -> unix timestamp on stdout, or non-zero if unreadable.
# Two hops: the manifest names the config blob, the config blob has the date.
#
# Parsed with python3 rather than sed: the registry pretty-prints its JSON, so
# the "config" object spans several lines and a line-oriented regex silently
# matches nothing. python3 is already required here (build-full-image.sh serves
# the mirror with http.server), so this adds no dependency.
tag_created() {
local repo="$1" tag="$2" manifest cfg created
local accept='application/vnd.docker.distribution.manifest.v2+json'
manifest=$(curl -sf -m 10 -H "Accept: ${accept}" \
"http://${REGISTRY}/v2/${repo}/manifests/${tag}" 2>/dev/null) || return 1
cfg=$(printf '%s' "${manifest}" | python3 -c \
'import sys,json; print(json.load(sys.stdin)["config"]["digest"])' 2>/dev/null) \
|| return 1
[[ -n "${cfg}" ]] || return 1
created=$(curl -sf -m 10 "http://${REGISTRY}/v2/${repo}/blobs/${cfg}" 2>/dev/null \
| python3 -c 'import sys,json; print(json.load(sys.stdin).get("created",""))' \
2>/dev/null) || return 1
[[ -n "${created}" ]] || return 1
# Registry timestamps carry nanoseconds, which `date -d` rejects; keep the
# seconds and the offset, drop the fraction.
created="${created/Z/+00:00}"
created="$(printf '%s' "${created}" | sed -E 's/\.[0-9]+([+-][0-9:]+)?$/\1/')"
date -d "${created}" +%s 2>/dev/null || return 1
}
case "${1:-}" in
-V|--version) echo "notify.sh ${PROJECT_VERSION}"; exit 0 ;;
run) shift; cmd_run "$@" ;;
stale) cmd_stale ;;
test) post "sbo-testbuild: test" "Notification test at $(date '+%F %T')." 2 ;;
*) echo "usage: $0 {run <label> <cmd...>|stale|test} [-V]" >&2; exit 2 ;;
esac
|
