From 5fb8cef38353f27d40b89b62a60b1c48e1a1dd5f Mon Sep 17 00:00:00 2001
From: "Danilo M." <danix@danix.xyz>
Date: Wed, 23 Sep 2026 09:41:49 +0200
Subject: image-builder: retry docker build once on failure

The -current full build failed on 2026-09-23 after all ten steps had
passed, at layer export, losing a race on the containerd snapshotter's
content lock ("failed to open writer: ref ... locked for 74ms ...
unavailable"). Disk was not the issue this time (73G free). The build
cache pre-prune that works around this only runs under --force, so the
normal gated nightly build had no guard.

Add docker_build() to lib.sh: docker build, retried once. By the time
the export fails every step is cached, so the retry is little more than
a re-export. Used by build-full-image.sh and build-sbo-testbuild.sh,
the two scripts that export ~20-33G images; bootstrap.sh's image is
small and has never hit this.

Self-check covers success, one failure, and two failures with a stub.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 image-builder/build-full-image.sh    |  2 +-
 image-builder/build-sbo-testbuild.sh |  2 +-
 image-builder/lib.sh                 | 15 +++++++++++++++
 image-builder/test-image-builder.sh  | 12 ++++++++++++
 4 files changed, 29 insertions(+), 2 deletions(-)

(limited to 'image-builder')

diff --git a/image-builder/build-full-image.sh b/image-builder/build-full-image.sh
index 47e034b..9a2b83c 100755
--- a/image-builder/build-full-image.sh
+++ b/image-builder/build-full-image.sh
@@ -212,7 +212,7 @@ DOCKERFILE
     # docker build would otherwise fall through to push a nonexistent tag and
     # log "Done". Check each step and return non-zero on failure.
     _log "  Building ${FULL_TAG}..."
-    if ! docker build "${BUILD_FLAGS[@]}" \
+    if ! docker_build "${BUILD_FLAGS[@]}" \
         -t "${FULL_TAG}" \
         "${WORKDIR}"; then
         _warn "  build failed for ${FULL_TAG}; not pushing."
diff --git a/image-builder/build-sbo-testbuild.sh b/image-builder/build-sbo-testbuild.sh
index 2bb8487..3949c51 100755
--- a/image-builder/build-sbo-testbuild.sh
+++ b/image-builder/build-sbo-testbuild.sh
@@ -126,7 +126,7 @@ DOCKERFILE
     # (see Main), which suppresses `set -e` in this subshell, so a failed build
     # would otherwise push a nonexistent tag and log "Done".
     _log "  Building ${TB_TAG}..."
-    if ! docker build "${BUILD_FLAGS[@]}" -t "${TB_TAG}" "${WORKDIR}"; then
+    if ! docker_build "${BUILD_FLAGS[@]}" -t "${TB_TAG}" "${WORKDIR}"; then
         _warn "  build failed for ${TB_TAG}; not pushing."
         return 1
     fi
diff --git a/image-builder/lib.sh b/image-builder/lib.sh
index 8a22828..9c519ac 100644
--- a/image-builder/lib.sh
+++ b/image-builder/lib.sh
@@ -91,3 +91,18 @@ require_mount() {
     local dir="${base}/slackware64-${version}"
     [[ -d "$dir" ]] || _err "NAS tree not mounted: ${dir}"
 }
+
+# docker_build ARGS...
+# `docker build`, retried once on failure. The export of a large image can lose
+# a race on the containerd snapshotter's content lock ("failed to open writer:
+# ref ... locked for 74ms ... unavailable"); the -current full build did on
+# 2026-09-23 after all ten steps had passed. The steps are cached by then, so a
+# retry is little more than a re-export.
+# ponytail: blind single retry; a real step failure costs one extra attempt
+# (a full rebuild under --force/--no-cache). Match on the error text if that
+# ever matters.
+docker_build() {
+    docker build "$@" && return 0
+    _warn "  docker build failed; retrying once..."
+    docker build "$@"
+}
diff --git a/image-builder/test-image-builder.sh b/image-builder/test-image-builder.sh
index 19f3ff8..fe6cff6 100755
--- a/image-builder/test-image-builder.sh
+++ b/image-builder/test-image-builder.sh
@@ -83,6 +83,18 @@ DOCKER_RC=0
 ( require_docker ) 2>/dev/null; check_rc "require_docker daemon up" 0 $?
 unset -f docker
 
+# --- docker_build: one retry, then give up ---
+# Stub fails the first FAILS calls, counting attempts in a file (the calls run
+# in $() subshells, so a shell variable would not survive).
+docker() { local n; n=$(( $(cat "$tmp/calls") + 1 )); echo "$n" > "$tmp/calls"; (( n > FAILS )); }
+for FAILS in 0 1 2; do
+    echo 0 > "$tmp/calls"
+    ( docker_build -t x . ) 2>/dev/null; rc=$?
+    check_rc "docker_build rc after $FAILS failures" "$(( FAILS > 1 ))" "$rc"
+    check "docker_build attempts after $FAILS failures" "$(( FAILS > 1 ? 2 : FAILS + 1 ))" "$(cat "$tmp/calls")"
+done
+unset -f docker
+
 # --- registry-gc.sh: -V / --help parse without touching docker; syntax valid ---
 gc_ver="$(sed -n 's/^PROJECT_VERSION="\([^"]*\)".*/\1/p' registry-gc.sh)"
 check "registry-gc.sh -V" "registry-gc.sh $gc_ver" "$(bash ./registry-gc.sh -V)"
-- 
cgit v1.2.3