image-builder: add safe registry GC and stop OCI-index breakageHEADmaster

The registry never reclaims blobs, so its store grows until the disk fills and the nightly builds fail with "no space left on device". Add registry-gc.sh, run weekly (Sunday 08:00), plus a daily dangling-image prune.

registry-gc.sh refuses to run while a build is active, stops the registry for a stable blob graph, deletes only untagged manifests (-m) and their blobs, restarts via an EXIT trap, and verifies a tag still pulls.

distribution 2.8.x GC does not follow OCI image indexes, so -m deletes their child manifests (distribution#3178). Default BuildKit provenance made every pushed tag an OCI index, which made -m destructive. Build scripts now pass --provenance=false (plain schema2), and registry-gc.sh refuses to run if any tag is still an index.

1 files changed, 192 insertions, 0 deletions

diff --git a/image-builder/registry-gc.sh b/image-builder/registry-gc.sh
new file mode 100644
index 0000000..fab37bb
--- /dev/null
+++ b/image-builder/registry-gc.sh
@@ -0,0 +1,192 @@
+#!/bin/bash
+#
+# Copyright (C) 2026 Danilo M. <danix@danix.xyz>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# registry-gc.sh — garbage-collect the LAN registry's blob store.
+#
+# The registry retains every blob ever pushed; without garbage collection the
+# storage dir grows without bound until the disk fills and the nightly image
+# builds fail with "no space left on device". This runs the distribution
+# garbage collector safely:
+#   * refuses to run while a build/push is in flight,
+#   * stops the registry so the manifest/blob graph is stable,
+#   * deletes only untagged manifests (-m) and the blobs they alone
+#     reference; every tag is preserved,
+#   * restarts the registry via an EXIT trap, even on failure,
+#   * verifies a tag still resolves afterwards.
+# Intended as a weekly cron job, after the build chain and before the daily
+# cache prune. See README.
+set -euo pipefail
+PROJECT_VERSION="1.1.1"   # bump via sed across all scripts; see CLAUDE.md Releases
+HERE="$(cd "$(dirname "$0")" && pwd)"
+source "${HERE}/config"
+LOG_TAG=registry-gc
+source "${HERE}/lib.sh"
+
+: "${REGISTRY_CONTAINER:=registry}"
+
+usage() {
+    cat <<EOF
+Usage: registry-gc.sh [OPTIONS]
+
+Garbage-collect the blob store of the LAN registry container.
+
+Options:
+  --dry-run         Scan and report what would be deleted; delete nothing and
+                    leave the registry running.
+  --container NAME  Registry container name (default: ${REGISTRY_CONTAINER}).
+  -V                Print version and exit.
+  -h, --help        Show this help.
+EOF
+}
+
+DRY_RUN=false
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -V)          echo "registry-gc.sh $PROJECT_VERSION"; exit 0 ;;
+        --dry-run)   DRY_RUN=true;            shift   ;;
+        --container) REGISTRY_CONTAINER="$2"; shift 2 ;;
+        -h|--help)   usage; exit 0 ;;
+        *) _err "unknown argument: $1" ;;
+    esac
+done
+
+require_docker
+
+docker inspect "${REGISTRY_CONTAINER}" >/dev/null 2>&1 \
+    || _err "registry container not found: ${REGISTRY_CONTAINER}"
+
+# Resolve the host directory backing the container's /var/lib/registry mount.
+STORAGE="$(docker inspect \
+    --format '{{range .Mounts}}{{if eq .Destination "/var/lib/registry"}}{{.Source}}{{end}}{{end}}' \
+    "${REGISTRY_CONTAINER}")"
+[[ -n "${STORAGE}" ]] || _err "no /var/lib/registry mount on ${REGISTRY_CONTAINER}"
+[[ -d "${STORAGE}" ]] || _err "registry storage dir missing: ${STORAGE}"
+
+REG_IMAGE="$(docker inspect --format '{{.Config.Image}}' "${REGISTRY_CONTAINER}")"
+REGISTRY_PORT="${REGISTRY##*:}"
+
+_log "registry=${REGISTRY_CONTAINER} image=${REG_IMAGE} storage=${STORAGE}"
+
+# GC config: same filesystem root as the live registry, read-only maintenance
+# (the documented safe mode for collection), delete enabled so unreferenced
+# blobs are actually reclaimed.
+CFG_DIR="$(mktemp -d /tmp/registry-gc.XXXXXX)"
+trap 'rm -rf "${CFG_DIR}"' EXIT
+cat > "${CFG_DIR}/config.yml" <<'YAML'
+version: 0.1
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+  delete:
+    enabled: true
+  maintenance:
+    readonly:
+      enabled: true
+YAML
+
+# Run the collector in a throwaway container against the same storage.
+gc() {
+    docker run --rm \
+        -v "${STORAGE}:/var/lib/registry" \
+        -v "${CFG_DIR}/config.yml:/etc/docker/registry/config.yml:ro" \
+        "${REG_IMAGE}" bin/registry garbage-collect "$@" \
+        /etc/docker/registry/config.yml
+}
+
+# Hard safety gate: distribution 2.8.x GC does not follow OCI image indexes /
+# manifest lists, so `-m` deletes their child manifests (distribution issue
+# #3178). Every tagged manifest must be a plain schema2/OCI image manifest.
+# If any tag is an index, refuse: rebuild the images without provenance (the
+# build scripts pass --provenance=false) before collecting again.
+assert_gc_safe() {
+    local base="http://localhost:${REGISTRY_PORT}"
+    local repos repo tags tag ct
+    repos="$(curl -sf "${base}/v2/_catalog" \
+        | sed -n 's/.*"repositories":\[\([^]]*\)\].*/\1/p' | tr ',' '\n' | tr -d ' "')"
+    [[ -n "${repos}" ]] || _err "cannot read registry catalog at ${base}/v2/_catalog"
+    for repo in ${repos}; do
+        tags="$(curl -sf "${base}/v2/${repo}/tags/list" \
+            | sed -n 's/.*"tags":\[\([^]]*\)\].*/\1/p' | tr ',' '\n' | tr -d ' "')"
+        for tag in ${tags}; do
+            ct="$(curl -sfI -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \
+                "${base}/v2/${repo}/manifests/${tag}" \
+                | tr -d '\r' | awk 'tolower($1)=="content-type:"{print $2}')"
+            case "${ct}" in
+                *index*|*manifest.list*)
+                    _err "tag ${repo}:${tag} is an OCI index/manifest list (${ct}); refusing to GC. Rebuild images with --provenance=false (see build scripts)." ;;
+            esac
+            _log "  tag ${repo}:${tag} -> ${ct:-unknown}"
+        done
+    done
+}
+
+assert_gc_safe
+
+if [[ "${DRY_RUN}" == "true" ]]; then
+    _log "dry-run: scanning ${STORAGE} (registry left running; nothing deleted)"
+    gc --dry-run -m
+    _log "dry-run complete; no changes made."
+    exit 0
+fi
+
+# Never race a build/push: the blob graph must be stable while we sweep.
+if pgrep -f 'bootstrap.sh|build-full-image.sh|build-sbo-testbuild.sh' >/dev/null; then
+    _err "a build script is running; refusing to GC. Re-run after the chain finishes."
+fi
+
+stopped=false
+restart_registry() {
+    [[ "${stopped}" == "true" ]] || return 0
+    _log "restarting ${REGISTRY_CONTAINER}..."
+    docker start "${REGISTRY_CONTAINER}" >/dev/null
+    local i
+    for i in $(seq 1 30); do
+        curl -sf -o /dev/null "http://localhost:${REGISTRY_PORT}/v2/" && break
+        sleep 1
+    done
+    stopped=false
+}
+trap 'restart_registry; rm -rf "${CFG_DIR}"' EXIT
+
+before_size="$(du -sh "${STORAGE}" 2>/dev/null | cut -f1)"
+snap_tag="${REGISTRY}/sbo-base:current"
+snap_digest="$(docker inspect --format '{{index .RepoDigests 0}}' "${snap_tag}" 2>/dev/null || true)"
+
+_log "stopping ${REGISTRY_CONTAINER} for a stable blob graph..."
+docker stop "${REGISTRY_CONTAINER}" >/dev/null && stopped=true
+
+_log "garbage-collect dry-run (audit; deletes nothing)..."
+gc --dry-run -m
+
+_log "garbage-collect (untagged manifests + their blobs; tags preserved)..."
+gc -m
+
+restart_registry
+
+after_size="$(du -sh "${STORAGE}" 2>/dev/null | cut -f1)"
+_log "storage: ${before_size} -> ${after_size}"
+
+if [[ -n "${snap_digest}" ]]; then
+    if docker pull "${snap_tag}" >/dev/null 2>&1; then
+        new_digest="$(docker inspect --format '{{index .RepoDigests 0}}' "${snap_tag}" 2>/dev/null || true)"
+        if [[ "${new_digest}" == "${snap_digest}" ]]; then
+            _log "verify OK: ${snap_tag} still resolves, digest unchanged"
+        else
+            _warn "verify: ${snap_tag} digest changed: ${snap_digest} -> ${new_digest}"
+        fi
+    else
+        _warn "verify FAILED: ${snap_tag} no longer pulls after GC"
+    fi
+fi
+
+_log "=== Done ==="