From cd4f992a29da96b835c14545a0cb05e7e87ca291 Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Thu, 10 Sep 2026 21:02:49 +0200 Subject: image-builder: add safe registry GC and stop OCI-index breakage The registry never reclaims blobs, so its store grows until the disk fills and the nightly builds fail with "no space left on device". Add registry-gc.sh, run weekly (Sunday 08:00), plus a daily dangling-image prune. registry-gc.sh refuses to run while a build is active, stops the registry for a stable blob graph, deletes only untagged manifests (-m) and their blobs, restarts via an EXIT trap, and verifies a tag still pulls. distribution 2.8.x GC does not follow OCI image indexes, so -m deletes their child manifests (distribution#3178). Default BuildKit provenance made every pushed tag an OCI index, which made -m destructive. Build scripts now pass --provenance=false (plain schema2), and registry-gc.sh refuses to run if any tag is still an index. --- image-builder/README | 55 +++++++++- image-builder/bootstrap.sh | 6 +- image-builder/build-full-image.sh | 6 +- image-builder/build-sbo-testbuild.sh | 6 +- image-builder/config | 1 + image-builder/registry-gc.sh | 192 +++++++++++++++++++++++++++++++++++ image-builder/test-image-builder.sh | 6 ++ 7 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 image-builder/registry-gc.sh (limited to 'image-builder') diff --git a/image-builder/README b/image-builder/README index 8711b1d..86c463f 100644 --- a/image-builder/README +++ b/image-builder/README @@ -10,6 +10,9 @@ Three scripts, chained (see docs/specs/2026-07-13-image-builder-design.md): build-full-image.sh sbo-full:{ver} FROM base, all series build-sbo-testbuild.sh sbo-testbuild:{ver} FROM full, + sbopkg + tools +Plus one maintenance script (not part of the chain): + registry-gc.sh reclaim unreferenced blobs from the registry store + All settings live in ./config. VM setup (docker.noland.dnx, Slackware x86_64, 4 vCPU / 4 GB / 80 GB) @@ -22,9 +25,9 @@ VM setup (docker.noland.dnx, Slackware x86_64, 4 vCPU / 4 GB / 80 GB) Each is a full mirror (PACKAGES.TXT, ChangeLog.txt, slackware64/, patches/, extra/). Root must be able to read them (bootstrap runs installpkg as root). -3. Run a LAN registry: +3. Run a LAN registry (storage on the same disk as docker, bind-mounted): docker run -d --restart=always -p 5000:5000 \ - -v /opt/registry/data:/var/lib/registry --name registry registry:2 + -v /opt/sbo-testbuild/registry:/var/lib/registry --name registry registry:2 4. Mark the registry insecure (plain HTTP) on the VM AND every pulling client (this dev box, the buildsystem VM). In /etc/docker/daemon.json: @@ -49,7 +52,18 @@ VM setup (docker.noland.dnx, Slackware x86_64, 4 vCPU / 4 GB / 80 GB) # 15.0 (ready ~06:35) 0 5 * * * /path/to/sbo-dockerbuild/image-builder/bootstrap.sh --version 15.0 >> /var/log/sbo-testbuild.log 2>&1 20 5 * * * /path/to/sbo-dockerbuild/image-builder/build-full-image.sh --version 15.0 >> /var/log/sbo-testbuild.log 2>&1 - 30 6 * * * /path/to/sbo-dockerbuild/image-builder/build-sbo-testbuild.sh --version 15.0 >> /var/log/sbo-testbuild.log 2>&1 + 30 6 * * * /path/to/sbo-dockerbuild/image-builder/build-sbo-testbuild.sh --version 15.0 >> /var/log/sbo-testbuild.log 2>&1 + + Post-build cleanup, after the chain (which ends ~06:30) and before the 15:00 + cache prune: + # daily: drop dangling images left behind when a tag moves to a new build + 0 7 * * * docker image prune -f >> /var/log/sbo-testbuild.log 2>&1 + # weekly (Sunday): reclaim unreferenced blobs from the registry store + 0 8 * * 0 /path/to/sbo-dockerbuild/image-builder/registry-gc.sh >> /var/log/sbo-testbuild.log 2>&1 + + The registry never reclaims blobs on its own, so without the weekly GC its + storage grows until the disk fills and the nightly builds fail with + "no space left on device" (see the section below). 7. Ensure docker.noland.dnx resolves on the LAN (static IP or DNS). @@ -64,6 +78,41 @@ Then confirm: Flags: --force (rebuild unconditionally), --version (one variant). +Registry garbage collection (registry-gc.sh) +-------------------------------------------- +The registry keeps every blob ever pushed; it never reclaims on its own. Left +alone, the store grows until the disk fills and the nightly builds fail. Two +cleanups keep it bounded: + + docker image prune -f (daily) removes dangling images left in the docker + store when a tag moves to a freshly built image. + registry-gc.sh (weekly) reclaims unreferenced blobs from the + registry's own store. + +registry-gc.sh is deliberately conservative: + * it refuses to run while any build script is active, so it can never race a + push (cron runs it at 08:00 Sunday, well after the ~06:30 chain); + * it stops the registry so the manifest/blob graph is stable, and restarts it + via an EXIT trap even if collection fails part-way; + * it deletes only untagged manifests (-m) and the blobs they alone + reference, so every tag keeps resolving; + * it verifies afterwards that a tag still pulls. + +Why the build scripts pass --provenance=false: with default BuildKit +provenance, `docker push` stores an OCI image index (the image plus an +attestation manifest). Distribution 2.8.x garbage collection does not follow +OCI indexes, so `-m` would delete their child manifests and orphan the layer +blobs (distribution issue #3178). Disabling provenance keeps each tag a plain +Docker schema2 manifest, which the collector handles correctly. registry-gc.sh +refuses to run if it finds any tag that is still an index, so this cannot +regress silently. + +Preview without touching anything (registry stays up, nothing is deleted): + ./registry-gc.sh --dry-run + +Storage path is resolved from the running container's /var/lib/registry mount, +so the script follows the registry wherever it is mounted. + Tests ----- bash test-image-builder.sh # pure-logic self-check, no docker diff --git a/image-builder/bootstrap.sh b/image-builder/bootstrap.sh index 7316116..e632e7a 100755 --- a/image-builder/bootstrap.sh +++ b/image-builder/bootstrap.sh @@ -541,7 +541,11 @@ DOCKERFILE # (see Main), which suppresses `set -e` in this subshell, so a failed build # would otherwise push a nonexistent tag and log "Done". _log "Building and pushing ${TAG}..." - if ! docker build --no-cache -t "${TAG}" "${WORKDIR}"; then + # --provenance=false keeps the pushed manifest a plain Docker schema2 + # manifest rather than an OCI image index. The registry's garbage collector + # (distribution 2.8.x) does not follow OCI indexes and, with -m, deletes + # their child manifests; schema2 avoids that. See registry-gc.sh. + if ! docker build --provenance=false --no-cache -t "${TAG}" "${WORKDIR}"; then _warn "build failed for ${TAG}; not pushing." return 1 fi diff --git a/image-builder/build-full-image.sh b/image-builder/build-full-image.sh index 3706186..6f88db7 100755 --- a/image-builder/build-full-image.sh +++ b/image-builder/build-full-image.sh @@ -200,7 +200,11 @@ LABEL ${DIGEST_LABEL}="${BASE_DIGEST}" CMD ["/bin/bash"] DOCKERFILE - local BUILD_FLAGS=() + # --provenance=false keeps the pushed manifest a plain Docker schema2 + # manifest rather than an OCI image index. The registry's garbage collector + # (distribution 2.8.x) does not follow OCI indexes and, with -m, deletes + # their child manifests; schema2 avoids that. See registry-gc.sh. + local BUILD_FLAGS=(--provenance=false) [[ "${FORCE}" == "true" ]] && BUILD_FLAGS+=(--no-cache) # Explicit exit checks: build_variant runs in an `if ! (...)` condition diff --git a/image-builder/build-sbo-testbuild.sh b/image-builder/build-sbo-testbuild.sh index f982a91..8d08a03 100755 --- a/image-builder/build-sbo-testbuild.sh +++ b/image-builder/build-sbo-testbuild.sh @@ -116,7 +116,11 @@ LABEL ${PKGS_LABEL}="${PKGS_HASH}" CMD ["/bin/bash"] DOCKERFILE - local BUILD_FLAGS=() + # --provenance=false keeps the pushed manifest a plain Docker schema2 + # manifest rather than an OCI image index. The registry's garbage collector + # (distribution 2.8.x) does not follow OCI indexes and, with -m, deletes + # their child manifests; schema2 avoids that. See registry-gc.sh. + local BUILD_FLAGS=(--provenance=false) [[ "${FORCE}" == "true" ]] && BUILD_FLAGS+=(--no-cache) # Explicit exit checks: build_variant runs in an `if ! (...)` condition # (see Main), which suppresses `set -e` in this subshell, so a failed build diff --git a/image-builder/config b/image-builder/config index 8b52313..3815cb9 100644 --- a/image-builder/config +++ b/image-builder/config @@ -2,6 +2,7 @@ # Sourced by bootstrap.sh, build-full-image.sh, build-sbo-testbuild.sh, tests. REGISTRY="docker.noland.dnx:5000" +REGISTRY_CONTAINER="registry" # container running the LAN registry (registry-gc.sh) MIRROR="file:///mnt/nas" VARIANTS=(current 15.0) # x86_64 only for now PKGDIR="/opt/sbo-testbuild/pkgs" # sbopkg + sbo-maintainer-tools .txz diff --git a/image-builder/registry-gc.sh b/image-builder/registry-gc.sh new file mode 100644 index 0000000..fab37bb --- /dev/null +++ b/image-builder/registry-gc.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# +# Copyright (C) 2026 Danilo M. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# registry-gc.sh — garbage-collect the LAN registry's blob store. +# +# The registry retains every blob ever pushed; without garbage collection the +# storage dir grows without bound until the disk fills and the nightly image +# builds fail with "no space left on device". This runs the distribution +# garbage collector safely: +# * refuses to run while a build/push is in flight, +# * stops the registry so the manifest/blob graph is stable, +# * deletes only untagged manifests (-m) and the blobs they alone +# reference; every tag is preserved, +# * restarts the registry via an EXIT trap, even on failure, +# * verifies a tag still resolves afterwards. +# Intended as a weekly cron job, after the build chain and before the daily +# cache prune. See README. +set -euo pipefail +PROJECT_VERSION="1.1.1" # bump via sed across all scripts; see CLAUDE.md Releases +HERE="$(cd "$(dirname "$0")" && pwd)" +source "${HERE}/config" +LOG_TAG=registry-gc +source "${HERE}/lib.sh" + +: "${REGISTRY_CONTAINER:=registry}" + +usage() { + cat </dev/null 2>&1 \ + || _err "registry container not found: ${REGISTRY_CONTAINER}" + +# Resolve the host directory backing the container's /var/lib/registry mount. +STORAGE="$(docker inspect \ + --format '{{range .Mounts}}{{if eq .Destination "/var/lib/registry"}}{{.Source}}{{end}}{{end}}' \ + "${REGISTRY_CONTAINER}")" +[[ -n "${STORAGE}" ]] || _err "no /var/lib/registry mount on ${REGISTRY_CONTAINER}" +[[ -d "${STORAGE}" ]] || _err "registry storage dir missing: ${STORAGE}" + +REG_IMAGE="$(docker inspect --format '{{.Config.Image}}' "${REGISTRY_CONTAINER}")" +REGISTRY_PORT="${REGISTRY##*:}" + +_log "registry=${REGISTRY_CONTAINER} image=${REG_IMAGE} storage=${STORAGE}" + +# GC config: same filesystem root as the live registry, read-only maintenance +# (the documented safe mode for collection), delete enabled so unreferenced +# blobs are actually reclaimed. +CFG_DIR="$(mktemp -d /tmp/registry-gc.XXXXXX)" +trap 'rm -rf "${CFG_DIR}"' EXIT +cat > "${CFG_DIR}/config.yml" <<'YAML' +version: 0.1 +storage: + filesystem: + rootdirectory: /var/lib/registry + delete: + enabled: true + maintenance: + readonly: + enabled: true +YAML + +# Run the collector in a throwaway container against the same storage. +gc() { + docker run --rm \ + -v "${STORAGE}:/var/lib/registry" \ + -v "${CFG_DIR}/config.yml:/etc/docker/registry/config.yml:ro" \ + "${REG_IMAGE}" bin/registry garbage-collect "$@" \ + /etc/docker/registry/config.yml +} + +# Hard safety gate: distribution 2.8.x GC does not follow OCI image indexes / +# manifest lists, so `-m` deletes their child manifests (distribution issue +# #3178). Every tagged manifest must be a plain schema2/OCI image manifest. +# If any tag is an index, refuse: rebuild the images without provenance (the +# build scripts pass --provenance=false) before collecting again. +assert_gc_safe() { + local base="http://localhost:${REGISTRY_PORT}" + local repos repo tags tag ct + repos="$(curl -sf "${base}/v2/_catalog" \ + | sed -n 's/.*"repositories":\[\([^]]*\)\].*/\1/p' | tr ',' '\n' | tr -d ' "')" + [[ -n "${repos}" ]] || _err "cannot read registry catalog at ${base}/v2/_catalog" + for repo in ${repos}; do + tags="$(curl -sf "${base}/v2/${repo}/tags/list" \ + | sed -n 's/.*"tags":\[\([^]]*\)\].*/\1/p' | tr ',' '\n' | tr -d ' "')" + for tag in ${tags}; do + ct="$(curl -sfI -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \ + "${base}/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | awk 'tolower($1)=="content-type:"{print $2}')" + case "${ct}" in + *index*|*manifest.list*) + _err "tag ${repo}:${tag} is an OCI index/manifest list (${ct}); refusing to GC. Rebuild images with --provenance=false (see build scripts)." ;; + esac + _log " tag ${repo}:${tag} -> ${ct:-unknown}" + done + done +} + +assert_gc_safe + +if [[ "${DRY_RUN}" == "true" ]]; then + _log "dry-run: scanning ${STORAGE} (registry left running; nothing deleted)" + gc --dry-run -m + _log "dry-run complete; no changes made." + exit 0 +fi + +# Never race a build/push: the blob graph must be stable while we sweep. +if pgrep -f 'bootstrap.sh|build-full-image.sh|build-sbo-testbuild.sh' >/dev/null; then + _err "a build script is running; refusing to GC. Re-run after the chain finishes." +fi + +stopped=false +restart_registry() { + [[ "${stopped}" == "true" ]] || return 0 + _log "restarting ${REGISTRY_CONTAINER}..." + docker start "${REGISTRY_CONTAINER}" >/dev/null + local i + for i in $(seq 1 30); do + curl -sf -o /dev/null "http://localhost:${REGISTRY_PORT}/v2/" && break + sleep 1 + done + stopped=false +} +trap 'restart_registry; rm -rf "${CFG_DIR}"' EXIT + +before_size="$(du -sh "${STORAGE}" 2>/dev/null | cut -f1)" +snap_tag="${REGISTRY}/sbo-base:current" +snap_digest="$(docker inspect --format '{{index .RepoDigests 0}}' "${snap_tag}" 2>/dev/null || true)" + +_log "stopping ${REGISTRY_CONTAINER} for a stable blob graph..." +docker stop "${REGISTRY_CONTAINER}" >/dev/null && stopped=true + +_log "garbage-collect dry-run (audit; deletes nothing)..." +gc --dry-run -m + +_log "garbage-collect (untagged manifests + their blobs; tags preserved)..." +gc -m + +restart_registry + +after_size="$(du -sh "${STORAGE}" 2>/dev/null | cut -f1)" +_log "storage: ${before_size} -> ${after_size}" + +if [[ -n "${snap_digest}" ]]; then + if docker pull "${snap_tag}" >/dev/null 2>&1; then + new_digest="$(docker inspect --format '{{index .RepoDigests 0}}' "${snap_tag}" 2>/dev/null || true)" + if [[ "${new_digest}" == "${snap_digest}" ]]; then + _log "verify OK: ${snap_tag} still resolves, digest unchanged" + else + _warn "verify: ${snap_tag} digest changed: ${snap_digest} -> ${new_digest}" + fi + else + _warn "verify FAILED: ${snap_tag} no longer pulls after GC" + fi +fi + +_log "=== Done ===" diff --git a/image-builder/test-image-builder.sh b/image-builder/test-image-builder.sh index 5781b22..19f3ff8 100755 --- a/image-builder/test-image-builder.sh +++ b/image-builder/test-image-builder.sh @@ -83,6 +83,12 @@ DOCKER_RC=0 ( require_docker ) 2>/dev/null; check_rc "require_docker daemon up" 0 $? unset -f docker +# --- registry-gc.sh: -V / --help parse without touching docker; syntax valid --- +gc_ver="$(sed -n 's/^PROJECT_VERSION="\([^"]*\)".*/\1/p' registry-gc.sh)" +check "registry-gc.sh -V" "registry-gc.sh $gc_ver" "$(bash ./registry-gc.sh -V)" +bash -n registry-gc.sh; check_rc "registry-gc.sh syntax" 0 $? +bash ./registry-gc.sh --help >/dev/null 2>&1; check_rc "registry-gc.sh --help" 0 $? + echo "----" echo "PASS: $pass FAIL: $fail" [[ "$fail" -eq 0 ]] -- cgit v1.2.3