From 562c3d20be083998aa6e126783e152bf06aa9829 Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Tue, 20 Oct 2026 19:45:08 +0200 Subject: sbb-remote 0.8.0: bundled deps from upstream's deps list hint-update downloads every URL the version bump changed for its md5, not only the first (a vendor tarball carrying the version kept its old md5). A package named in the new [manifests] table (default: neovim's cmake.deps/deps.txt) has its extra DOWNLOAD URLs matched to the deps list at the new version, by GitHub repo then by stem, and moved where the version differs: mkhint's url mode. manifest_packages and the refusal of those packages are gone. Co-Authored-By: Claude Opus 5.5 --- remote/config.example.toml | 6 +- remote/sbb-remote | 133 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 122 insertions(+), 17 deletions(-) (limited to 'remote') diff --git a/remote/config.example.toml b/remote/config.example.toml index 0fbe801..18ade4c 100644 --- a/remote/config.example.toml +++ b/remote/config.example.toml @@ -12,5 +12,9 @@ hintdir = "/etc/slackrepo/SBo-danix/hintfiles" default_hintdir = "" # searched after hintdir when set genreprc = "/root/.genreprc" # REPOSOWNERGPG names the signing key slackrepo_logdir = "/var/log/slackrepo/SBo-danix" -manifest_packages = ["neovim"] # bundled deps: hint-update refuses them (phase 4b) phantom_deps_file = "/etc/sb-builder/phantom-deps" # deps -current ships; one per line, # comments + +# bundled deps: package = its upstream deps list URL, {VERSION} = the new version. +# hint-update moves the package's extra DOWNLOAD URLs to the versions listed there. +[manifests] +neovim = "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt" diff --git a/remote/sbb-remote b/remote/sbb-remote index c37b44c..07c2a55 100755 --- a/remote/sbb-remote +++ b/remote/sbb-remote @@ -40,7 +40,7 @@ import time import tomllib import urllib.request -VERSION = "0.7.0" +VERSION = "0.8.0" DEFAULTS = { "state_dir": "/var/lib/sb-builder", "repo_dir": "/var/lib/sbopkg/SBo-danix", @@ -53,7 +53,8 @@ DEFAULTS = { "default_hintdir": "", "genreprc": "/root/.genreprc", "slackrepo_logdir": "/var/log/slackrepo/SBo-danix", - "manifest_packages": ["neovim"], # bundled deps: left to mkhint until phase 4b + # bundled deps: package -> deps list URL, {VERSION} the new version + "manifests": {"neovim": "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt"}, "phantom_deps_file": "/etc/sb-builder/phantom-deps", # deps -current ships: DELREQUIREd } FLAGS = ("--yes", "--no") @@ -90,9 +91,10 @@ def load_config(path=None): if type(v) is not type(DEFAULTS[k]): raise Error(f"config: {k}: expected {type(DEFAULTS[k]).__name__}") cfg[k] = v - for k in ("overlays", "manifest_packages"): - if not all(isinstance(o, str) for o in cfg[k]): - raise Error(f"config: {k}: expected list of strings") + if not all(isinstance(o, str) for o in cfg["overlays"]): + raise Error("config: overlays: expected list of strings") + if not all(isinstance(u, str) for u in cfg["manifests"].values()): + raise Error("config: manifests: expected strings") return cfg @@ -187,8 +189,6 @@ def hint_args(cfg, kind, args): if pkg in seen: raise Error(f"{pkg} named twice") seen.add(pkg) - if ver is not None and pkg in cfg["manifest_packages"]: - raise Error(f"{pkg}: bundled deps: use mkhint until phase 4b") if kind == "hint-new" and os.path.exists(target): raise Error(f"{pkg}: hint exists") return pairs, no_dl @@ -710,28 +710,128 @@ def _md5_url(url): return h.hexdigest() -def apply_version(text, info, new, no_dl, log): +def _fetch(url): + req = urllib.request.Request(url, headers={"User-Agent": "sbb-remote"}) + with urllib.request.urlopen(req, timeout=DL_TIMEOUT) as r: + return r.read().decode(errors="replace") + + +# ── bundled deps: a hint's extra DOWNLOAD URLs against upstream's deps list ── +# mkhint's url mode. Both sides of a pair reduce to "the part that isn't the +# name", so one version in two URL shapes (v0.13.tar.gz, lua-compat-5.3-0.13.tar.gz) +# compares equal. + +MANIFEST_URL = re.compile(r"""https?://[^\s"'<>()]+""") +ARCHIVE = re.compile(r"\.(?:tar\.(?:gz|xz|bz2|zst)|tgz|zip)$") + + +def _url_repo(url): + """owner/repo of a GitHub URL, lowercased (GitHub ignores case); "" elsewhere. + + neovim/deps is a shared blob repo that names the host, not the dep: "". + """ + m = re.search(r"github\.com/([^/]+)/([^/]+)", url) + repo = f"{m.group(1)}/{m.group(2).removesuffix('.git')}".lower() if m else "" + return "" if repo.endswith("/deps") else repo + + +def _url_base(url): + return ARCHIVE.sub("", url.rsplit("/", 1)[-1]) + + +def _url_stem(url): + """The basename without archive extension and trailing - or -.""" + m = re.match(r"(.+)-(?:[vV]?\d.*|[0-9a-f]{7,})$", _url_base(url)) + return m.group(1) if m else _url_base(url) + + +def _url_version(url): + """On GitHub the basename past "-" (the whole basename when it lacks + it); elsewhere its trailing - or -. No leading v, '-' as '_'. + + ponytail: off GitHub the name is a guess; a "-" + basename there misparses. Add a name hint if such a dep ever appears. + """ + base, repo = _url_base(url), _url_repo(url).split("/")[-1] + if repo: + ver = base[len(repo) + 1:] if base.lower().startswith((repo + "-", repo + "_")) else base + else: + m = re.search(r"-([vV]?\d[^/]*)$", base) or re.search(r"-([0-9a-f]{7,})$", base) + ver = m.group(1) if m else base + return ver.removeprefix("v").removeprefix("V").replace("-", "_") + + +def _dep_name(url): + return _url_repo(url).split("/")[-1] or _url_stem(url) + + +def _match(url, found): + """The manifest URL for a hint URL: same GitHub repo, else same stem.""" + repo = _url_repo(url) + m = repo and next((f for f in found if _url_repo(f) == repo), None) + return m or next((f for f in found if _url_stem(f) == _url_stem(url)), None) + + +def reconcile(text, manifest, log): + """DOWNLOAD's extra URLs moved to the versions of manifest, a deps list's text. + + The first URL is the package itself: the version swap moved it already. + """ + found = sorted(set(MANIFEST_URL.findall(manifest))) + if not found: + raise Error("manifest has no URLs") + urls = hint_tokens(text, "DOWNLOAD") + used = {_match(urls[0], found)} if urls else set() + for i, url in enumerate(urls[1:], 1): + m = _match(url, found) + if m is None: + log(f"{_dep_name(url)} no manifest match, left as is") + continue + used.add(m) + old, new = _url_version(url), _url_version(m) + if old != new: + log(f"{_dep_name(url)} {old} -> {new}") + urls[i] = m + if extra := len(set(found) - used): + log(f"manifest has {extra} deps not in the hint") + if urls == hint_tokens(text, "DOWNLOAD"): + return text + return hint_set(text, "DOWNLOAD", " \\\n ".join(urls)) + + +def apply_version(text, info, new, no_dl, log, manifest=None): """The hint text at version new (None: keep the version), log(str) per download. - An unpinned hint (DELREQUIRES only) gets the .info pin first. Downloads the - first URL of each DOWNLOAD pair for its md5; continuation URLs keep theirs, - as in mkhint. no_dl: no download, DOWNLOAD="no" in place of every - DOWNLOAD*/MD5SUM* line, since DOWNLOAD_$ARCH would override it. + An unpinned hint (DELREQUIRES only) gets the .info pin first. manifest: the + package's deps list URL template; its extra DOWNLOAD URLs follow it. + Downloads the first URL of each DOWNLOAD pair and every URL the bump + changed, for its md5; the others keep theirs. no_dl: no download, + DOWNLOAD="no" in place of every DOWNLOAD*/MD5SUM* line, since + DOWNLOAD_$ARCH would override it. """ if not hint_tokens(text, "VERSION"): text = _pin(info) + ("" if hint_tokens(text, "ARCH") else 'ARCH="x86_64"\n') + text + before = {dl: hint_tokens(text, dl) for dl, _ in DL_PAIRS} if new is not None: text = swap_version(text, hint_tokens(text, "VERSION")[0], new) if no_dl: return hint_drop(text, r"(?:DOWNLOAD|MD5SUM)\w*") + 'DOWNLOAD="no"\n' if new is None: return text + if manifest: + url = manifest.replace("{VERSION}", new) + log(f"manifest {url}") + text = reconcile(text, _fetch(url), log) for dl, sums in DL_PAIRS: - urls = hint_tokens(text, dl) + urls, old = hint_tokens(text, dl), before[dl] if not urls or urls[0] in SKIP_URLS: continue - log(f"downloading {urls[0]}") - text = hint_set(text, sums, " \\\n ".join([_md5_url(urls[0]), *hint_tokens(text, sums)[1:]])) + md5s = (hint_tokens(text, sums) + [""] * len(urls))[:len(urls)] + for i, url in enumerate(urls): + if i == 0 or i >= len(old) or url != old[i]: + log(f"downloading {url}") + md5s[i] = _md5_url(url) + text = hint_set(text, sums, " \\\n ".join(md5s)) return text @@ -864,7 +964,8 @@ def cmd_hint(cfg, kind=None, *args): if text is None: text = new_hint(info) _save(target, apply_version(text, info, ver, no_dl, - lambda s, p=pkg: print(f"{p}: {s}", flush=True))) + lambda s, p=pkg: print(f"{p}: {s}", flush=True), + cfg["manifests"].get(pkg))) print(f"{pkg}: ok", flush=True) ok.append(pkg) except (Error, OSError) as e: -- cgit v1.2.3