sbb-remote 0.8.0: bundled deps from upstream's deps list

hint-update downloads every URL the version bump changed for its md5, not
only the first (a vendor tarball carrying the version kept its old md5).
A package named in the new [manifests] table (default: neovim's
cmake.deps/deps.txt) has its extra DOWNLOAD URLs matched to the deps list
at the new version, by GitHub repo then by stem, and moved where the
version differs: mkhint's url mode. manifest_packages and the refusal of
those packages are gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

4 files changed, 196 insertions, 20 deletions

diff --git a/remote/config.example.toml b/remote/config.example.toml
index 0fbe801..18ade4c 100644
--- a/remote/config.example.toml
+++ b/remote/config.example.toml
@@ -12,5 +12,9 @@ hintdir         = "/etc/slackrepo/SBo-danix/hintfiles"
 default_hintdir = ""                     # searched after hintdir when set
 genreprc        = "/root/.genreprc"      # REPOSOWNERGPG names the signing key
 slackrepo_logdir = "/var/log/slackrepo/SBo-danix"
-manifest_packages = ["neovim"]           # bundled deps: hint-update refuses them (phase 4b)
 phantom_deps_file = "/etc/sb-builder/phantom-deps" # deps -current ships; one per line, # comments
+
+# bundled deps: package = its upstream deps list URL, {VERSION} = the new version.
+# hint-update moves the package's extra DOWNLOAD URLs to the versions listed there.
+[manifests]
+neovim = "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt"
diff --git a/remote/sbb-remote b/remote/sbb-remote
index c37b44c..07c2a55 100755
--- a/remote/sbb-remote
+++ b/remote/sbb-remote
@@ -40,7 +40,7 @@ import time
 import tomllib
 import urllib.request
 
-VERSION = "0.7.0"
+VERSION = "0.8.0"
 DEFAULTS = {
     "state_dir": "/var/lib/sb-builder",
     "repo_dir": "/var/lib/sbopkg/SBo-danix",
@@ -53,7 +53,8 @@ DEFAULTS = {
     "default_hintdir": "",
     "genreprc": "/root/.genreprc",
     "slackrepo_logdir": "/var/log/slackrepo/SBo-danix",
-    "manifest_packages": ["neovim"],  # bundled deps: left to mkhint until phase 4b
+    # bundled deps: package -> deps list URL, {VERSION} the new version
+    "manifests": {"neovim": "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt"},
     "phantom_deps_file": "/etc/sb-builder/phantom-deps",  # deps -current ships: DELREQUIREd
 }
 FLAGS = ("--yes", "--no")
@@ -90,9 +91,10 @@ def load_config(path=None):
         if type(v) is not type(DEFAULTS[k]):
             raise Error(f"config: {k}: expected {type(DEFAULTS[k]).__name__}")
         cfg[k] = v
-    for k in ("overlays", "manifest_packages"):
-        if not all(isinstance(o, str) for o in cfg[k]):
-            raise Error(f"config: {k}: expected list of strings")
+    if not all(isinstance(o, str) for o in cfg["overlays"]):
+        raise Error("config: overlays: expected list of strings")
+    if not all(isinstance(u, str) for u in cfg["manifests"].values()):
+        raise Error("config: manifests: expected strings")
     return cfg
 
 
@@ -187,8 +189,6 @@ def hint_args(cfg, kind, args):
         if pkg in seen:
             raise Error(f"{pkg} named twice")
         seen.add(pkg)
-        if ver is not None and pkg in cfg["manifest_packages"]:
-            raise Error(f"{pkg}: bundled deps: use mkhint until phase 4b")
         if kind == "hint-new" and os.path.exists(target):
             raise Error(f"{pkg}: hint exists")
     return pairs, no_dl
@@ -710,28 +710,128 @@ def _md5_url(url):
     return h.hexdigest()
 
 
-def apply_version(text, info, new, no_dl, log):
+def _fetch(url):
+    req = urllib.request.Request(url, headers={"User-Agent": "sbb-remote"})
+    with urllib.request.urlopen(req, timeout=DL_TIMEOUT) as r:
+        return r.read().decode(errors="replace")
+
+
+# ── bundled deps: a hint's extra DOWNLOAD URLs against upstream's deps list ──
+# mkhint's url mode. Both sides of a pair reduce to "the part that isn't the
+# name", so one version in two URL shapes (v0.13.tar.gz, lua-compat-5.3-0.13.tar.gz)
+# compares equal.
+
+MANIFEST_URL = re.compile(r"""https?://[^\s"'<>()]+""")
+ARCHIVE = re.compile(r"\.(?:tar\.(?:gz|xz|bz2|zst)|tgz|zip)$")
+
+
+def _url_repo(url):
+    """owner/repo of a GitHub URL, lowercased (GitHub ignores case); "" elsewhere.
+
+    neovim/deps is a shared blob repo that names the host, not the dep: "".
+    """
+    m = re.search(r"github\.com/([^/]+)/([^/]+)", url)
+    repo = f"{m.group(1)}/{m.group(2).removesuffix('.git')}".lower() if m else ""
+    return "" if repo.endswith("/deps") else repo
+
+
+def _url_base(url):
+    return ARCHIVE.sub("", url.rsplit("/", 1)[-1])
+
+
+def _url_stem(url):
+    """The basename without archive extension and trailing -<version> or -<sha>."""
+    m = re.match(r"(.+)-(?:[vV]?\d.*|[0-9a-f]{7,})$", _url_base(url))
+    return m.group(1) if m else _url_base(url)
+
+
+def _url_version(url):
+    """On GitHub the basename past "<repo>-" (the whole basename when it lacks
+    it); elsewhere its trailing -<version> or -<sha>. No leading v, '-' as '_'.
+
+    ponytail: off GitHub the name is a guess; a "<name-with-digits>-<ver>"
+    basename there misparses. Add a name hint if such a dep ever appears.
+    """
+    base, repo = _url_base(url), _url_repo(url).split("/")[-1]
+    if repo:
+        ver = base[len(repo) + 1:] if base.lower().startswith((repo + "-", repo + "_")) else base
+    else:
+        m = re.search(r"-([vV]?\d[^/]*)$", base) or re.search(r"-([0-9a-f]{7,})$", base)
+        ver = m.group(1) if m else base
+    return ver.removeprefix("v").removeprefix("V").replace("-", "_")
+
+
+def _dep_name(url):
+    return _url_repo(url).split("/")[-1] or _url_stem(url)
+
+
+def _match(url, found):
+    """The manifest URL for a hint URL: same GitHub repo, else same stem."""
+    repo = _url_repo(url)
+    m = repo and next((f for f in found if _url_repo(f) == repo), None)
+    return m or next((f for f in found if _url_stem(f) == _url_stem(url)), None)
+
+
+def reconcile(text, manifest, log):
+    """DOWNLOAD's extra URLs moved to the versions of manifest, a deps list's text.
+
+    The first URL is the package itself: the version swap moved it already.
+    """
+    found = sorted(set(MANIFEST_URL.findall(manifest)))
+    if not found:
+        raise Error("manifest has no URLs")
+    urls = hint_tokens(text, "DOWNLOAD")
+    used = {_match(urls[0], found)} if urls else set()
+    for i, url in enumerate(urls[1:], 1):
+        m = _match(url, found)
+        if m is None:
+            log(f"{_dep_name(url)} no manifest match, left as is")
+            continue
+        used.add(m)
+        old, new = _url_version(url), _url_version(m)
+        if old != new:
+            log(f"{_dep_name(url)} {old} -> {new}")
+            urls[i] = m
+    if extra := len(set(found) - used):
+        log(f"manifest has {extra} deps not in the hint")
+    if urls == hint_tokens(text, "DOWNLOAD"):
+        return text
+    return hint_set(text, "DOWNLOAD", " \\\n    ".join(urls))
+
+
+def apply_version(text, info, new, no_dl, log, manifest=None):
     """The hint text at version new (None: keep the version), log(str) per download.
 
-    An unpinned hint (DELREQUIRES only) gets the .info pin first. Downloads the
-    first URL of each DOWNLOAD pair for its md5; continuation URLs keep theirs,
-    as in mkhint. no_dl: no download, DOWNLOAD="no" in place of every
-    DOWNLOAD*/MD5SUM* line, since DOWNLOAD_$ARCH would override it.
+    An unpinned hint (DELREQUIRES only) gets the .info pin first. manifest: the
+    package's deps list URL template; its extra DOWNLOAD URLs follow it.
+    Downloads the first URL of each DOWNLOAD pair and every URL the bump
+    changed, for its md5; the others keep theirs. no_dl: no download,
+    DOWNLOAD="no" in place of every DOWNLOAD*/MD5SUM* line, since
+    DOWNLOAD_$ARCH would override it.
     """
     if not hint_tokens(text, "VERSION"):
         text = _pin(info) + ("" if hint_tokens(text, "ARCH") else 'ARCH="x86_64"\n') + text
+    before = {dl: hint_tokens(text, dl) for dl, _ in DL_PAIRS}
     if new is not None:
         text = swap_version(text, hint_tokens(text, "VERSION")[0], new)
     if no_dl:
         return hint_drop(text, r"(?:DOWNLOAD|MD5SUM)\w*") + 'DOWNLOAD="no"\n'
     if new is None:
         return text
+    if manifest:
+        url = manifest.replace("{VERSION}", new)
+        log(f"manifest {url}")
+        text = reconcile(text, _fetch(url), log)
     for dl, sums in DL_PAIRS:
-        urls = hint_tokens(text, dl)
+        urls, old = hint_tokens(text, dl), before[dl]
         if not urls or urls[0] in SKIP_URLS:
             continue
-        log(f"downloading {urls[0]}")
-        text = hint_set(text, sums, " \\\n    ".join([_md5_url(urls[0]), *hint_tokens(text, sums)[1:]]))
+        md5s = (hint_tokens(text, sums) + [""] * len(urls))[:len(urls)]
+        for i, url in enumerate(urls):
+            if i == 0 or i >= len(old) or url != old[i]:
+                log(f"downloading {url}")
+                md5s[i] = _md5_url(url)
+        text = hint_set(text, sums, " \\\n    ".join(md5s))
     return text
 
 
@@ -864,7 +964,8 @@ def cmd_hint(cfg, kind=None, *args):
             if text is None:
                 text = new_hint(info)
             _save(target, apply_version(text, info, ver, no_dl,
-                                        lambda s, p=pkg: print(f"{p}: {s}", flush=True)))
+                                        lambda s, p=pkg: print(f"{p}: {s}", flush=True),
+                                        cfg["manifests"].get(pkg)))
             print(f"{pkg}: ok", flush=True)
             ok.append(pkg)
         except (Error, OSError) as e:
diff --git a/sb_builder/__init__.py b/sb_builder/__init__.py
index f4a1f98..e4a5697 100644
--- a/sb_builder/__init__.py
+++ b/sb_builder/__init__.py
@@ -16,4 +16,4 @@
 
 """sb-builder: Qt6 GUI driving the SlackBuild tooling on a build VM."""
 
-VERSION = "0.7.0"  # must match remote/sbb-remote VERSION
+VERSION = "0.8.0"  # must match remote/sbb-remote VERSION
diff --git a/tests/test_remote.py b/tests/test_remote.py
index 57d492c..30bdc73 100644
--- a/tests/test_remote.py
+++ b/tests/test_remote.py
@@ -135,6 +135,12 @@ class ConfigTest(Base):
         with self.assertRaisesRegex(sbb.Error, "bogus: unknown key"):
             sbb.load_config(self.write("bogus = 1\n"))
 
+    def test_manifests(self):
+        self.assertEqual(sbb.load_config(self.write("[manifests]\nx = \"u{VERSION}\"\n"))["manifests"],
+                         {"x": "u{VERSION}"})
+        with self.assertRaisesRegex(sbb.Error, "manifests: expected strings"):
+            sbb.load_config(self.write("[manifests]\nx = 1\n"))
+
     def test_wrong_type(self):
         with self.assertRaisesRegex(sbb.Error, "state_dir: expected str"):
             sbb.load_config(self.write("state_dir = 1\n"))
@@ -302,6 +308,73 @@ class ApplyVersionTest(Base):
             self.apply(f'VERSION="1.0"\nDOWNLOAD="{self.www.url}/x-1.0.tgz"\nMD5SUM="m"\n', "3.0")
 
 
+GH = "https://github.com"
+# neovim 0.12.4: hint URLs (SBo shapes) and their deps.txt counterparts
+NVIM_HINT = [f"{GH}/neovim/neovim/archive/v0.12.4/neovim-0.12.4.tar.gz",
+             f"{GH}/luajit/LuaJIT/archive/fbb36bb/LuaJIT-fbb36bb6bfa88716a47c58bcf9ce9f2ef752abac.tar.gz",
+             f"{GH}/luvit/luv/releases/download/1.52.1-0/luv-1.52.1-0.tar.gz",
+             f"{GH}/lunarmodules/lua-compat-5.3/archive/v0.13/lua-compat-5.3-0.13.tar.gz",
+             f"{GH}/tree-sitter/tree-sitter/archive/v0.26.7/tree-sitter-0.26.7.tar.gz",
+             "https://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.1.0.tar.gz",
+             f"{GH}/JuliaStrings/utf8proc/archive/v2.11.3/utf8proc-2.11.3.tar.gz"]
+NVIM_DEPS = [f"{GH}/luajit/luajit/archive/fbb36bb6bfa88716a47c58bcf9ce9f2ef752abac.tar.gz",
+             f"{GH}/luvit/luv/archive/1.52.1-0.tar.gz",
+             f"{GH}/lunarmodules/lua-compat-5.3/archive/v0.13.tar.gz",
+             f"{GH}/tree-sitter/tree-sitter-c/archive/v0.24.1.tar.gz",
+             f"{GH}/tree-sitter/tree-sitter/archive/v0.26.7.tar.gz",
+             f"{GH}/neovim/deps/raw/d495ee6f79e7962a53ad79670cb92488abe0b9b4/opt/lpeg-1.1.0.tar.gz",
+             f"{GH}/juliastrings/utf8proc/archive/v2.11.3.tar.gz"]
+
+
+class BundledDepsTest(Base):
+    def test_hint_and_manifest_shapes_match_with_equal_versions(self):
+        for url in NVIM_HINT[1:]:
+            m = sbb._match(url, NVIM_DEPS)
+            with self.subTest(url=url):
+                self.assertIsNotNone(m)
+                self.assertEqual(sbb._url_version(url), sbb._url_version(m))
+        self.assertIsNone(sbb._match(NVIM_HINT[0], NVIM_DEPS))
+        self.assertEqual([sbb._url_version(u) for u in NVIM_HINT[2:4]], ["1.52.1_0", "0.13"])
+        self.assertEqual(sbb._dep_name(NVIM_HINT[5]), "lpeg")
+
+    def test_reconcile(self):
+        hint = 'DOWNLOAD="' + " \\\n    ".join(NVIM_HINT[:3] + [f"{GH}/a/gone/archive/v1/gone-1.tar.gz"]) + '"\n'
+        deps = "\n".join(f"X_URL {u}\nX_SHA256 ab" for u in NVIM_DEPS).replace("1.52.1-0", "1.53.0-0")
+        log = []
+        out = sbb.reconcile(hint, deps, log.append)
+        self.assertEqual(sbb.hint_tokens(out, "DOWNLOAD"),
+                         NVIM_HINT[:2] + [f"{GH}/luvit/luv/archive/1.53.0-0.tar.gz",
+                                          f"{GH}/a/gone/archive/v1/gone-1.tar.gz"])
+        self.assertEqual(log, ["luv 1.52.1_0 -> 1.53.0_0", "gone no manifest match, left as is",
+                               "manifest has 5 deps not in the hint"])
+        self.assertIs(sbb.reconcile(hint, deps.replace("1.53.0-0", "1.52.1-0"), log.append), hint)
+        with self.assertRaisesRegex(sbb.Error, "no URLs"):
+            sbb.reconcile(hint, "moved\n", log.append)
+
+    def test_apply_version_follows_manifest(self):
+        www = Served(self, {"x-2.0.tgz": b"two", "dep-0.6.tgz": b"dep11", "keep-3.tgz": b"keep"})
+        u, log = www.url, []
+        with open(os.path.join(self.tmp, "www", "deps-2.0.txt"), "w") as f:
+            f.write(f"DEP_URL {u}/dep-0.6.tgz\nKEEP_URL {u}/keep-3.tgz\n")
+        text = (f'VERSION="1.0"\nDOWNLOAD="{u}/x-1.0.tgz \\\n    {u}/dep-0.5.tgz \\\n    {u}/keep-3.tgz"\n'
+                'MD5SUM="a \\\n    b \\\n    c"\n')
+        out = sbb.apply_version(text, "", "2.0", False, log.append, u + "/deps-{VERSION}.txt")
+        self.assertEqual(out, f'VERSION="2.0"\nDOWNLOAD="{u}/x-2.0.tgz \\\n    {u}/dep-0.6.tgz \\\n'
+                              f'    {u}/keep-3.tgz"\nMD5SUM="{md5(b"two")} \\\n    {md5(b"dep11")} \\\n    c"\n')
+        self.assertEqual(log, [f"manifest {u}/deps-2.0.txt", "dep 0.5 -> 0.6",
+                               f"downloading {u}/x-2.0.tgz", f"downloading {u}/dep-0.6.tgz"])
+        with self.assertRaises(OSError):  # no deps list at 3.0: the package fails, hint untouched
+            sbb.apply_version(text, "", "3.0", False, log.append, u + "/deps-{VERSION}.txt")
+
+    def test_changed_continuation_url_gets_its_md5(self):
+        www = Served(self, {"f-2.0.tgz": b"two", "f-2.0-vendor.tgz": b"vendor"})
+        u, log = www.url, []
+        text = f'VERSION="1.0"\nDOWNLOAD="{u}/f-1.0.tgz \\\n    {u}/f-1.0-vendor.tgz"\nMD5SUM="a \\\n    b"\n'
+        self.assertIn(f'MD5SUM="{md5(b"two")} \\\n    {md5(b"vendor")}"',
+                      sbb.apply_version(text, "", "2.0", False, log.append))
+        self.assertEqual(log, [f"downloading {u}/f-2.0.tgz", f"downloading {u}/f-2.0-vendor.tgz"])
+
+
 class HintJobsTest(Base):
     def setUp(self):
         super().setUp()
@@ -349,14 +422,12 @@ class HintJobsTest(Base):
                                 ("hint-update", ["nope=1"], "no such package"),
                                 ("hint-update", ["x=1", "x=2"], "twice"),
                                 ("hint-update", ["x=1", "--force"], "unknown option"),
-                                ("hint-update", ["neovim=1"], "bundled deps"),
                                 ("hint-update", ["shadowed=1"], "not managed"),
                                 ("hint-new", [], "hint-new <pkg>"),
                                 ("hint-new", ["x", "2.0"], "no such package: 2.0"),  # no version any more
                                 ("hint-new", ["x", "x"], "twice")):
             with self.subTest(kind=kind, args=args), self.assertRaisesRegex(sbb.Error, why):
                 sbb.KINDS[kind](self.cfg, args)
-        sbb.KINDS["hint-new"](self.cfg, ["neovim"])  # no version: no bundled deps to touch
         self.make_pkg("misc/y", self.info, hint="")
         with self.assertRaisesRegex(sbb.Error, "y: hint exists"):
             sbb.KINDS["hint-new"](self.cfg, ["x", "y"])  # one with a hint refuses the whole job