diff options
| -rw-r--r-- | remote/config.example.toml | 6 | ||||
| -rwxr-xr-x | remote/sbb-remote | 133 | ||||
| -rw-r--r-- | sb_builder/__init__.py | 2 | ||||
| -rw-r--r-- | tests/test_remote.py | 75 |
4 files changed, 196 insertions, 20 deletions
diff --git a/remote/config.example.toml b/remote/config.example.toml index 0fbe801..18ade4c 100644 --- a/remote/config.example.toml +++ b/remote/config.example.toml @@ -12,5 +12,9 @@ hintdir = "/etc/slackrepo/SBo-danix/hintfiles" default_hintdir = "" # searched after hintdir when set genreprc = "/root/.genreprc" # REPOSOWNERGPG names the signing key slackrepo_logdir = "/var/log/slackrepo/SBo-danix" -manifest_packages = ["neovim"] # bundled deps: hint-update refuses them (phase 4b) phantom_deps_file = "/etc/sb-builder/phantom-deps" # deps -current ships; one per line, # comments + +# bundled deps: package = its upstream deps list URL, {VERSION} = the new version. +# hint-update moves the package's extra DOWNLOAD URLs to the versions listed there. +[manifests] +neovim = "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt" diff --git a/remote/sbb-remote b/remote/sbb-remote index c37b44c..07c2a55 100755 --- a/remote/sbb-remote +++ b/remote/sbb-remote @@ -40,7 +40,7 @@ import time import tomllib import urllib.request -VERSION = "0.7.0" +VERSION = "0.8.0" DEFAULTS = { "state_dir": "/var/lib/sb-builder", "repo_dir": "/var/lib/sbopkg/SBo-danix", @@ -53,7 +53,8 @@ DEFAULTS = { "default_hintdir": "", "genreprc": "/root/.genreprc", "slackrepo_logdir": "/var/log/slackrepo/SBo-danix", - "manifest_packages": ["neovim"], # bundled deps: left to mkhint until phase 4b + # bundled deps: package -> deps list URL, {VERSION} the new version + "manifests": {"neovim": "https://raw.githubusercontent.com/neovim/neovim/v{VERSION}/cmake.deps/deps.txt"}, "phantom_deps_file": "/etc/sb-builder/phantom-deps", # deps -current ships: DELREQUIREd } FLAGS = ("--yes", "--no") @@ -90,9 +91,10 @@ def load_config(path=None): if type(v) is not type(DEFAULTS[k]): raise Error(f"config: {k}: expected {type(DEFAULTS[k]).__name__}") cfg[k] = v - for k in ("overlays", "manifest_packages"): - if not all(isinstance(o, str) for o in cfg[k]): - raise Error(f"config: {k}: expected list of strings") + if not all(isinstance(o, str) for o in cfg["overlays"]): + raise Error("config: overlays: expected list of strings") + if not all(isinstance(u, str) for u in cfg["manifests"].values()): + raise Error("config: manifests: expected strings") return cfg @@ -187,8 +189,6 @@ def hint_args(cfg, kind, args): if pkg in seen: raise Error(f"{pkg} named twice") seen.add(pkg) - if ver is not None and pkg in cfg["manifest_packages"]: - raise Error(f"{pkg}: bundled deps: use mkhint until phase 4b") if kind == "hint-new" and os.path.exists(target): raise Error(f"{pkg}: hint exists") return pairs, no_dl @@ -710,28 +710,128 @@ def _md5_url(url): return h.hexdigest() -def apply_version(text, info, new, no_dl, log): +def _fetch(url): + req = urllib.request.Request(url, headers={"User-Agent": "sbb-remote"}) + with urllib.request.urlopen(req, timeout=DL_TIMEOUT) as r: + return r.read().decode(errors="replace") + + +# ── bundled deps: a hint's extra DOWNLOAD URLs against upstream's deps list ── +# mkhint's url mode. Both sides of a pair reduce to "the part that isn't the +# name", so one version in two URL shapes (v0.13.tar.gz, lua-compat-5.3-0.13.tar.gz) +# compares equal. + +MANIFEST_URL = re.compile(r"""https?://[^\s"'<>()]+""") +ARCHIVE = re.compile(r"\.(?:tar\.(?:gz|xz|bz2|zst)|tgz|zip)$") + + +def _url_repo(url): + """owner/repo of a GitHub URL, lowercased (GitHub ignores case); "" elsewhere. + + neovim/deps is a shared blob repo that names the host, not the dep: "". + """ + m = re.search(r"github\.com/([^/]+)/([^/]+)", url) + repo = f"{m.group(1)}/{m.group(2).removesuffix('.git')}".lower() if m else "" + return "" if repo.endswith("/deps") else repo + + +def _url_base(url): + return ARCHIVE.sub("", url.rsplit("/", 1)[-1]) + + +def _url_stem(url): + """The basename without archive extension and trailing -<version> or -<sha>.""" + m = re.match(r"(.+)-(?:[vV]?\d.*|[0-9a-f]{7,})$", _url_base(url)) + return m.group(1) if m else _url_base(url) + + +def _url_version(url): + """On GitHub the basename past "<repo>-" (the whole basename when it lacks + it); elsewhere its trailing -<version> or -<sha>. No leading v, '-' as '_'. + + ponytail: off GitHub the name is a guess; a "<name-with-digits>-<ver>" + basename there misparses. Add a name hint if such a dep ever appears. + """ + base, repo = _url_base(url), _url_repo(url).split("/")[-1] + if repo: + ver = base[len(repo) + 1:] if base.lower().startswith((repo + "-", repo + "_")) else base + else: + m = re.search(r"-([vV]?\d[^/]*)$", base) or re.search(r"-([0-9a-f]{7,})$", base) + ver = m.group(1) if m else base + return ver.removeprefix("v").removeprefix("V").replace("-", "_") + + +def _dep_name(url): + return _url_repo(url).split("/")[-1] or _url_stem(url) + + +def _match(url, found): + """The manifest URL for a hint URL: same GitHub repo, else same stem.""" + repo = _url_repo(url) + m = repo and next((f for f in found if _url_repo(f) == repo), None) + return m or next((f for f in found if _url_stem(f) == _url_stem(url)), None) + + +def reconcile(text, manifest, log): + """DOWNLOAD's extra URLs moved to the versions of manifest, a deps list's text. + + The first URL is the package itself: the version swap moved it already. + """ + found = sorted(set(MANIFEST_URL.findall(manifest))) + if not found: + raise Error("manifest has no URLs") + urls = hint_tokens(text, "DOWNLOAD") + used = {_match(urls[0], found)} if urls else set() + for i, url in enumerate(urls[1:], 1): + m = _match(url, found) + if m is None: + log(f"{_dep_name(url)} no manifest match, left as is") + continue + used.add(m) + old, new = _url_version(url), _url_version(m) + if old != new: + log(f"{_dep_name(url)} {old} -> {new}") + urls[i] = m + if extra := len(set(found) - used): + log(f"manifest has {extra} deps not in the hint") + if urls == hint_tokens(text, "DOWNLOAD"): + return text + return hint_set(text, "DOWNLOAD", " \\\n ".join(urls)) + + +def apply_version(text, info, new, no_dl, log, manifest=None): """The hint text at version new (None: keep the version), log(str) per download. - An unpinned hint (DELREQUIRES only) gets the .info pin first. Downloads the - first URL of each DOWNLOAD pair for its md5; continuation URLs keep theirs, - as in mkhint. no_dl: no download, DOWNLOAD="no" in place of every - DOWNLOAD*/MD5SUM* line, since DOWNLOAD_$ARCH would override it. + An unpinned hint (DELREQUIRES only) gets the .info pin first. manifest: the + package's deps list URL template; its extra DOWNLOAD URLs follow it. + Downloads the first URL of each DOWNLOAD pair and every URL the bump + changed, for its md5; the others keep theirs. no_dl: no download, + DOWNLOAD="no" in place of every DOWNLOAD*/MD5SUM* line, since + DOWNLOAD_$ARCH would override it. """ if not hint_tokens(text, "VERSION"): text = _pin(info) + ("" if hint_tokens(text, "ARCH") else 'ARCH="x86_64"\n') + text + before = {dl: hint_tokens(text, dl) for dl, _ in DL_PAIRS} if new is not None: text = swap_version(text, hint_tokens(text, "VERSION")[0], new) if no_dl: return hint_drop(text, r"(?:DOWNLOAD|MD5SUM)\w*") + 'DOWNLOAD="no"\n' if new is None: return text + if manifest: + url = manifest.replace("{VERSION}", new) + log(f"manifest {url}") + text = reconcile(text, _fetch(url), log) for dl, sums in DL_PAIRS: - urls = hint_tokens(text, dl) + urls, old = hint_tokens(text, dl), before[dl] if not urls or urls[0] in SKIP_URLS: continue - log(f"downloading {urls[0]}") - text = hint_set(text, sums, " \\\n ".join([_md5_url(urls[0]), *hint_tokens(text, sums)[1:]])) + md5s = (hint_tokens(text, sums) + [""] * len(urls))[:len(urls)] + for i, url in enumerate(urls): + if i == 0 or i >= len(old) or url != old[i]: + log(f"downloading {url}") + md5s[i] = _md5_url(url) + text = hint_set(text, sums, " \\\n ".join(md5s)) return text @@ -864,7 +964,8 @@ def cmd_hint(cfg, kind=None, *args): if text is None: text = new_hint(info) _save(target, apply_version(text, info, ver, no_dl, - lambda s, p=pkg: print(f"{p}: {s}", flush=True))) + lambda s, p=pkg: print(f"{p}: {s}", flush=True), + cfg["manifests"].get(pkg))) print(f"{pkg}: ok", flush=True) ok.append(pkg) except (Error, OSError) as e: diff --git a/sb_builder/__init__.py b/sb_builder/__init__.py index f4a1f98..e4a5697 100644 --- a/sb_builder/__init__.py +++ b/sb_builder/__init__.py @@ -16,4 +16,4 @@ """sb-builder: Qt6 GUI driving the SlackBuild tooling on a build VM.""" -VERSION = "0.7.0" # must match remote/sbb-remote VERSION +VERSION = "0.8.0" # must match remote/sbb-remote VERSION diff --git a/tests/test_remote.py b/tests/test_remote.py index 57d492c..30bdc73 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -135,6 +135,12 @@ class ConfigTest(Base): with self.assertRaisesRegex(sbb.Error, "bogus: unknown key"): sbb.load_config(self.write("bogus = 1\n")) + def test_manifests(self): + self.assertEqual(sbb.load_config(self.write("[manifests]\nx = \"u{VERSION}\"\n"))["manifests"], + {"x": "u{VERSION}"}) + with self.assertRaisesRegex(sbb.Error, "manifests: expected strings"): + sbb.load_config(self.write("[manifests]\nx = 1\n")) + def test_wrong_type(self): with self.assertRaisesRegex(sbb.Error, "state_dir: expected str"): sbb.load_config(self.write("state_dir = 1\n")) @@ -302,6 +308,73 @@ class ApplyVersionTest(Base): self.apply(f'VERSION="1.0"\nDOWNLOAD="{self.www.url}/x-1.0.tgz"\nMD5SUM="m"\n', "3.0") +GH = "https://github.com" +# neovim 0.12.4: hint URLs (SBo shapes) and their deps.txt counterparts +NVIM_HINT = [f"{GH}/neovim/neovim/archive/v0.12.4/neovim-0.12.4.tar.gz", + f"{GH}/luajit/LuaJIT/archive/fbb36bb/LuaJIT-fbb36bb6bfa88716a47c58bcf9ce9f2ef752abac.tar.gz", + f"{GH}/luvit/luv/releases/download/1.52.1-0/luv-1.52.1-0.tar.gz", + f"{GH}/lunarmodules/lua-compat-5.3/archive/v0.13/lua-compat-5.3-0.13.tar.gz", + f"{GH}/tree-sitter/tree-sitter/archive/v0.26.7/tree-sitter-0.26.7.tar.gz", + "https://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.1.0.tar.gz", + f"{GH}/JuliaStrings/utf8proc/archive/v2.11.3/utf8proc-2.11.3.tar.gz"] +NVIM_DEPS = [f"{GH}/luajit/luajit/archive/fbb36bb6bfa88716a47c58bcf9ce9f2ef752abac.tar.gz", + f"{GH}/luvit/luv/archive/1.52.1-0.tar.gz", + f"{GH}/lunarmodules/lua-compat-5.3/archive/v0.13.tar.gz", + f"{GH}/tree-sitter/tree-sitter-c/archive/v0.24.1.tar.gz", + f"{GH}/tree-sitter/tree-sitter/archive/v0.26.7.tar.gz", + f"{GH}/neovim/deps/raw/d495ee6f79e7962a53ad79670cb92488abe0b9b4/opt/lpeg-1.1.0.tar.gz", + f"{GH}/juliastrings/utf8proc/archive/v2.11.3.tar.gz"] + + +class BundledDepsTest(Base): + def test_hint_and_manifest_shapes_match_with_equal_versions(self): + for url in NVIM_HINT[1:]: + m = sbb._match(url, NVIM_DEPS) + with self.subTest(url=url): + self.assertIsNotNone(m) + self.assertEqual(sbb._url_version(url), sbb._url_version(m)) + self.assertIsNone(sbb._match(NVIM_HINT[0], NVIM_DEPS)) + self.assertEqual([sbb._url_version(u) for u in NVIM_HINT[2:4]], ["1.52.1_0", "0.13"]) + self.assertEqual(sbb._dep_name(NVIM_HINT[5]), "lpeg") + + def test_reconcile(self): + hint = 'DOWNLOAD="' + " \\\n ".join(NVIM_HINT[:3] + [f"{GH}/a/gone/archive/v1/gone-1.tar.gz"]) + '"\n' + deps = "\n".join(f"X_URL {u}\nX_SHA256 ab" for u in NVIM_DEPS).replace("1.52.1-0", "1.53.0-0") + log = [] + out = sbb.reconcile(hint, deps, log.append) + self.assertEqual(sbb.hint_tokens(out, "DOWNLOAD"), + NVIM_HINT[:2] + [f"{GH}/luvit/luv/archive/1.53.0-0.tar.gz", + f"{GH}/a/gone/archive/v1/gone-1.tar.gz"]) + self.assertEqual(log, ["luv 1.52.1_0 -> 1.53.0_0", "gone no manifest match, left as is", + "manifest has 5 deps not in the hint"]) + self.assertIs(sbb.reconcile(hint, deps.replace("1.53.0-0", "1.52.1-0"), log.append), hint) + with self.assertRaisesRegex(sbb.Error, "no URLs"): + sbb.reconcile(hint, "moved\n", log.append) + + def test_apply_version_follows_manifest(self): + www = Served(self, {"x-2.0.tgz": b"two", "dep-0.6.tgz": b"dep11", "keep-3.tgz": b"keep"}) + u, log = www.url, [] + with open(os.path.join(self.tmp, "www", "deps-2.0.txt"), "w") as f: + f.write(f"DEP_URL {u}/dep-0.6.tgz\nKEEP_URL {u}/keep-3.tgz\n") + text = (f'VERSION="1.0"\nDOWNLOAD="{u}/x-1.0.tgz \\\n {u}/dep-0.5.tgz \\\n {u}/keep-3.tgz"\n' + 'MD5SUM="a \\\n b \\\n c"\n') + out = sbb.apply_version(text, "", "2.0", False, log.append, u + "/deps-{VERSION}.txt") + self.assertEqual(out, f'VERSION="2.0"\nDOWNLOAD="{u}/x-2.0.tgz \\\n {u}/dep-0.6.tgz \\\n' + f' {u}/keep-3.tgz"\nMD5SUM="{md5(b"two")} \\\n {md5(b"dep11")} \\\n c"\n') + self.assertEqual(log, [f"manifest {u}/deps-2.0.txt", "dep 0.5 -> 0.6", + f"downloading {u}/x-2.0.tgz", f"downloading {u}/dep-0.6.tgz"]) + with self.assertRaises(OSError): # no deps list at 3.0: the package fails, hint untouched + sbb.apply_version(text, "", "3.0", False, log.append, u + "/deps-{VERSION}.txt") + + def test_changed_continuation_url_gets_its_md5(self): + www = Served(self, {"f-2.0.tgz": b"two", "f-2.0-vendor.tgz": b"vendor"}) + u, log = www.url, [] + text = f'VERSION="1.0"\nDOWNLOAD="{u}/f-1.0.tgz \\\n {u}/f-1.0-vendor.tgz"\nMD5SUM="a \\\n b"\n' + self.assertIn(f'MD5SUM="{md5(b"two")} \\\n {md5(b"vendor")}"', + sbb.apply_version(text, "", "2.0", False, log.append)) + self.assertEqual(log, [f"downloading {u}/f-2.0.tgz", f"downloading {u}/f-2.0-vendor.tgz"]) + + class HintJobsTest(Base): def setUp(self): super().setUp() @@ -349,14 +422,12 @@ class HintJobsTest(Base): ("hint-update", ["nope=1"], "no such package"), ("hint-update", ["x=1", "x=2"], "twice"), ("hint-update", ["x=1", "--force"], "unknown option"), - ("hint-update", ["neovim=1"], "bundled deps"), ("hint-update", ["shadowed=1"], "not managed"), ("hint-new", [], "hint-new <pkg>"), ("hint-new", ["x", "2.0"], "no such package: 2.0"), # no version any more ("hint-new", ["x", "x"], "twice")): with self.subTest(kind=kind, args=args), self.assertRaisesRegex(sbb.Error, why): sbb.KINDS[kind](self.cfg, args) - sbb.KINDS["hint-new"](self.cfg, ["neovim"]) # no version: no bundled deps to touch self.make_pkg("misc/y", self.info, hint="") with self.assertRaisesRegex(sbb.Error, "y: hint exists"): sbb.KINDS["hint-new"](self.cfg, ["x", "y"]) # one with a hint refuses the whole job |