From c27fe5e9334572e00fedd4aa67d90b236c5da3b7 Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Mon, 10 Aug 2026 18:24:24 +0200 Subject: feat: fan model listing out across providers and route by id A provider that is unreachable or whose filter matched nothing is reported rather than fatal: local models must stay usable when the network is down. Clients are built on demand so a key is resolved only when that provider is really used. The model listing carries its own timeout, separate from the per-turn one: the picker has to populate fast, and a dead cloud provider must not hang it for minutes. Client.models() takes list_timeout (default 30) and uses httpx.Timeout(list_timeout, connect=5), and MultiClient threads its own list_timeout through to each listing call. The fan-out stays serial on purpose: KeyResolver._cache is unlocked and only safe while every caller resolves on the GUI thread. Co-Authored-By: Claude Opus 5 --- llamachat/backend.py | 86 ++++++++++++++++++++++++++++++++++-- test_llamachat.py | 122 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 180 insertions(+), 28 deletions(-) diff --git a/llamachat/backend.py b/llamachat/backend.py index fea2de0..ca76559 100644 --- a/llamachat/backend.py +++ b/llamachat/backend.py @@ -25,6 +25,7 @@ import httpx from PIL import Image from . import search +from . import providers as providers_mod THUMB_SIZE = (128, 128) @@ -178,11 +179,20 @@ class Client: """Bearer auth when the provider needs it, nothing when it does not.""" return {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} - def models(self) -> list[str]: - """Model ids the router currently offers.""" + def models(self, list_timeout: int = 30) -> list[str]: + """Model ids the router currently offers. + + `list_timeout` is the overall deadline for the listing call, separate + from the per-turn `self.timeout` used by chat: the picker has to + populate fast, and a dead provider must not hang it for minutes. + The connect deadline is shorter still, so an unreachable host is + reported quickly while a reachable one still gets the full window. + """ try: resp = httpx.get( - f"{self.base_url}/v1/models", timeout=10, headers=self._headers() + f"{self.base_url}/v1/models", + timeout=httpx.Timeout(list_timeout, connect=5), + headers=self._headers(), ) resp.raise_for_status() payload = resp.json() @@ -375,6 +385,76 @@ class Client: raise BackendError(f"Request failed: {exc}") +class MultiClient: + """One facade over every configured provider. + + Holds a `Client` per provider, built on demand so a key is resolved only + when that provider is actually used. The UI talks in prefixed model ids + and never needs to know which endpoint one lives on. + """ + + def __init__( + self, table, timeout=300, list_timeout=30, resolver=None, + client_factory=Client, + ): + self.table = table + self.timeout = timeout + self.list_timeout = list_timeout + self.resolver = resolver or providers_mod.KeyResolver() + self._factory = client_factory + self._clients: dict[str, Client] = {} + + def client_for(self, model_id: str) -> Client: + """The client that serves this model, resolving its key on first use.""" + name, _ = providers_mod.split(model_id, self.table) + if name not in self._clients: + provider = self.table[name] + self._clients[name] = self._factory( + provider.base_url, + timeout=self.timeout, + api_key=self.resolver.resolve(provider), + ) + return self._clients[name] + + def wire_name(self, model_id: str) -> str: + """The model name the provider itself expects, without our prefix.""" + _, model = providers_mod.split(model_id, self.table) + return model + + def models(self) -> tuple[list[str], list[str]]: + """Every offered model id, plus notes about what went wrong. + + A provider that is unreachable or whose filter matched nothing must + not stop the others being listed: local models have to stay usable + when the network is down. + + The fan-out is serial on purpose: `KeyResolver._cache` is unlocked + and only safe while every caller resolves on the GUI thread. Threading + this would spawn two pinentry prompts for one hardware token. + """ + listed: list[str] = [] + problems: list[str] = [] + for name, provider in self.table.items(): + try: + available = self._listing_client(provider).models( + list_timeout=self.list_timeout + ) + except (BackendError, providers_mod.KeyResolutionError) as exc: + problems.append(f"{name}: {exc}") + continue + kept = providers_mod.apply_filter(provider, available) + if available and not kept: + problems.append( + f"{name}: 0 of {len(available)} models matched filter" + ) + listed.extend(providers_mod.qualify(name, m) for m in kept) + return listed, problems + + def _listing_client(self, provider) -> Client: + """Listing needs a client too, and needs the key for a private API.""" + return self.client_for(providers_mod.qualify(provider.name, "")) + + # How much of the first exchange the titling request is shown. Enough to # see what the conversation is about, far short of a whole context. TITLE_EXCERPT = 1200 diff --git a/test_llamachat.py b/test_llamachat.py index 0251c78..1217176 100755 --- a/test_llamachat.py +++ b/test_llamachat.py @@ -14,6 +14,7 @@ # GNU General Public License for more details. """Self-checks for the non-GUI logic. Run: ./test_llamachat.py""" +import contextlib import json import os import subprocess @@ -1849,10 +1850,7 @@ def test_client_auth_header(): return _HttpxResponse({"data": [{"id": "m1"}]}) import httpx - original = httpx.get - try: - httpx.get = _recorder - + with _patched(httpx, "get", _recorder): assert backend.Client("http://x.example.org").models() == ["m1"] assert "Authorization" not in sent["headers"] @@ -1860,8 +1858,6 @@ def test_client_auth_header(): "http://x.example.org", api_key="sk-test-not-a-real-key" ).models() assert sent["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key" - finally: - httpx.get = original # The streaming path is the one that carries every real turn, and no # other test reaches it: the search tests all stub _stream_once out. @@ -1881,10 +1877,7 @@ def test_client_auth_header(): sent["headers"] = headers or {} return _Stream() - original_stream = httpx.stream - try: - httpx.stream = _stream_recorder - + with _patched(httpx, "stream", _stream_recorder): client = backend.Client("http://x.example.org") assert list(client._stream_once("m", [], tools=None)) == [("content", "hi")] assert "Authorization" not in sent["headers"] @@ -1894,14 +1887,91 @@ def test_client_auth_header(): ) list(keyed._stream_once("m", [], tools=None)) assert sent["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key" - finally: - httpx.stream = original_stream # The key is not on the client's repr, which reaches logs and tracebacks. assert "sk-test-not-a-real-key" not in repr(keyed) print("ok client authorization header") +def test_multi_client(): + """Models fan out across providers; requests route by model id.""" + from llamachat import providers + + table = providers.parse( + { + "providers": { + "local": {"base_url": "http://localhost:8181"}, + "together": { + "base_url": "https://api.example.org", + "api_key": "env:MULTI_TEST_KEY", + "filter": ["qwen"], + }, + "down": {"base_url": "https://dead.example.org"}, + } + } + ) + os.environ["MULTI_TEST_KEY"] = "sk-test-not-a-real-key" + + listings = { + "http://localhost:8181": ["gemma4", "qwen3.5-9b"], + "https://api.example.org": [ + "Qwen/Qwen2.5-72B", + "meta-llama/Llama-3.3-70B", + ], + } + built = [] + + class _StubClient: + def __init__(self, base_url, timeout=300, api_key=""): + self.base_url = base_url + self.api_key = api_key + built.append(self) + + def models(self, list_timeout=30): + if self.base_url not in listings: + raise backend.BackendError(f"cannot reach {self.base_url}") + return listings[self.base_url] + + multi = backend.MultiClient( + table, timeout=300, resolver=providers.KeyResolver(), + client_factory=_StubClient, + ) + listed, problems = multi.models() + + # Local models stay bare, cloud ones are prefixed, and the filter cut + # the Llama model out of together's listing. + assert listed == ["gemma4", "qwen3.5-9b", "together:Qwen/Qwen2.5-72B"] + + # The unreachable provider is reported, and did not break the rest. + assert any("down" in p for p in problems) + + # Routing: the client for a cloud model carries that provider's key. + client = multi.client_for("together:Qwen/Qwen2.5-72B") + assert client.base_url == "https://api.example.org" + assert client.api_key == "sk-test-not-a-real-key" + + # And a local model gets the local client with no key at all. + local = multi.client_for("gemma4") + assert local.base_url == "http://localhost:8181" + assert local.api_key == "" + + # The bare model name is what goes on the wire, not the prefixed id. + assert multi.wire_name("together:Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-72B" + assert multi.wire_name("gemma4") == "gemma4" + + # A filter that matches nothing is reported by name with counts. + table["together"].filter = ["zzz"] + empty = backend.MultiClient( + table, timeout=300, resolver=providers.KeyResolver(), + client_factory=_StubClient, + ) + _, notes = empty.models() + assert any("together: 0 of 2" in n for n in notes) + + del os.environ["MULTI_TEST_KEY"] + print("ok multi-provider client") + + class _FakeResponse: """Enough of an http.client response for urlopen's context manager.""" @@ -1940,6 +2010,17 @@ def _with_urlopen(body, capture=None): return original +@contextlib.contextmanager +def _patched(obj, name, value): + """Swap an attribute for the duration of the block, restored even on error.""" + original = getattr(obj, name) + setattr(obj, name, value) + try: + yield + finally: + setattr(obj, name, original) + + def test_search_tool_schema(): from llamachat import search @@ -2545,13 +2626,9 @@ def test_title_request(): import httpx - original = httpx.post - httpx.post = fake_post - try: + with _patched(httpx, "post", fake_post): client = backend.Client("http://router:8181") out = client.complete("m", [{"role": "user", "content": "hi"}], max_tokens=32) - finally: - httpx.post = original assert out == " A Title ", out assert seen["url"] == "http://router:8181/v1/chat/completions" @@ -2567,12 +2644,9 @@ def test_title_request(): # A keyed provider authenticates on this path too, not only on models(). # It runs after the body assertions above because it overwrites `seen`. - httpx.post = fake_post - try: + with _patched(httpx, "post", fake_post): keyed = backend.Client("http://router:8181", api_key="sk-test-not-a-real-key") assert keyed.complete("m", []) == " A Title " - finally: - httpx.post = original assert seen["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key" # A reply with no choices is empty rather than an exception. @@ -2580,11 +2654,8 @@ def test_title_request(): def json(self): return {} - httpx.post = lambda url, json=None, timeout=None, headers=None: Empty() - try: + with _patched(httpx, "post", lambda url, json=None, timeout=None, headers=None: Empty()): assert backend.Client("http://x").complete("m", []) == "" - finally: - httpx.post = original print("ok title request") @@ -2662,6 +2733,7 @@ if __name__ == "__main__": test_metadata_and_cost() test_token_column_migration() test_client_auth_header() + test_multi_client() test_search_tool_schema() test_search_results_sanitising() test_tool_call_accumulation() -- cgit v1.2.3