feat: fan model listing out across providers and route by id

A provider that is unreachable or whose filter matched nothing is
reported rather than fatal: local models must stay usable when the
network is down. Clients are built on demand so a key is resolved only
when that provider is really used.

The model listing carries its own timeout, separate from the per-turn
one: the picker has to populate fast, and a dead cloud provider must
not hang it for minutes. Client.models() takes list_timeout (default 30)
and uses httpx.Timeout(list_timeout, connect=5), and MultiClient threads
its own list_timeout through to each listing call. The fan-out stays
serial on purpose: KeyResolver._cache is unlocked and only safe while
every caller resolves on the GUI thread.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

2 files changed, 180 insertions, 28 deletions

diff --git a/llamachat/backend.py b/llamachat/backend.py
index fea2de0..ca76559 100644
--- a/llamachat/backend.py
+++ b/llamachat/backend.py
@@ -25,6 +25,7 @@ import httpx
 from PIL import Image
 
 from . import search
+from . import providers as providers_mod
 
 THUMB_SIZE = (128, 128)
 
@@ -178,11 +179,20 @@ class Client:
         """Bearer auth when the provider needs it, nothing when it does not."""
         return {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
 
-    def models(self) -> list[str]:
-        """Model ids the router currently offers."""
+    def models(self, list_timeout: int = 30) -> list[str]:
+        """Model ids the router currently offers.
+
+        `list_timeout` is the overall deadline for the listing call, separate
+        from the per-turn `self.timeout` used by chat: the picker has to
+        populate fast, and a dead provider must not hang it for minutes.
+        The connect deadline is shorter still, so an unreachable host is
+        reported quickly while a reachable one still gets the full window.
+        """
         try:
             resp = httpx.get(
-                f"{self.base_url}/v1/models", timeout=10, headers=self._headers()
+                f"{self.base_url}/v1/models",
+                timeout=httpx.Timeout(list_timeout, connect=5),
+                headers=self._headers(),
             )
             resp.raise_for_status()
             payload = resp.json()
@@ -375,6 +385,76 @@ class Client:
             raise BackendError(f"Request failed: {exc}")
 
 
+class MultiClient:
+    """One facade over every configured provider.
+
+    Holds a `Client` per provider, built on demand so a key is resolved only
+    when that provider is actually used. The UI talks in prefixed model ids
+    and never needs to know which endpoint one lives on.
+    """
+
+    def __init__(
+        self, table, timeout=300, list_timeout=30, resolver=None,
+        client_factory=Client,
+    ):
+        self.table = table
+        self.timeout = timeout
+        self.list_timeout = list_timeout
+        self.resolver = resolver or providers_mod.KeyResolver()
+        self._factory = client_factory
+        self._clients: dict[str, Client] = {}
+
+    def client_for(self, model_id: str) -> Client:
+        """The client that serves this model, resolving its key on first use."""
+        name, _ = providers_mod.split(model_id, self.table)
+        if name not in self._clients:
+            provider = self.table[name]
+            self._clients[name] = self._factory(
+                provider.base_url,
+                timeout=self.timeout,
+                api_key=self.resolver.resolve(provider),
+            )
+        return self._clients[name]
+
+    def wire_name(self, model_id: str) -> str:
+        """The model name the provider itself expects, without our prefix."""
+        _, model = providers_mod.split(model_id, self.table)
+        return model
+
+    def models(self) -> tuple[list[str], list[str]]:
+        """Every offered model id, plus notes about what went wrong.
+
+        A provider that is unreachable or whose filter matched nothing must
+        not stop the others being listed: local models have to stay usable
+        when the network is down.
+
+        The fan-out is serial on purpose: `KeyResolver._cache` is unlocked
+        and only safe while every caller resolves on the GUI thread. Threading
+        this would spawn two pinentry prompts for one hardware token.
+        """
+        listed: list[str] = []
+        problems: list[str] = []
+        for name, provider in self.table.items():
+            try:
+                available = self._listing_client(provider).models(
+                    list_timeout=self.list_timeout
+                )
+            except (BackendError, providers_mod.KeyResolutionError) as exc:
+                problems.append(f"{name}: {exc}")
+                continue
+            kept = providers_mod.apply_filter(provider, available)
+            if available and not kept:
+                problems.append(
+                    f"{name}: 0 of {len(available)} models matched filter"
+                )
+            listed.extend(providers_mod.qualify(name, m) for m in kept)
+        return listed, problems
+
+    def _listing_client(self, provider) -> Client:
+        """Listing needs a client too, and needs the key for a private API."""
+        return self.client_for(providers_mod.qualify(provider.name, ""))
+
+
 # How much of the first exchange the titling request is shown. Enough to
 # see what the conversation is about, far short of a whole context.
 TITLE_EXCERPT = 1200
diff --git a/test_llamachat.py b/test_llamachat.py
index 0251c78..1217176 100755
--- a/test_llamachat.py
+++ b/test_llamachat.py
@@ -14,6 +14,7 @@
 # GNU General Public License for more details.
 """Self-checks for the non-GUI logic. Run: ./test_llamachat.py"""
 
+import contextlib
 import json
 import os
 import subprocess
@@ -1849,10 +1850,7 @@ def test_client_auth_header():
         return _HttpxResponse({"data": [{"id": "m1"}]})
 
     import httpx
-    original = httpx.get
-    try:
-        httpx.get = _recorder
-
+    with _patched(httpx, "get", _recorder):
         assert backend.Client("http://x.example.org").models() == ["m1"]
         assert "Authorization" not in sent["headers"]
 
@@ -1860,8 +1858,6 @@ def test_client_auth_header():
             "http://x.example.org", api_key="sk-test-not-a-real-key"
         ).models()
         assert sent["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key"
-    finally:
-        httpx.get = original
 
     # The streaming path is the one that carries every real turn, and no
     # other test reaches it: the search tests all stub _stream_once out.
@@ -1881,10 +1877,7 @@ def test_client_auth_header():
         sent["headers"] = headers or {}
         return _Stream()
 
-    original_stream = httpx.stream
-    try:
-        httpx.stream = _stream_recorder
-
+    with _patched(httpx, "stream", _stream_recorder):
         client = backend.Client("http://x.example.org")
         assert list(client._stream_once("m", [], tools=None)) == [("content", "hi")]
         assert "Authorization" not in sent["headers"]
@@ -1894,14 +1887,91 @@ def test_client_auth_header():
         )
         list(keyed._stream_once("m", [], tools=None))
         assert sent["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key"
-    finally:
-        httpx.stream = original_stream
 
     # The key is not on the client's repr, which reaches logs and tracebacks.
     assert "sk-test-not-a-real-key" not in repr(keyed)
     print("ok  client authorization header")
 
 
+def test_multi_client():
+    """Models fan out across providers; requests route by model id."""
+    from llamachat import providers
+
+    table = providers.parse(
+        {
+            "providers": {
+                "local": {"base_url": "http://localhost:8181"},
+                "together": {
+                    "base_url": "https://api.example.org",
+                    "api_key": "env:MULTI_TEST_KEY",
+                    "filter": ["qwen"],
+                },
+                "down": {"base_url": "https://dead.example.org"},
+            }
+        }
+    )
+    os.environ["MULTI_TEST_KEY"] = "sk-test-not-a-real-key"
+
+    listings = {
+        "http://localhost:8181": ["gemma4", "qwen3.5-9b"],
+        "https://api.example.org": [
+            "Qwen/Qwen2.5-72B",
+            "meta-llama/Llama-3.3-70B",
+        ],
+    }
+    built = []
+
+    class _StubClient:
+        def __init__(self, base_url, timeout=300, api_key=""):
+            self.base_url = base_url
+            self.api_key = api_key
+            built.append(self)
+
+        def models(self, list_timeout=30):
+            if self.base_url not in listings:
+                raise backend.BackendError(f"cannot reach {self.base_url}")
+            return listings[self.base_url]
+
+    multi = backend.MultiClient(
+        table, timeout=300, resolver=providers.KeyResolver(),
+        client_factory=_StubClient,
+    )
+    listed, problems = multi.models()
+
+    # Local models stay bare, cloud ones are prefixed, and the filter cut
+    # the Llama model out of together's listing.
+    assert listed == ["gemma4", "qwen3.5-9b", "together:Qwen/Qwen2.5-72B"]
+
+    # The unreachable provider is reported, and did not break the rest.
+    assert any("down" in p for p in problems)
+
+    # Routing: the client for a cloud model carries that provider's key.
+    client = multi.client_for("together:Qwen/Qwen2.5-72B")
+    assert client.base_url == "https://api.example.org"
+    assert client.api_key == "sk-test-not-a-real-key"
+
+    # And a local model gets the local client with no key at all.
+    local = multi.client_for("gemma4")
+    assert local.base_url == "http://localhost:8181"
+    assert local.api_key == ""
+
+    # The bare model name is what goes on the wire, not the prefixed id.
+    assert multi.wire_name("together:Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-72B"
+    assert multi.wire_name("gemma4") == "gemma4"
+
+    # A filter that matches nothing is reported by name with counts.
+    table["together"].filter = ["zzz"]
+    empty = backend.MultiClient(
+        table, timeout=300, resolver=providers.KeyResolver(),
+        client_factory=_StubClient,
+    )
+    _, notes = empty.models()
+    assert any("together: 0 of 2" in n for n in notes)
+
+    del os.environ["MULTI_TEST_KEY"]
+    print("ok  multi-provider client")
+
+
 class _FakeResponse:
     """Enough of an http.client response for urlopen's context manager."""
 
@@ -1940,6 +2010,17 @@ def _with_urlopen(body, capture=None):
     return original
 
 
+@contextlib.contextmanager
+def _patched(obj, name, value):
+    """Swap an attribute for the duration of the block, restored even on error."""
+    original = getattr(obj, name)
+    setattr(obj, name, value)
+    try:
+        yield
+    finally:
+        setattr(obj, name, original)
+
+
 def test_search_tool_schema():
     from llamachat import search
 
@@ -2545,13 +2626,9 @@ def test_title_request():
 
     import httpx
 
-    original = httpx.post
-    httpx.post = fake_post
-    try:
+    with _patched(httpx, "post", fake_post):
         client = backend.Client("http://router:8181")
         out = client.complete("m", [{"role": "user", "content": "hi"}], max_tokens=32)
-    finally:
-        httpx.post = original
 
     assert out == " A Title ", out
     assert seen["url"] == "http://router:8181/v1/chat/completions"
@@ -2567,12 +2644,9 @@ def test_title_request():
 
     # A keyed provider authenticates on this path too, not only on models().
     # It runs after the body assertions above because it overwrites `seen`.
-    httpx.post = fake_post
-    try:
+    with _patched(httpx, "post", fake_post):
         keyed = backend.Client("http://router:8181", api_key="sk-test-not-a-real-key")
         assert keyed.complete("m", []) == " A Title "
-    finally:
-        httpx.post = original
     assert seen["headers"]["Authorization"] == "Bearer sk-test-not-a-real-key"
 
     # A reply with no choices is empty rather than an exception.
@@ -2580,11 +2654,8 @@ def test_title_request():
         def json(self):
             return {}
 
-    httpx.post = lambda url, json=None, timeout=None, headers=None: Empty()
-    try:
+    with _patched(httpx, "post", lambda url, json=None, timeout=None, headers=None: Empty()):
         assert backend.Client("http://x").complete("m", []) == ""
-    finally:
-        httpx.post = original
     print("ok  title request")
 
 
@@ -2662,6 +2733,7 @@ if __name__ == "__main__":
     test_metadata_and_cost()
     test_token_column_migration()
     test_client_auth_header()
+    test_multi_client()
     test_search_tool_schema()
     test_search_results_sanitising()
     test_tool_call_accumulation()