diff options
| -rw-r--r-- | AGENTS.md | 28 | ||||
| -rw-r--r-- | assistant.py | 14 |
2 files changed, 38 insertions, 4 deletions
@@ -274,6 +274,34 @@ must not decide alone (unknown host, already cloned) are asked. Spec: - A tool added to `~/.config/desktop-assistant/tools/` needs a daemon restart: `TOOLS` is loaded once at import, not per request. +### Cloud models, live check (2026-10-23) + +Hetzner's inference API (`https://inference.hetzner.com/api/v1`, +`key_cmd = "pass hetzner.de/llamachat"`), model `Qwen3.8-27B`, through a +second daemon on a scratch socket, config and data dir; the real config +and history were untouched. + +- `--cloud-models hetzner`: 4 ids in 1.2 s, including the pass prompt. +- `models` listed the router's two and both configured cloud ones. +- "Che tempo fa domani a Roma?": `weather` with `Rome,IT`, first token + 4.6 s, ctx 775 then 953, `max` 0. +- "Quante email non lette ho?" in the same chat: no tool offered, it said + it has no way to read mail. Nothing personal went out. +- Voice clip (`05.wav`) in the cloud chat: transcribed by the local + Gemma (17 s: it was sleeping and reloaded), answered by the cloud with + `weather` for home. +- Same chat switched to Gemma: `ctx` max back to 32768, all tools + offered, mail answered (173 messages in 141 threads). Back to the + cloud: `weather` for Milan, 3.4 s; the mail result had gone back as + the history placeholder only. `open` reported the stored cloud model. +- **Hetzner sends `usage` on every chunk**, not once at the end: the + daemon sent ~60 `ctx` events per reply. Now one per request. +- Titles from the cloud took ~12 s: `enable_thinking` is not sent to + providers, so Qwen reasons over the title too. Harmless, after done. +- Qwen's replies start with a blank line or two after its reasoning; + Markdown drops them. +- The key appeared in no log line. + ### Text daemon, live check (2026-10-16) Model Gemma4-12B-qat-mtp, ctx 32768, preset reasoning-budget 4096. diff --git a/assistant.py b/assistant.py index fbca0ef..fac6b9c 100644 --- a/assistant.py +++ b/assistant.py @@ -908,7 +908,7 @@ def stream(conn, path, body, calls, meter=True, key=None, who="llama-server"): why = why.replace(key, "***") # some providers echo the key back raise RuntimeError(f"{who} {r.status}: {why[:2000]}") with r: - streamed, other = False, b"" + streamed, other, metered = False, b"", False for line in r: if not line.startswith(b"data: "): # The router answers a crashed model with a plain line, e.g. @@ -922,7 +922,10 @@ def stream(conn, path, body, calls, meter=True, key=None, who="llama-server"): if "error" in chunk: # a failure mid-stream arrives as a chunk err = chunk["error"] raise RuntimeError(err.get("message", str(err)) if isinstance(err, dict) else str(err)) - if chunk.get("usage") and meter: + # Once per request: llama-server sends usage at the end, some + # providers (Hetzner) with every chunk. + if chunk.get("usage") and meter and not metered: + metered = True # How full the model's context is: history, tool results and # tool definitions all count. ctx["used"] = chunk["usage"]["prompt_tokens"] @@ -1886,9 +1889,12 @@ def test(): assert DB.execute("SELECT model FROM conversations WHERE id = ?", (conv,)).fetchone()[0] == "fake:vendor/x:free" # The next ask with no model stays on the chat's. - script["replies"] = [[{"content": "Still cloud."}]] + # Usage on every chunk, as Hetzner sends it: one ctx event. + script["replies"] = [['data: {"choices": [{"delta": {"content": "Still "}}], ' + '"usage": {"prompt_tokens": 7}}\n\n'] * 2] send(type="ask", text="again") - until("done") + meters = [e for e in until("done") if e["type"] == "ctx"] + assert [e["used"] for e in meters] == [7], meters assert script["paths"][-1] == "/cloud/v1/chat/completions" # Reopened, the chat reports its model. send(type="new") |