From 885d65a2bbcde0e8e5094ae4589647875ee5838d Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Sat, 17 Oct 2026 10:50:09 +0200 Subject: assistant: pick the preferred model, loaded on demand The daemon answers with whichever model llama-server has loaded and never asks for one, so it never evicts a model another client is using. The Assistant settings page now lists the router's models with their state. Load saves one as the preferred model, in Prefs, and asks the router to load it now, polling /v1/models until it reports loaded; Prefer marks the loaded model as preferred without swapping. The swap happens only on that click, so the daemon's rule holds. AGENTS.md records the router's load endpoint and its two refusals, both measured: it lives on the server root, not under /v1. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +++ desktop/Prefs.qml | 14 +++++- desktop/modules/assistant/AssistantModule.qml | 57 ++++++++++++++++++++++ desktop/modules/assistant/AssistantPage.qml | 68 +++++++++++++++++++++++++++ desktop/modules/assistant/README.md | 7 +++ 5 files changed, 151 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 047adf7..79b8916 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -354,6 +354,12 @@ changing that component. The ones that generalise: GPU. Pass `autoload=false`, which answers 400 `model is not loaded` instead. Checking `/models` for `loaded` first is not enough on its own, because the model can unload between the two requests. +- **llama-server's router loads a model with `POST /models/load`** and a + `{"model": ""}` body, on the server root, not under `/v1`. Asking for + the loaded model answers 400 `model is already running`, and an unknown + id 404; both measured. With `--models-max 1` a load swaps out the + resident model for every client, so the assistant page does it only on a + click. - **`busctl get-property` escapes non-ASCII as octal.** A name with an accented letter reads `s "Do\303\250r"`-style, which no shell builtin decodes. `busctl --json=short` returns real UTF-8; `userctl` pipes it diff --git a/desktop/Prefs.qml b/desktop/Prefs.qml index 9f89a7f..ba567c4 100644 --- a/desktop/Prefs.qml +++ b/desktop/Prefs.qml @@ -35,6 +35,16 @@ Singleton { // (assistant/Presets.js, which also validates what is read here). property var presets: null + // The router model preferred for the assistant, "" for none. The daemon + // still uses whatever is loaded; the settings page loads this on demand. + property string assistantModel: "" + function setAssistantModel(id) { + if (id === root.assistantModel) return; + root.assistantModel = id; + const a = Object.assign({}, root.doc.assistant, { model: id }); + root.write(Object.assign({}, root.doc, { assistant: a })); + } + // The settings window's sidebar width, dragged by its edge. readonly property int sidebarMin: 160 readonly property int sidebarMax: 420 @@ -88,6 +98,8 @@ Singleton { if (pin !== root.pinnedIp) root.pinnedIp = pin; root.presets = parsed.assistant && parsed.assistant.presets !== undefined ? parsed.assistant.presets : null; + root.assistantModel = parsed.assistant && typeof parsed.assistant.model === "string" + ? parsed.assistant.model : ""; const w = parsed.settings ? parsed.settings.sidebar : undefined; root.settingsSidebar = typeof w === "number" && w >= root.sidebarMin && w <= root.sidebarMax ? Math.round(w) : 200; @@ -105,6 +117,6 @@ Singleton { onFileChanged: reload() onLoaded: root.reparse() // Missing file: the defaults, and nothing written. - onLoadFailed: { root.doc = {}; root.pinnedIp = ""; root.presets = null; root.settingsSidebar = 200; } + onLoadFailed: { root.doc = {}; root.pinnedIp = ""; root.presets = null; root.settingsSidebar = 200; root.assistantModel = ""; } } } diff --git a/desktop/modules/assistant/AssistantModule.qml b/desktop/modules/assistant/AssistantModule.qml index e89fe96..a85b2b3 100644 --- a/desktop/modules/assistant/AssistantModule.qml +++ b/desktop/modules/assistant/AssistantModule.qml @@ -52,6 +52,17 @@ Module { property var tools: [] // A write since the last restart, which the daemon has not read yet. property bool pending: false + + // The router's models, [{id, status}], status as it reports it: + // "loaded", "loading", "unloaded" and so on. Empty when it is down. + property var models: [] + // The id being loaded, "" when none is. + property string loading: "" + property string modelError: "" + // The router answers /models and /models/load on the server's root, and + // the OpenAI API under /v1, which is what the daemon's llm points at. + readonly property string llm: am.values.llm ?? am.defaults.llm ?? "http://localhost:8181/v1" + readonly property string routerRoot: am.llm.replace(/\/v1\/?$/, "") property bool restarting: false property string status: "" property string error: "" @@ -59,10 +70,12 @@ Module { searchItems: am.keys.map(k => ({ text: am.labels[k], key: "key:" + k })) .concat(am.envList.map(e => ({ text: e.key + " = " + e.value, key: "env:" + e.key }))) .concat(am.tools.map(t => ({ text: "tool: " + t.name, key: "tool:" + t.name }))) + .concat(am.models.map(m => ({ text: "model: " + m.id, key: "model:" + m.id }))) function refresh() { fv.reload(); am.listTools(); + am.listModels(); if (Object.keys(am.defaults).length === 0) am.run([am.python, am.daemonDir + "/assistant.py", "--defaults"], null, (code, out) => { try { if (code === 0) am.defaults = JSON.parse(out); } catch (e) {} @@ -156,6 +169,50 @@ Module { }); } + function listModels(done) { + am.run(["curl", "-sf", "-m", "3", am.llm + "/models"], null, (code, out) => { + try { + am.models = code === 0 ? JSON.parse(out).data.map(m => ({ id: m.id, status: m.status?.value ?? "" })) : []; + } catch (e) { am.models = []; } + if (done) done(); + }); + } + + // The router keeps one model resident, so this swaps out whatever is + // loaded, for every client: it runs only on a click. The id becomes the + // preferred one, and the list is polled until the router reports it + // loaded, since the request may return before the load finishes. + function loadModel(id) { + Prefs.setAssistantModel(id); + am.modelError = ""; + am.loading = id; + am.run(["curl", "-s", "-m", "600", "-X", "POST", "-H", "Content-Type: application/json", + "-d", JSON.stringify({ model: id }), am.routerRoot + "/models/load"], null, (code, out) => { + let err = code === 0 ? "" : "the router did not answer"; + try { + const e = JSON.parse(out).error; + // Already running is the outcome asked for, not a failure. + if (e && e.message !== "model is already running") err = e.message; + } catch (x) {} + if (err) { am.modelError = `${id}: ${err}`; am.loading = ""; am.listModels(); } + else { poll.left = 300; poll.start(); } + }); + } + property Timer poll: Timer { + property int left: 0 + interval: 1000 + repeat: true + onTriggered: am.listModels(() => { + const m = am.models.find(x => x.id === am.loading); + if (m && m.status === "loaded") { am.loading = ""; am.poll.stop(); } + else if (--am.poll.left <= 0) { + am.modelError = `${am.loading} did not finish loading`; + am.loading = ""; + am.poll.stop(); + } + }) + } + function restart() { am.restarting = true; am.run([am.daemonDir + "/assistant.sh", "restart"], null, (code, out) => { diff --git a/desktop/modules/assistant/AssistantPage.qml b/desktop/modules/assistant/AssistantPage.qml index e5d76c8..2e40ed4 100644 --- a/desktop/modules/assistant/AssistantPage.qml +++ b/desktop/modules/assistant/AssistantPage.qml @@ -162,6 +162,74 @@ Column { Heading { text: "Model" } + Small { + width: page.width + wrapMode: Text.Wrap + text: "The assistant answers with whichever model llama-server has loaded. Load makes one the " + + "preferred model and loads it now, which swaps out the loaded one for every client." + } + + Small { + visible: page.asst.models.length === 0 + text: "llama-server is not answering at " + page.asst.llm + color: Theme.red + } + + Repeater { + model: page.asst.models + + delegate: Rectangle { + id: modelRow + required property var modelData + readonly property bool loaded: modelData.status === "loaded" + readonly property bool busy: page.asst.loading === modelData.id + readonly property bool preferred: Prefs.assistantModel === modelData.id + objectName: "model:" + modelData.id + width: page.width + height: 40 + radius: 8 + color: loaded ? Qt.alpha(Theme.accent, 0.14) : Qt.alpha(Theme.surface, 0.35) + border.width: 1 + border.color: loaded ? Qt.alpha(Theme.accent, 0.5) : "transparent" + + Rectangle { + id: dot + anchors { left: parent.left; leftMargin: 12; verticalCenter: parent.verticalCenter } + width: 8; height: 8; radius: 4 + color: modelRow.loaded ? Theme.green : modelRow.busy ? Theme.yellow : Theme.overlay + } + Text { + id: modelName + anchors { left: dot.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: modelRow.modelData.id + color: Theme.text + font { family: Theme.fontFamily; pixelSize: Theme.fontSize - 2; bold: modelRow.loaded } + } + Small { + anchors { left: modelName.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: [modelRow.busy ? "loading…" : modelRow.modelData.status, + modelRow.preferred ? "preferred" : ""].filter(s => s).join(" · ") + color: modelRow.preferred ? Theme.accent : Theme.subtext + } + Button { + anchors { right: parent.right; rightMargin: 6; verticalCenter: parent.verticalCenter } + text: modelRow.loaded ? (modelRow.preferred ? "Loaded" : "Prefer") : "Load" + enabled: page.asst.loading === "" && !(modelRow.loaded && modelRow.preferred) + // A loaded model only becomes the preferred one: nothing to swap. + onClicked: modelRow.loaded ? Prefs.setAssistantModel(modelRow.modelData.id) + : page.asst.loadModel(modelRow.modelData.id) + } + } + } + + Small { + width: page.width + visible: page.asst.modelError !== "" + wrapMode: Text.Wrap + text: page.asst.modelError + color: Theme.red + } + Labelled { objectName: "key:llm" label: "Server" diff --git a/desktop/modules/assistant/README.md b/desktop/modules/assistant/README.md index 347f9ea..07cc7ad 100644 --- a/desktop/modules/assistant/README.md +++ b/desktop/modules/assistant/README.md @@ -12,6 +12,13 @@ box, Save and "Use the default". The tool environment (`[env]`) is one row per key, with an add row below. A tool's switch renames `tools/.toml` to `.toml.off`, which the daemon's `*.toml` glob skips. +The page also lists llama-server's models with their state. The daemon +always answers with whichever model is loaded and never asks for one, so +it never swaps a model out from under another client. Load is the +deliberate swap: it saves that model as the preferred one (in `Prefs`) +and asks the router to load it now, polling `/v1/models` until it reports +`loaded`. Prefer marks the already-loaded model as preferred. + The daemon reads all of this only at startup, so after a change the page says so and offers Restart, which runs `assistant.sh restart`; a daemon that does not come back shows the script's status, whose log tail says why. -- cgit v1.2.3