From 885d65a2bbcde0e8e5094ae4589647875ee5838d Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Sat, 17 Oct 2026 10:50:09 +0200 Subject: assistant: pick the preferred model, loaded on demand The daemon answers with whichever model llama-server has loaded and never asks for one, so it never evicts a model another client is using. The Assistant settings page now lists the router's models with their state. Load saves one as the preferred model, in Prefs, and asks the router to load it now, polling /v1/models until it reports loaded; Prefer marks the loaded model as preferred without swapping. The swap happens only on that click, so the daemon's rule holds. AGENTS.md records the router's load endpoint and its two refusals, both measured: it lives on the server root, not under /v1. Co-Authored-By: Claude Opus 5.5 --- desktop/modules/assistant/AssistantModule.qml | 57 ++++++++++++++++++++++ desktop/modules/assistant/AssistantPage.qml | 68 +++++++++++++++++++++++++++ desktop/modules/assistant/README.md | 7 +++ 3 files changed, 132 insertions(+) (limited to 'desktop/modules/assistant') diff --git a/desktop/modules/assistant/AssistantModule.qml b/desktop/modules/assistant/AssistantModule.qml index e89fe96..a85b2b3 100644 --- a/desktop/modules/assistant/AssistantModule.qml +++ b/desktop/modules/assistant/AssistantModule.qml @@ -52,6 +52,17 @@ Module { property var tools: [] // A write since the last restart, which the daemon has not read yet. property bool pending: false + + // The router's models, [{id, status}], status as it reports it: + // "loaded", "loading", "unloaded" and so on. Empty when it is down. + property var models: [] + // The id being loaded, "" when none is. + property string loading: "" + property string modelError: "" + // The router answers /models and /models/load on the server's root, and + // the OpenAI API under /v1, which is what the daemon's llm points at. + readonly property string llm: am.values.llm ?? am.defaults.llm ?? "http://localhost:8181/v1" + readonly property string routerRoot: am.llm.replace(/\/v1\/?$/, "") property bool restarting: false property string status: "" property string error: "" @@ -59,10 +70,12 @@ Module { searchItems: am.keys.map(k => ({ text: am.labels[k], key: "key:" + k })) .concat(am.envList.map(e => ({ text: e.key + " = " + e.value, key: "env:" + e.key }))) .concat(am.tools.map(t => ({ text: "tool: " + t.name, key: "tool:" + t.name }))) + .concat(am.models.map(m => ({ text: "model: " + m.id, key: "model:" + m.id }))) function refresh() { fv.reload(); am.listTools(); + am.listModels(); if (Object.keys(am.defaults).length === 0) am.run([am.python, am.daemonDir + "/assistant.py", "--defaults"], null, (code, out) => { try { if (code === 0) am.defaults = JSON.parse(out); } catch (e) {} @@ -156,6 +169,50 @@ Module { }); } + function listModels(done) { + am.run(["curl", "-sf", "-m", "3", am.llm + "/models"], null, (code, out) => { + try { + am.models = code === 0 ? JSON.parse(out).data.map(m => ({ id: m.id, status: m.status?.value ?? "" })) : []; + } catch (e) { am.models = []; } + if (done) done(); + }); + } + + // The router keeps one model resident, so this swaps out whatever is + // loaded, for every client: it runs only on a click. The id becomes the + // preferred one, and the list is polled until the router reports it + // loaded, since the request may return before the load finishes. + function loadModel(id) { + Prefs.setAssistantModel(id); + am.modelError = ""; + am.loading = id; + am.run(["curl", "-s", "-m", "600", "-X", "POST", "-H", "Content-Type: application/json", + "-d", JSON.stringify({ model: id }), am.routerRoot + "/models/load"], null, (code, out) => { + let err = code === 0 ? "" : "the router did not answer"; + try { + const e = JSON.parse(out).error; + // Already running is the outcome asked for, not a failure. + if (e && e.message !== "model is already running") err = e.message; + } catch (x) {} + if (err) { am.modelError = `${id}: ${err}`; am.loading = ""; am.listModels(); } + else { poll.left = 300; poll.start(); } + }); + } + property Timer poll: Timer { + property int left: 0 + interval: 1000 + repeat: true + onTriggered: am.listModels(() => { + const m = am.models.find(x => x.id === am.loading); + if (m && m.status === "loaded") { am.loading = ""; am.poll.stop(); } + else if (--am.poll.left <= 0) { + am.modelError = `${am.loading} did not finish loading`; + am.loading = ""; + am.poll.stop(); + } + }) + } + function restart() { am.restarting = true; am.run([am.daemonDir + "/assistant.sh", "restart"], null, (code, out) => { diff --git a/desktop/modules/assistant/AssistantPage.qml b/desktop/modules/assistant/AssistantPage.qml index e5d76c8..2e40ed4 100644 --- a/desktop/modules/assistant/AssistantPage.qml +++ b/desktop/modules/assistant/AssistantPage.qml @@ -162,6 +162,74 @@ Column { Heading { text: "Model" } + Small { + width: page.width + wrapMode: Text.Wrap + text: "The assistant answers with whichever model llama-server has loaded. Load makes one the " + + "preferred model and loads it now, which swaps out the loaded one for every client." + } + + Small { + visible: page.asst.models.length === 0 + text: "llama-server is not answering at " + page.asst.llm + color: Theme.red + } + + Repeater { + model: page.asst.models + + delegate: Rectangle { + id: modelRow + required property var modelData + readonly property bool loaded: modelData.status === "loaded" + readonly property bool busy: page.asst.loading === modelData.id + readonly property bool preferred: Prefs.assistantModel === modelData.id + objectName: "model:" + modelData.id + width: page.width + height: 40 + radius: 8 + color: loaded ? Qt.alpha(Theme.accent, 0.14) : Qt.alpha(Theme.surface, 0.35) + border.width: 1 + border.color: loaded ? Qt.alpha(Theme.accent, 0.5) : "transparent" + + Rectangle { + id: dot + anchors { left: parent.left; leftMargin: 12; verticalCenter: parent.verticalCenter } + width: 8; height: 8; radius: 4 + color: modelRow.loaded ? Theme.green : modelRow.busy ? Theme.yellow : Theme.overlay + } + Text { + id: modelName + anchors { left: dot.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: modelRow.modelData.id + color: Theme.text + font { family: Theme.fontFamily; pixelSize: Theme.fontSize - 2; bold: modelRow.loaded } + } + Small { + anchors { left: modelName.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: [modelRow.busy ? "loading…" : modelRow.modelData.status, + modelRow.preferred ? "preferred" : ""].filter(s => s).join(" · ") + color: modelRow.preferred ? Theme.accent : Theme.subtext + } + Button { + anchors { right: parent.right; rightMargin: 6; verticalCenter: parent.verticalCenter } + text: modelRow.loaded ? (modelRow.preferred ? "Loaded" : "Prefer") : "Load" + enabled: page.asst.loading === "" && !(modelRow.loaded && modelRow.preferred) + // A loaded model only becomes the preferred one: nothing to swap. + onClicked: modelRow.loaded ? Prefs.setAssistantModel(modelRow.modelData.id) + : page.asst.loadModel(modelRow.modelData.id) + } + } + } + + Small { + width: page.width + visible: page.asst.modelError !== "" + wrapMode: Text.Wrap + text: page.asst.modelError + color: Theme.red + } + Labelled { objectName: "key:llm" label: "Server" diff --git a/desktop/modules/assistant/README.md b/desktop/modules/assistant/README.md index 347f9ea..07cc7ad 100644 --- a/desktop/modules/assistant/README.md +++ b/desktop/modules/assistant/README.md @@ -12,6 +12,13 @@ box, Save and "Use the default". The tool environment (`[env]`) is one row per key, with an add row below. A tool's switch renames `tools/.toml` to `.toml.off`, which the daemon's `*.toml` glob skips. +The page also lists llama-server's models with their state. The daemon +always answers with whichever model is loaded and never asks for one, so +it never swaps a model out from under another client. Load is the +deliberate swap: it saves that model as the preferred one (in `Prefs`) +and asks the router to load it now, polling `/v1/models` until it reports +`loaded`. Prefer marks the already-loaded model as preferred. + The daemon reads all of this only at startup, so after a change the page says so and offers Restart, which runs `assistant.sh restart`; a daemon that does not come back shows the script's status, whose log tail says why. -- cgit v1.2.3