diff options
Diffstat (limited to 'desktop/modules')
| -rw-r--r-- | desktop/modules/assistant/AssistantModule.qml | 57 | ||||
| -rw-r--r-- | desktop/modules/assistant/AssistantPage.qml | 68 | ||||
| -rw-r--r-- | desktop/modules/assistant/README.md | 7 |
3 files changed, 132 insertions, 0 deletions
diff --git a/desktop/modules/assistant/AssistantModule.qml b/desktop/modules/assistant/AssistantModule.qml index e89fe96..a85b2b3 100644 --- a/desktop/modules/assistant/AssistantModule.qml +++ b/desktop/modules/assistant/AssistantModule.qml @@ -52,6 +52,17 @@ Module { property var tools: [] // A write since the last restart, which the daemon has not read yet. property bool pending: false + + // The router's models, [{id, status}], status as it reports it: + // "loaded", "loading", "unloaded" and so on. Empty when it is down. + property var models: [] + // The id being loaded, "" when none is. + property string loading: "" + property string modelError: "" + // The router answers /models and /models/load on the server's root, and + // the OpenAI API under /v1, which is what the daemon's llm points at. + readonly property string llm: am.values.llm ?? am.defaults.llm ?? "http://localhost:8181/v1" + readonly property string routerRoot: am.llm.replace(/\/v1\/?$/, "") property bool restarting: false property string status: "" property string error: "" @@ -59,10 +70,12 @@ Module { searchItems: am.keys.map(k => ({ text: am.labels[k], key: "key:" + k })) .concat(am.envList.map(e => ({ text: e.key + " = " + e.value, key: "env:" + e.key }))) .concat(am.tools.map(t => ({ text: "tool: " + t.name, key: "tool:" + t.name }))) + .concat(am.models.map(m => ({ text: "model: " + m.id, key: "model:" + m.id }))) function refresh() { fv.reload(); am.listTools(); + am.listModels(); if (Object.keys(am.defaults).length === 0) am.run([am.python, am.daemonDir + "/assistant.py", "--defaults"], null, (code, out) => { try { if (code === 0) am.defaults = JSON.parse(out); } catch (e) {} @@ -156,6 +169,50 @@ Module { }); } + function listModels(done) { + am.run(["curl", "-sf", "-m", "3", am.llm + "/models"], null, (code, out) => { + try { + am.models = code === 0 ? JSON.parse(out).data.map(m => ({ id: m.id, status: m.status?.value ?? "" })) : []; + } catch (e) { am.models = []; } + if (done) done(); + }); + } + + // The router keeps one model resident, so this swaps out whatever is + // loaded, for every client: it runs only on a click. The id becomes the + // preferred one, and the list is polled until the router reports it + // loaded, since the request may return before the load finishes. + function loadModel(id) { + Prefs.setAssistantModel(id); + am.modelError = ""; + am.loading = id; + am.run(["curl", "-s", "-m", "600", "-X", "POST", "-H", "Content-Type: application/json", + "-d", JSON.stringify({ model: id }), am.routerRoot + "/models/load"], null, (code, out) => { + let err = code === 0 ? "" : "the router did not answer"; + try { + const e = JSON.parse(out).error; + // Already running is the outcome asked for, not a failure. + if (e && e.message !== "model is already running") err = e.message; + } catch (x) {} + if (err) { am.modelError = `${id}: ${err}`; am.loading = ""; am.listModels(); } + else { poll.left = 300; poll.start(); } + }); + } + property Timer poll: Timer { + property int left: 0 + interval: 1000 + repeat: true + onTriggered: am.listModels(() => { + const m = am.models.find(x => x.id === am.loading); + if (m && m.status === "loaded") { am.loading = ""; am.poll.stop(); } + else if (--am.poll.left <= 0) { + am.modelError = `${am.loading} did not finish loading`; + am.loading = ""; + am.poll.stop(); + } + }) + } + function restart() { am.restarting = true; am.run([am.daemonDir + "/assistant.sh", "restart"], null, (code, out) => { diff --git a/desktop/modules/assistant/AssistantPage.qml b/desktop/modules/assistant/AssistantPage.qml index e5d76c8..2e40ed4 100644 --- a/desktop/modules/assistant/AssistantPage.qml +++ b/desktop/modules/assistant/AssistantPage.qml @@ -162,6 +162,74 @@ Column { Heading { text: "Model" } + Small { + width: page.width + wrapMode: Text.Wrap + text: "The assistant answers with whichever model llama-server has loaded. Load makes one the " + + "preferred model and loads it now, which swaps out the loaded one for every client." + } + + Small { + visible: page.asst.models.length === 0 + text: "llama-server is not answering at " + page.asst.llm + color: Theme.red + } + + Repeater { + model: page.asst.models + + delegate: Rectangle { + id: modelRow + required property var modelData + readonly property bool loaded: modelData.status === "loaded" + readonly property bool busy: page.asst.loading === modelData.id + readonly property bool preferred: Prefs.assistantModel === modelData.id + objectName: "model:" + modelData.id + width: page.width + height: 40 + radius: 8 + color: loaded ? Qt.alpha(Theme.accent, 0.14) : Qt.alpha(Theme.surface, 0.35) + border.width: 1 + border.color: loaded ? Qt.alpha(Theme.accent, 0.5) : "transparent" + + Rectangle { + id: dot + anchors { left: parent.left; leftMargin: 12; verticalCenter: parent.verticalCenter } + width: 8; height: 8; radius: 4 + color: modelRow.loaded ? Theme.green : modelRow.busy ? Theme.yellow : Theme.overlay + } + Text { + id: modelName + anchors { left: dot.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: modelRow.modelData.id + color: Theme.text + font { family: Theme.fontFamily; pixelSize: Theme.fontSize - 2; bold: modelRow.loaded } + } + Small { + anchors { left: modelName.right; leftMargin: 10; verticalCenter: parent.verticalCenter } + text: [modelRow.busy ? "loading…" : modelRow.modelData.status, + modelRow.preferred ? "preferred" : ""].filter(s => s).join(" · ") + color: modelRow.preferred ? Theme.accent : Theme.subtext + } + Button { + anchors { right: parent.right; rightMargin: 6; verticalCenter: parent.verticalCenter } + text: modelRow.loaded ? (modelRow.preferred ? "Loaded" : "Prefer") : "Load" + enabled: page.asst.loading === "" && !(modelRow.loaded && modelRow.preferred) + // A loaded model only becomes the preferred one: nothing to swap. + onClicked: modelRow.loaded ? Prefs.setAssistantModel(modelRow.modelData.id) + : page.asst.loadModel(modelRow.modelData.id) + } + } + } + + Small { + width: page.width + visible: page.asst.modelError !== "" + wrapMode: Text.Wrap + text: page.asst.modelError + color: Theme.red + } + Labelled { objectName: "key:llm" label: "Server" diff --git a/desktop/modules/assistant/README.md b/desktop/modules/assistant/README.md index 347f9ea..07cc7ad 100644 --- a/desktop/modules/assistant/README.md +++ b/desktop/modules/assistant/README.md @@ -12,6 +12,13 @@ box, Save and "Use the default". The tool environment (`[env]`) is one row per key, with an add row below. A tool's switch renames `tools/<name>.toml` to `<name>.toml.off`, which the daemon's `*.toml` glob skips. +The page also lists llama-server's models with their state. The daemon +always answers with whichever model is loaded and never asks for one, so +it never swaps a model out from under another client. Load is the +deliberate swap: it saves that model as the preferred one (in `Prefs`) +and asks the router to load it now, polling `/v1/models` until it reports +`loaded`. Prefer marks the already-loaded model as preferred. + The daemon reads all of this only at startup, so after a change the page says so and offers Restart, which runs `assistant.sh restart`; a daemon that does not come back shows the script's status, whose log tail says why. |