summaryrefslogtreecommitdiffstats
path: root/desktop/modules
diff options
context:
space:
mode:
Diffstat (limited to 'desktop/modules')
-rw-r--r--desktop/modules/ai/AiModule.qml7
-rw-r--r--desktop/modules/ai/AiPage.qml5
-rw-r--r--desktop/modules/ai/README.md6
3 files changed, 15 insertions, 3 deletions
diff --git a/desktop/modules/ai/AiModule.qml b/desktop/modules/ai/AiModule.qml
index 6df3d6b..d58ebfe 100644
--- a/desktop/modules/ai/AiModule.qml
+++ b/desktop/modules/ai/AiModule.qml
@@ -85,6 +85,13 @@ Module {
exec(["sh", "-c", 'pkill -TERM -P "$1"; kill -TERM "$1"', "sh", app.detail]);
}
+ // The router frees the model's VRAM; the next request for it loads it
+ // again. Its answer is not read, the next poll shows whether it worked.
+ function unload(model) {
+ exec(["curl", "-s", "-m", "30", "-X", "POST", "-H", "Content-Type: application/json",
+ "-d", JSON.stringify({ model: model }), "http://127.0.0.1:8181/models/unload"]);
+ }
+
// Detached, so a start outlives a quickshell reload. No exit code comes
// back; the next poll shows whether it worked.
function exec(cmd) {
diff --git a/desktop/modules/ai/AiPage.qml b/desktop/modules/ai/AiPage.qml
index e89132a..369c1b5 100644
--- a/desktop/modules/ai/AiPage.qml
+++ b/desktop/modules/ai/AiPage.qml
@@ -12,7 +12,8 @@
import QtQuick
import "../.."
-// Engines are read-only: stopping either would take its web UI down with it.
+// Engines are never stopped, that would take their web UI down with them;
+// llama's model can be unloaded, which frees the VRAM and leaves the server up.
Column {
id: page
@@ -42,6 +43,8 @@ Column {
return modelData.name === "llama" ? "No model loaded" : "Running";
return (modelData.busy ? "Generating" : "Idle") + " · " + modelData.model;
}
+ action: modelData.name === "llama" && modelData.model !== "" ? "Unload" : ""
+ onTriggered: page.ai.unload(modelData.model)
}
}
diff --git a/desktop/modules/ai/README.md b/desktop/modules/ai/README.md
index 5aec615..e71159b 100644
--- a/desktop/modules/ai/README.md
+++ b/desktop/modules/ai/README.md
@@ -8,8 +8,10 @@ The tile counts running apps and names any engine that is generating, or says
## Engines
llama.cpp (`llama-server`, router mode, port 8181) and stable-diffusion.cpp
-(`sd-server`, started by `~/bin/sd-webui`). Both rows are read-only: each
-engine also serves a web UI, and stopping it would take that down.
+(`sd-server`, started by `~/bin/sd-webui`). Neither is ever stopped: each
+engine also serves a web UI, and stopping it would take that down. llama's
+row offers Unload while a model is loaded, `POST /models/unload` to the
+router, which frees the VRAM and leaves the server up.
llama's state comes from its API: `/models` for the loaded model, then
`/slots?model=<id>&autoload=false` for whether a slot is processing. The