From 2d51494f7aa574da3b054a8bbdcf2532ff32a71f Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Mon, 19 Oct 2026 11:36:00 +0200 Subject: assistant: grey out the mic when the model cannot hear AssistantModule keeps each router model's input_modalities and exposes hears: the model loading, else the first not unloaded (the daemon's rule), reports audio. Unknown inputs or a router that is down count as yes; the daemon's error covers the rest. The model list is now read on every canvas open, not only when a preferred model is set, so hears is current when the card shows. Checked: the rule against the router's live /models (Gemma resident: on; Qwen resident or loading: off; router down: on). Not yet seen by eye after a desktop shell reload. Co-Authored-By: Claude Opus 5.5 --- desktop/assistant/Assistant.qml | 5 ++++- desktop/assistant/Card.qml | 3 ++- desktop/assistant/README.md | 4 +++- desktop/modules/assistant/AssistantModule.qml | 25 ++++++++++++++++++------- desktop/shell.qml | 2 +- 5 files changed, 28 insertions(+), 11 deletions(-) (limited to 'desktop') diff --git a/desktop/assistant/Assistant.qml b/desktop/assistant/Assistant.qml index 865f157..c4d0771 100644 --- a/desktop/assistant/Assistant.qml +++ b/desktop/assistant/Assistant.qml @@ -212,9 +212,12 @@ Scope { property var mic: null // the recording's Process, null when not recording property int micSecs: 0 readonly property bool recording: root.mic !== null + // Whether the model in use takes audio. Set from shell.qml. + property bool hears: true function record() { - if (root.recording || !root.connected || root.busy || root.opRunning || root.blocked) return; + if (root.recording || !root.connected || !root.hears || root.busy || root.opRunning + || root.blocked) return; root.notice = ""; root.micSecs = 0; root.mic = micRun.createObject(root); diff --git a/desktop/assistant/Card.qml b/desktop/assistant/Card.qml index d8826fa..85066fe 100644 --- a/desktop/assistant/Card.qml +++ b/desktop/assistant/Card.qml @@ -443,7 +443,8 @@ Rectangle { : "" danger: card.canvas.recording enabled: card.canvas.connected && (card.canvas.recording - || (!card.canvas.busy && !card.canvas.opRunning && !card.canvas.blocked)) + || (card.canvas.hears && !card.canvas.busy && !card.canvas.opRunning + && !card.canvas.blocked)) onClicked: { if (card.canvas.recording) card.canvas.endRecording(true); else card.canvas.record(); diff --git a/desktop/assistant/README.md b/desktop/assistant/README.md index bd61cb4..be95eb7 100644 --- a/desktop/assistant/README.md +++ b/desktop/assistant/README.md @@ -34,7 +34,9 @@ beneath the overlay that holds the keyboard. The mic records with `pw-record` into `$XDG_RUNTIME_DIR/assistant-mic.wav` until clicked again (at most 59 s), then asks with it: the daemon has the -model transcribe it and answers the words. +model transcribe it and answers the words. The mic is greyed out while the +model in use reports no `audio` input (`AssistantModule.hears`, from the +router's `/models`, read on every open). Keys: Enter sends, Shift+Enter is a newline, Escape cancels a recording, then closes the picker, the history panel, then the card. Stop replaces diff --git a/desktop/modules/assistant/AssistantModule.qml b/desktop/modules/assistant/AssistantModule.qml index f738ea6..1e3c1e0 100644 --- a/desktop/modules/assistant/AssistantModule.qml +++ b/desktop/modules/assistant/AssistantModule.qml @@ -72,11 +72,19 @@ Module { // A write since the last restart, which the daemon has not read yet. property bool pending: false - // The router's models, [{id, status}], status as it reports it: - // "loaded", "loading", "unloaded" and so on. Empty when it is down. + // The router's models, [{id, status, inputs}], status as it reports it: + // "loaded", "loading", "unloaded" and so on; inputs its + // input_modalities, null when not reported. Empty when it is down. property var models: [] // The id being loaded, "" when none is. property string loading: "" + // Whether the assistant's model takes audio, for the canvas's mic: the + // one loading, else the daemon's pick, the first not unloaded. Unknown + // counts as yes, and the daemon's own error says otherwise. + readonly property bool hears: { + const m = am.models.find(x => am.loading !== "" ? x.id === am.loading : x.status !== "unloaded"); + return !m?.inputs || m.inputs.includes("audio"); + } property string modelError: "" // The router answers /models and /models/load on the server's root, and // the OpenAI API under /v1, which is what the daemon's llm points at. @@ -233,7 +241,9 @@ Module { function listModels(done) { am.run(["curl", "-sf", "-m", "3", am.llm + "/models"], null, (code, out) => { try { - am.models = code === 0 ? JSON.parse(out).data.map(m => ({ id: m.id, status: m.status?.value ?? "" })) : []; + am.models = code === 0 ? JSON.parse(out).data.map(m => ({ + id: m.id, status: m.status?.value ?? "", + inputs: m.architecture?.input_modalities ?? null })) : []; } catch (e) { am.models = []; } if (done) done(); }); @@ -262,13 +272,14 @@ Module { // Called when the assistant canvas opens: the preferred model, if one is // set, still listed, and neither loaded nor loading, is loaded. One model // and one client at a time is how this desktop runs, so opening the - // assistant is the moment to bring its model back. + // assistant is the moment to bring its model back. The list is read on + // every open, preferred model or not, so hears is current. function ensurePreferred() { - const id = Prefs.assistantModel; - if (id === "" || am.loading !== "") return; + if (am.loading !== "") return; am.listModels(() => { + const id = Prefs.assistantModel; const m = am.models.find(x => x.id === id); - if (m && m.status !== "loaded" && m.status !== "loading") am.loadModel(id); + if (id !== "" && m && m.status !== "loaded" && m.status !== "loading") am.loadModel(id); }); } diff --git a/desktop/shell.qml b/desktop/shell.qml index 3874b14..a660ee2 100644 --- a/desktop/shell.qml +++ b/desktop/shell.qml @@ -118,7 +118,7 @@ ShellRoot { // The assistant canvas on SUPER+Home, a view over desktop-assistant's // socket. See assistant/README.md. - Assistant { id: assistant; swapping: asstSettings.loading } + Assistant { id: assistant; swapping: asstSettings.loading; hears: asstSettings.hears } // Opening the assistant loads its preferred model if another one is in // VRAM. See modules/assistant/README.md. -- cgit v1.2.3