From 161f4ee8fd956b192ec8e028052c2ec98b098852 Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Sat, 26 Sep 2026 09:53:27 +0200 Subject: feat(assistant): overlay for the voice assistant Reads desktop-assistant's event socket, JSON lines on a Unix socket the assistant owns: state, context use, the running tool and audio levels. A circle shows the name and used/total context, a wrench and the tool name while one runs, and pulsing dots while it transcribes or thinks. Around it three sine ripples follow the loudness, the user's voice and the reply in two palette colours from Theme, so a udt change recolours them live. It fades out a configurable delay after the reply. The ripple is driven by FrameAnimation, not a Timer: a 16 ms Timer drifted against the display refresh and visibly stuttered. Standalone like the other components; it reconnects every 2 s while the assistant is down, and the assistant runs the same without it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + assistant/Assistant.qml | 255 ++++++++++++++++++++++++++++++++++++++++++++++++ assistant/Place.js | 23 +++++ assistant/README.md | 17 ++++ assistant/Theme.qml | 1 + assistant/shell.qml | 16 +++ assistant/test_place.js | 32 ++++++ 7 files changed, 345 insertions(+) create mode 100644 assistant/Assistant.qml create mode 100644 assistant/Place.js create mode 100644 assistant/README.md create mode 120000 assistant/Theme.qml create mode 100644 assistant/shell.qml create mode 100644 assistant/test_place.js diff --git a/AGENTS.md b/AGENTS.md index a5b0395..b4e6136 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ them runs alone, and running one does not require the others. One of them, appearance/ wallpaper picker and colour scheme switcher window-switcher/ open windows as live previews in a grid, on ALT+TAB keybinds/ live keybind reminder, on SUPER+k + assistant/ overlay for the voice assistant, fed by its event socket They are started from `~/.config/hypr/sections/autostart.lua` and keep running for the whole session. diff --git a/assistant/Assistant.qml b/assistant/Assistant.qml new file mode 100644 index 0000000..4117882 --- /dev/null +++ b/assistant/Assistant.qml @@ -0,0 +1,255 @@ +// Copyright (C) 2026 Danilo M. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +import Quickshell +import Quickshell.Io +import Quickshell.Wayland +import Quickshell.Hyprland +import QtQuick +import "Place.js" as Place + +// The voice assistant's overlay. desktop-assistant broadcasts its state, +// context use and audio levels as JSON lines on a Unix socket; this only +// listens, and keeps reconnecting while the assistant is down. +Scope { + id: root + + // Replaced by the assistant's hello; these render only before it. + property string name: "Assistant" + property string position: "center" + property real hideDelay: 3 + property string userColor: "accent" + property string assistantColor: "green" + + property int ctxUsed: 0 + property int ctxMax: 0 + property string state: "idle" + property string tool: "" + property real level: 0 // latest loudness, 0..1 + property string who: "user" // whose voice that was + property real heardAt: 0 // when it arrived, ms + property bool shown: false + property var shownScreen: Quickshell.screens[0] + + // Faded, not switched: the window stays loaded until the fade has run. + property real fade: shown ? 1 : 0 + Behavior on fade { NumberAnimation { duration: 200 } } + + // Colour roles are palette names from udt, so a theme change recolours + // the waves live. An unknown name falls back to the accent. + function colorOf(role) { return Theme[role] ?? Theme.accent; } + + function handle(line) { + let e; + try { e = JSON.parse(line); } catch (err) { return; } + if (e.type === "hello") { + root.name = e.name; + root.position = e.position; + root.hideDelay = e.hide_delay; + root.userColor = e.user_color; + root.assistantColor = e.assistant_color; + root.ctxMax = e.ctx_max; + root.setState(e.state, e.tool ?? ""); + } else if (e.type === "state") { + root.setState(e.state, e.tool ?? ""); + } else if (e.type === "ctx") { + root.ctxUsed = e.used; + root.ctxMax = e.max; + } else if (e.type === "level") { + root.level = e.value; + root.who = e.who; + root.heardAt = Date.now(); + } + } + + function setState(s, t) { + root.state = s; + root.tool = t; + if (s === "listening" && !root.shown) { + // The monitor in focus when it appears, not wherever it last was. + const focused = Hyprland.focusedMonitor?.name; + root.shownScreen = Quickshell.screens.find(x => x.name === focused) ?? Quickshell.screens[0]; + root.level = 0; + root.shown = true; + } + if (s === "idle") hideTimer.restart(); else hideTimer.stop(); + } + + Timer { + id: hideTimer + interval: root.hideDelay * 1000 + onTriggered: root.shown = false + } + + Socket { + id: sock + path: `${Quickshell.env("XDG_RUNTIME_DIR") || "/tmp"}/desktop-assistant.sock` + connected: true + onConnectionStateChanged: if (!connected) root.shown = false + parser: SplitParser { onRead: line => root.handle(line) } + } + + // The assistant may start after this shell, or restart: keep trying. + Timer { + interval: 2000 + repeat: true + running: !sock.connected + onTriggered: sock.connected = true + } + + // A config with no visible window exits, reporting nothing. This 1x1 + // click-through window holds the shell open while the overlay is hidden. + PanelWindow { + anchors { top: true; left: true } + implicitWidth: 1 + implicitHeight: 1 + color: "transparent" + exclusionMode: ExclusionMode.Ignore + mask: Region {} + WlrLayershell.keyboardFocus: WlrKeyboardFocus.None + } + + LazyLoader { + active: root.fade > 0 + + PanelWindow { + screen: root.shownScreen + anchors { top: true; bottom: true; left: true; right: true } + color: "transparent" + exclusionMode: ExclusionMode.Ignore + mask: Region {} + WlrLayershell.layer: WlrLayer.Overlay + WlrLayershell.namespace: "quickshell-assistant" + WlrLayershell.keyboardFocus: WlrKeyboardFocus.None + + Item { + id: orb + // Every size below scales from this one value. + readonly property real zoom: 1.3 + readonly property int size: 360 * zoom + readonly property var spot: Place.at(root.position, parent.width, parent.height, size, 24) + x: spot.x + y: spot.y + width: size + height: size + opacity: root.fade + + // The waves: three sine ripples round the circle, drifting at + // different speeds, their height following the loudness and + // their colour whoever is talking. Silence settles them into a + // thin flat ring. + Canvas { + id: waves + anchors.fill: parent + renderTarget: Canvas.FramebufferObject + property real amp: 0 + property real phase: 0 + + // One repaint per display frame (vsync, not a timer: a + // 16 ms Timer drifted against the refresh and stuttered). + // Motion scales by the frame's real duration, so a late + // frame does not jump. The level stream only sets the + // height; a level older than 150 ms means the audio + // stopped, so the height eases back to zero. + FrameAnimation { + running: root.fade > 0 + onTriggered: { + const target = Date.now() - root.heardAt < 150 ? root.level : 0; + waves.amp += (target - waves.amp) * (1 - Math.exp(-frameTime * 15)); + waves.phase += frameTime * 3; + waves.requestPaint(); + } + } + + onPaint: { + const ctx = getContext("2d"); + ctx.reset(); + const c = width / 2, r0 = 86 * orb.zoom, reach = 70 * orb.zoom, n = 128; + ctx.strokeStyle = `${root.colorOf(root.who === "user" ? root.userColor : root.assistantColor)}`; + ctx.lineWidth = 2.5 * orb.zoom; + [[3, 1.3, 0.9], [5, -0.9, 0.55], [7, 2.1, 0.35]].forEach(([k, speed, alpha]) => { + ctx.globalAlpha = alpha; + ctx.beginPath(); + for (let i = 0; i <= n; i++) { + const a = i / n * 2 * Math.PI; + const r = r0 + 4 + amp * reach * (0.55 + 0.45 * Math.sin(k * a + phase * speed)); + const x = c + r * Math.cos(a), y = c + r * Math.sin(a); + if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); + } + ctx.closePath(); + ctx.stroke(); + }); + } + } + + Rectangle { + anchors.centerIn: parent + width: 160 * orb.zoom + height: width + radius: width / 2 + color: Qt.rgba(Theme.base.r, Theme.base.g, Theme.base.b, 0.8) + border.color: root.colorOf(root.assistantColor) + border.width: 2 + + Column { + anchors.centerIn: parent + spacing: 4 + + Text { + anchors.horizontalCenter: parent.horizontalCenter + text: root.name + color: Theme.text + font.family: Theme.fontFamily + font.pixelSize: (Theme.fontSize + 2) * orb.zoom + font.bold: true + } + // Working without sound (transcribing, thinking): + // three dots pulsing in turn, on the waves' clock. + Row { + anchors.horizontalCenter: parent.horizontalCenter + visible: root.state === "thinking" || root.state === "transcribing" + spacing: 6 * orb.zoom + Repeater { + model: 3 + Rectangle { + required property int index + width: 7 * orb.zoom + height: width + radius: width / 2 + color: root.colorOf(root.assistantColor) + opacity: 0.25 + 0.75 * (0.5 + 0.5 * Math.sin(waves.phase * 2 - index * 0.9)) + } + } + } + // nf-fa-wrench, present in Inconsolata Nerd Font. + Text { + anchors.horizontalCenter: parent.horizontalCenter + visible: root.state === "tool" + text: "" + color: Theme.peach + font.family: Theme.iconFamily + font.pixelSize: 22 * orb.zoom + } + Text { + anchors.horizontalCenter: parent.horizontalCenter + text: root.state === "tool" ? root.tool + : root.ctxMax > 0 ? `${root.ctxUsed} / ${root.ctxMax}` + : `${root.ctxUsed}` + color: Theme.subtext + font.family: Theme.fontFamily + font.pixelSize: (Theme.fontSize - 3) * orb.zoom + } + } + } + } + } + } +} diff --git a/assistant/Place.js b/assistant/Place.js new file mode 100644 index 0000000..f6d895c --- /dev/null +++ b/assistant/Place.js @@ -0,0 +1,23 @@ +.pragma library +// Copyright (C) 2026 Danilo M. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// Top-left corner of a size x size square on a W x H screen, margin away +// from the edges its position names. Anything unknown is the centre. +function at(position, W, H, size, margin) { + const x = position.endsWith("left") ? margin + : position.endsWith("right") ? W - size - margin + : (W - size) / 2; + const y = position.startsWith("top") ? margin + : position.startsWith("bottom") ? H - size - margin + : (H - size) / 2; + return { x, y }; +} diff --git a/assistant/README.md b/assistant/README.md new file mode 100644 index 0000000..9c989cf --- /dev/null +++ b/assistant/README.md @@ -0,0 +1,17 @@ +# assistant + +Overlay for desktop-assistant, the local voice assistant. It appears when the +mic button is pressed and shows the assistant's name, its model context use, +the tool running, and a ring of sound waves that follows the real audio: the +user's voice in one palette colour, the reply in another. It fades out a few +seconds after the reply. + +The assistant owns a Unix socket, `$XDG_RUNTIME_DIR/desktop-assistant.sock`, +and writes one JSON event per line (`hello`, `state`, `ctx`, `level`); this +component only reads it and reconnects every 2 s while the assistant is down. +Name, position, fade delay and the two colour roles come from the +assistant's `config.toml` `[overlay]` table, in the `hello` event. The +protocol is specified in the desktop-assistant repo, +`docs/superpowers/specs/2026-09-26-overlay-design.md`. + +Run: `qs -p `. Test the position maths: `node assistant/test_place.js`. diff --git a/assistant/Theme.qml b/assistant/Theme.qml new file mode 120000 index 0000000..3d2e40f --- /dev/null +++ b/assistant/Theme.qml @@ -0,0 +1 @@ +../shared/Theme.qml \ No newline at end of file diff --git a/assistant/shell.qml b/assistant/shell.qml new file mode 100644 index 0000000..06e247a --- /dev/null +++ b/assistant/shell.qml @@ -0,0 +1,16 @@ +// Copyright (C) 2026 Danilo M. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +import Quickshell + +ShellRoot { + Assistant {} +} diff --git a/assistant/test_place.js b/assistant/test_place.js new file mode 100644 index 0000000..9f1376e --- /dev/null +++ b/assistant/test_place.js @@ -0,0 +1,32 @@ +// Copyright (C) 2026 Danilo M. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// Run: node assistant/test_place.js +// Place.js is a QML library, so its .pragma line is stripped before node +// evaluates it. + +const fs = require("fs"); +const assert = require("assert"); + +const src = fs.readFileSync(__dirname + "/Place.js", "utf8") + .replace(/^\.pragma library$/m, ""); +const P = new Function(src + "\nreturn { at };")(); + +assert.deepStrictEqual(P.at("center", 1920, 1080, 360, 24), { x: 780, y: 360 }); +assert.deepStrictEqual(P.at("top", 1920, 1080, 360, 24), { x: 780, y: 24 }); +assert.deepStrictEqual(P.at("bottom", 1920, 1080, 360, 24), { x: 780, y: 696 }); +assert.deepStrictEqual(P.at("top-left", 1920, 1080, 360, 24), { x: 24, y: 24 }); +assert.deepStrictEqual(P.at("top-right", 1920, 1080, 360, 24), { x: 1536, y: 24 }); +assert.deepStrictEqual(P.at("bottom-left", 1920, 1080, 360, 24), { x: 24, y: 696 }); +assert.deepStrictEqual(P.at("bottom-right", 1920, 1080, 360, 24), { x: 1536, y: 696 }); +// An unknown position is the centre, not an error. +assert.deepStrictEqual(P.at("middle", 1920, 1080, 360, 24), { x: 780, y: 360 }); +console.log("ok"); -- cgit v1.2.3