aboutsummaryrefslogtreecommitdiffstats
path: root/assistant
diff options
context:
space:
mode:
authorDanilo M. <danix@danix.xyz>2026-09-26 09:53:27 +0200
committerDanilo M. <danix@danix.xyz>2026-09-26 09:53:27 +0200
commit161f4ee8fd956b192ec8e028052c2ec98b098852 (patch)
tree0e5a440d79a6a617f5e94662cefaae9a2a4ad6d7 /assistant
parent34eaeeb06ba0e5a831ff1dd17fa199b13e52b965 (diff)
downloadquickshell-161f4ee8fd956b192ec8e028052c2ec98b098852.tar.gz
quickshell-161f4ee8fd956b192ec8e028052c2ec98b098852.zip
feat(assistant): overlay for the voice assistant
Reads desktop-assistant's event socket, JSON lines on a Unix socket the assistant owns: state, context use, the running tool and audio levels. A circle shows the name and used/total context, a wrench and the tool name while one runs, and pulsing dots while it transcribes or thinks. Around it three sine ripples follow the loudness, the user's voice and the reply in two palette colours from Theme, so a udt change recolours them live. It fades out a configurable delay after the reply. The ripple is driven by FrameAnimation, not a Timer: a 16 ms Timer drifted against the display refresh and visibly stuttered. Standalone like the other components; it reconnects every 2 s while the assistant is down, and the assistant runs the same without it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Diffstat (limited to 'assistant')
-rw-r--r--assistant/Assistant.qml255
-rw-r--r--assistant/Place.js23
-rw-r--r--assistant/README.md17
l---------assistant/Theme.qml1
-rw-r--r--assistant/shell.qml16
-rw-r--r--assistant/test_place.js32
6 files changed, 344 insertions, 0 deletions
diff --git a/assistant/Assistant.qml b/assistant/Assistant.qml
new file mode 100644
index 0000000..4117882
--- /dev/null
+++ b/assistant/Assistant.qml
@@ -0,0 +1,255 @@
+// Copyright (C) 2026 Danilo M. <danix@danix.xyz>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+import Quickshell
+import Quickshell.Io
+import Quickshell.Wayland
+import Quickshell.Hyprland
+import QtQuick
+import "Place.js" as Place
+
+// The voice assistant's overlay. desktop-assistant broadcasts its state,
+// context use and audio levels as JSON lines on a Unix socket; this only
+// listens, and keeps reconnecting while the assistant is down.
+Scope {
+ id: root
+
+ // Replaced by the assistant's hello; these render only before it.
+ property string name: "Assistant"
+ property string position: "center"
+ property real hideDelay: 3
+ property string userColor: "accent"
+ property string assistantColor: "green"
+
+ property int ctxUsed: 0
+ property int ctxMax: 0
+ property string state: "idle"
+ property string tool: ""
+ property real level: 0 // latest loudness, 0..1
+ property string who: "user" // whose voice that was
+ property real heardAt: 0 // when it arrived, ms
+ property bool shown: false
+ property var shownScreen: Quickshell.screens[0]
+
+ // Faded, not switched: the window stays loaded until the fade has run.
+ property real fade: shown ? 1 : 0
+ Behavior on fade { NumberAnimation { duration: 200 } }
+
+ // Colour roles are palette names from udt, so a theme change recolours
+ // the waves live. An unknown name falls back to the accent.
+ function colorOf(role) { return Theme[role] ?? Theme.accent; }
+
+ function handle(line) {
+ let e;
+ try { e = JSON.parse(line); } catch (err) { return; }
+ if (e.type === "hello") {
+ root.name = e.name;
+ root.position = e.position;
+ root.hideDelay = e.hide_delay;
+ root.userColor = e.user_color;
+ root.assistantColor = e.assistant_color;
+ root.ctxMax = e.ctx_max;
+ root.setState(e.state, e.tool ?? "");
+ } else if (e.type === "state") {
+ root.setState(e.state, e.tool ?? "");
+ } else if (e.type === "ctx") {
+ root.ctxUsed = e.used;
+ root.ctxMax = e.max;
+ } else if (e.type === "level") {
+ root.level = e.value;
+ root.who = e.who;
+ root.heardAt = Date.now();
+ }
+ }
+
+ function setState(s, t) {
+ root.state = s;
+ root.tool = t;
+ if (s === "listening" && !root.shown) {
+ // The monitor in focus when it appears, not wherever it last was.
+ const focused = Hyprland.focusedMonitor?.name;
+ root.shownScreen = Quickshell.screens.find(x => x.name === focused) ?? Quickshell.screens[0];
+ root.level = 0;
+ root.shown = true;
+ }
+ if (s === "idle") hideTimer.restart(); else hideTimer.stop();
+ }
+
+ Timer {
+ id: hideTimer
+ interval: root.hideDelay * 1000
+ onTriggered: root.shown = false
+ }
+
+ Socket {
+ id: sock
+ path: `${Quickshell.env("XDG_RUNTIME_DIR") || "/tmp"}/desktop-assistant.sock`
+ connected: true
+ onConnectionStateChanged: if (!connected) root.shown = false
+ parser: SplitParser { onRead: line => root.handle(line) }
+ }
+
+ // The assistant may start after this shell, or restart: keep trying.
+ Timer {
+ interval: 2000
+ repeat: true
+ running: !sock.connected
+ onTriggered: sock.connected = true
+ }
+
+ // A config with no visible window exits, reporting nothing. This 1x1
+ // click-through window holds the shell open while the overlay is hidden.
+ PanelWindow {
+ anchors { top: true; left: true }
+ implicitWidth: 1
+ implicitHeight: 1
+ color: "transparent"
+ exclusionMode: ExclusionMode.Ignore
+ mask: Region {}
+ WlrLayershell.keyboardFocus: WlrKeyboardFocus.None
+ }
+
+ LazyLoader {
+ active: root.fade > 0
+
+ PanelWindow {
+ screen: root.shownScreen
+ anchors { top: true; bottom: true; left: true; right: true }
+ color: "transparent"
+ exclusionMode: ExclusionMode.Ignore
+ mask: Region {}
+ WlrLayershell.layer: WlrLayer.Overlay
+ WlrLayershell.namespace: "quickshell-assistant"
+ WlrLayershell.keyboardFocus: WlrKeyboardFocus.None
+
+ Item {
+ id: orb
+ // Every size below scales from this one value.
+ readonly property real zoom: 1.3
+ readonly property int size: 360 * zoom
+ readonly property var spot: Place.at(root.position, parent.width, parent.height, size, 24)
+ x: spot.x
+ y: spot.y
+ width: size
+ height: size
+ opacity: root.fade
+
+ // The waves: three sine ripples round the circle, drifting at
+ // different speeds, their height following the loudness and
+ // their colour whoever is talking. Silence settles them into a
+ // thin flat ring.
+ Canvas {
+ id: waves
+ anchors.fill: parent
+ renderTarget: Canvas.FramebufferObject
+ property real amp: 0
+ property real phase: 0
+
+ // One repaint per display frame (vsync, not a timer: a
+ // 16 ms Timer drifted against the refresh and stuttered).
+ // Motion scales by the frame's real duration, so a late
+ // frame does not jump. The level stream only sets the
+ // height; a level older than 150 ms means the audio
+ // stopped, so the height eases back to zero.
+ FrameAnimation {
+ running: root.fade > 0
+ onTriggered: {
+ const target = Date.now() - root.heardAt < 150 ? root.level : 0;
+ waves.amp += (target - waves.amp) * (1 - Math.exp(-frameTime * 15));
+ waves.phase += frameTime * 3;
+ waves.requestPaint();
+ }
+ }
+
+ onPaint: {
+ const ctx = getContext("2d");
+ ctx.reset();
+ const c = width / 2, r0 = 86 * orb.zoom, reach = 70 * orb.zoom, n = 128;
+ ctx.strokeStyle = `${root.colorOf(root.who === "user" ? root.userColor : root.assistantColor)}`;
+ ctx.lineWidth = 2.5 * orb.zoom;
+ [[3, 1.3, 0.9], [5, -0.9, 0.55], [7, 2.1, 0.35]].forEach(([k, speed, alpha]) => {
+ ctx.globalAlpha = alpha;
+ ctx.beginPath();
+ for (let i = 0; i <= n; i++) {
+ const a = i / n * 2 * Math.PI;
+ const r = r0 + 4 + amp * reach * (0.55 + 0.45 * Math.sin(k * a + phase * speed));
+ const x = c + r * Math.cos(a), y = c + r * Math.sin(a);
+ if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y);
+ }
+ ctx.closePath();
+ ctx.stroke();
+ });
+ }
+ }
+
+ Rectangle {
+ anchors.centerIn: parent
+ width: 160 * orb.zoom
+ height: width
+ radius: width / 2
+ color: Qt.rgba(Theme.base.r, Theme.base.g, Theme.base.b, 0.8)
+ border.color: root.colorOf(root.assistantColor)
+ border.width: 2
+
+ Column {
+ anchors.centerIn: parent
+ spacing: 4
+
+ Text {
+ anchors.horizontalCenter: parent.horizontalCenter
+ text: root.name
+ color: Theme.text
+ font.family: Theme.fontFamily
+ font.pixelSize: (Theme.fontSize + 2) * orb.zoom
+ font.bold: true
+ }
+ // Working without sound (transcribing, thinking):
+ // three dots pulsing in turn, on the waves' clock.
+ Row {
+ anchors.horizontalCenter: parent.horizontalCenter
+ visible: root.state === "thinking" || root.state === "transcribing"
+ spacing: 6 * orb.zoom
+ Repeater {
+ model: 3
+ Rectangle {
+ required property int index
+ width: 7 * orb.zoom
+ height: width
+ radius: width / 2
+ color: root.colorOf(root.assistantColor)
+ opacity: 0.25 + 0.75 * (0.5 + 0.5 * Math.sin(waves.phase * 2 - index * 0.9))
+ }
+ }
+ }
+ // nf-fa-wrench, present in Inconsolata Nerd Font.
+ Text {
+ anchors.horizontalCenter: parent.horizontalCenter
+ visible: root.state === "tool"
+ text: ""
+ color: Theme.peach
+ font.family: Theme.iconFamily
+ font.pixelSize: 22 * orb.zoom
+ }
+ Text {
+ anchors.horizontalCenter: parent.horizontalCenter
+ text: root.state === "tool" ? root.tool
+ : root.ctxMax > 0 ? `${root.ctxUsed} / ${root.ctxMax}`
+ : `${root.ctxUsed}`
+ color: Theme.subtext
+ font.family: Theme.fontFamily
+ font.pixelSize: (Theme.fontSize - 3) * orb.zoom
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/assistant/Place.js b/assistant/Place.js
new file mode 100644
index 0000000..f6d895c
--- /dev/null
+++ b/assistant/Place.js
@@ -0,0 +1,23 @@
+.pragma library
+// Copyright (C) 2026 Danilo M. <danix@danix.xyz>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// Top-left corner of a size x size square on a W x H screen, margin away
+// from the edges its position names. Anything unknown is the centre.
+function at(position, W, H, size, margin) {
+ const x = position.endsWith("left") ? margin
+ : position.endsWith("right") ? W - size - margin
+ : (W - size) / 2;
+ const y = position.startsWith("top") ? margin
+ : position.startsWith("bottom") ? H - size - margin
+ : (H - size) / 2;
+ return { x, y };
+}
diff --git a/assistant/README.md b/assistant/README.md
new file mode 100644
index 0000000..9c989cf
--- /dev/null
+++ b/assistant/README.md
@@ -0,0 +1,17 @@
+# assistant
+
+Overlay for desktop-assistant, the local voice assistant. It appears when the
+mic button is pressed and shows the assistant's name, its model context use,
+the tool running, and a ring of sound waves that follows the real audio: the
+user's voice in one palette colour, the reply in another. It fades out a few
+seconds after the reply.
+
+The assistant owns a Unix socket, `$XDG_RUNTIME_DIR/desktop-assistant.sock`,
+and writes one JSON event per line (`hello`, `state`, `ctx`, `level`); this
+component only reads it and reconnects every 2 s while the assistant is down.
+Name, position, fade delay and the two colour roles come from the
+assistant's `config.toml` `[overlay]` table, in the `hello` event. The
+protocol is specified in the desktop-assistant repo,
+`docs/superpowers/specs/2026-09-26-overlay-design.md`.
+
+Run: `qs -p <this dir>`. Test the position maths: `node assistant/test_place.js`.
diff --git a/assistant/Theme.qml b/assistant/Theme.qml
new file mode 120000
index 0000000..3d2e40f
--- /dev/null
+++ b/assistant/Theme.qml
@@ -0,0 +1 @@
+../shared/Theme.qml \ No newline at end of file
diff --git a/assistant/shell.qml b/assistant/shell.qml
new file mode 100644
index 0000000..06e247a
--- /dev/null
+++ b/assistant/shell.qml
@@ -0,0 +1,16 @@
+// Copyright (C) 2026 Danilo M. <danix@danix.xyz>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+import Quickshell
+
+ShellRoot {
+ Assistant {}
+}
diff --git a/assistant/test_place.js b/assistant/test_place.js
new file mode 100644
index 0000000..9f1376e
--- /dev/null
+++ b/assistant/test_place.js
@@ -0,0 +1,32 @@
+// Copyright (C) 2026 Danilo M. <danix@danix.xyz>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// Run: node assistant/test_place.js
+// Place.js is a QML library, so its .pragma line is stripped before node
+// evaluates it.
+
+const fs = require("fs");
+const assert = require("assert");
+
+const src = fs.readFileSync(__dirname + "/Place.js", "utf8")
+ .replace(/^\.pragma library$/m, "");
+const P = new Function(src + "\nreturn { at };")();
+
+assert.deepStrictEqual(P.at("center", 1920, 1080, 360, 24), { x: 780, y: 360 });
+assert.deepStrictEqual(P.at("top", 1920, 1080, 360, 24), { x: 780, y: 24 });
+assert.deepStrictEqual(P.at("bottom", 1920, 1080, 360, 24), { x: 780, y: 696 });
+assert.deepStrictEqual(P.at("top-left", 1920, 1080, 360, 24), { x: 24, y: 24 });
+assert.deepStrictEqual(P.at("top-right", 1920, 1080, 360, 24), { x: 1536, y: 24 });
+assert.deepStrictEqual(P.at("bottom-left", 1920, 1080, 360, 24), { x: 24, y: 696 });
+assert.deepStrictEqual(P.at("bottom-right", 1920, 1080, 360, 24), { x: 1536, y: 696 });
+// An unknown position is the centre, not an error.
+assert.deepStrictEqual(P.at("middle", 1920, 1080, 360, 24), { x: 780, y: 360 });
+console.log("ok");