Add Athena WebRTC bridge for OpenClaw Talk

2026-09-16 14:10:39 +02:00
parent 276df0eae5
commit e86c905968
14 changed files with 1223 additions and 13 deletions
@@ -1,7 +1,8 @@
 # Athena Local Talk for OpenClaw

-This private OpenClaw provider connects browser/Desktop Talk to the existing
-Athena speech stack:
+This private OpenClaw provider connects Gateway-relay Talk sessions to the
+existing Athena speech stack. An experimental browser WebRTC path is available
+through the separate `services/athena-realtime-voice` service:

 1. local VAD collects a spoken utterance,
 2. Athena Whisper transcribes it,
@@ -9,8 +10,8 @@ Athena speech stack:
   tools,
 4. Athena Qwen3-TTS returns PCM audio to the Talk client.

-Long replies are synthesized incrementally. The first short phrase starts
-playing as soon as it is ready while the next phrase is generated in parallel.
+The older `gateway-relay` path synthesizes long replies incrementally. The
+experimental WebRTC path currently synthesizes each agent reply as one clip.

 The provider intentionally uses half-duplex audio: microphone input is paused
 while a response is being transcribed, generated, synthesized, or played. This
@@ -30,11 +31,12 @@ Recommended `talk.realtime` configuration:
  "model": "athena-local",
  "speakerVoice": "alloy",
  "mode": "realtime",
-  "transport": "gateway-relay",
+  "transport": "webrtc",
  "brain": "agent-consult",
  "providers": {
    "athena-talk": {
      "modelProvider": "llama-cpp",
+      "realtimeUpstreamUrl": "http://192.168.1.212:8090/v1/realtime/calls",
      "language": "de",
      "vadThreshold": 0.018,
      "silenceDurationMs": 750,
@@ -59,4 +61,32 @@ openclaw plugins inspect athena-talk --runtime --json
 Restart the gateway once if the installation does not trigger an automatic
 reload. The managed plugin copy is stored in OpenClaw's persistent data
 directory, so normal image updates do not remove it. Hermes remains unchanged;
-both clients reuse the same OpenAI-compatible Athena speech endpoints.
+the provider reuses the same OpenAI-compatible Athena speech endpoints.
+
+## Browser and Mac app status (OpenClaw 2026.9.4)
+
+The original 1.0.0 provider implements `gateway-relay` only. A direct Gateway call to
+`talk.session.create` with `mode=realtime`, `transport=gateway-relay`, and
+`brain=agent-consult` creates an Athena Talk session successfully. The Control
+UI and Mac app first call `talk.client.create`, which rejects that transport
+with `talk.client.create is client-owned; use talk.session.create for
+gateway-relay`. Their fallback to `talk.session.create` is not observed in the
+affected installation, so the UI still reports a misleading authentication
+error. A successfully created test session is tied to its Gateway connection
+and disappears when that connection closes.
+
+The 2026.9.4 browser `provider-websocket` client accepts only the built-in
+`google-live-bidi` protocol and validates the Google WebSocket hostname.
+Version 1.1.0 adds a separate `webrtc` browser session using the UI's
+OpenAI-style WebRTC protocol and the Athena realtime voice bridge. The plugin
+proxies the SDP offer through the existing OpenClaw HTTPS origin. On
+2026-09-16, the installed plugin created a browser session successfully; a
+synthetic spoken request traversed the real OpenClaw offer route, private
+WireGuard media connection, Athena Whisper, and Qwen3-TTS and returned audio.
+The installed browser code also recognizes the bridge's
+`openclaw_agent_consult` call. A real UI microphone-to-agent-to-speaker call
+has **not** yet been observed, so this remains experimental. The Mac setting
+“Echtzeitweiterleitung über Gateway verwenden” must be off for the browser-owned
+WebRTC path. `gateway-relay` remains available as a reversible fallback. Never
+select `provider-websocket` for this service. See
+`services/athena-realtime-voice/README.md` for setup and limits.
@@ -1,6 +1,80 @@
-import { randomUUID } from "node:crypto";
+import { generateKeyPairSync, randomUUID, sign, verify } from "node:crypto";
 import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
 const AUDIO_FORMAT = { encoding: "pcm16", sampleRateHz: 24000, channels: 1 };
+const BROWSER_OFFER_PATH = "/plugins/athena-talk/realtime/calls";
+const BROWSER_KEY_PATH = "/plugins/athena-talk/realtime/public-key";
+const MAX_OFFER_BYTES = 64 * 1024;
+const browserKeys = generateKeyPairSync("ed25519");
+const publicKeyPem = browserKeys.publicKey.export({ format: "pem", type: "spki" }).toString();
+function base64url(value) {
+    return Buffer.from(value).toString("base64url");
+}
+function createBrowserToken() {
+    const expiresAt = Date.now() + 60_000;
+    const payload = base64url(JSON.stringify({ exp: Math.floor(expiresAt / 1000), jti: randomUUID() }));
+    const signature = sign(null, Buffer.from(payload), browserKeys.privateKey).toString("base64url");
+    return { token: `${payload}.${signature}`, expiresAt };
+}
+function validBrowserToken(value) {
+    const [payload, signature, extra] = value.split(".");
+    if (!payload || !signature || extra !== undefined)
+        return false;
+    try {
+        if (!verify(null, Buffer.from(payload), browserKeys.publicKey, Buffer.from(signature, "base64url"))) {
+            return false;
+        }
+        const claims = record(JSON.parse(Buffer.from(payload, "base64url").toString("utf8")));
+        return typeof claims.jti === "string" && Number.isInteger(claims.exp) &&
+            claims.exp > Math.floor(Date.now() / 1000) &&
+            claims.exp <= Math.floor(Date.now() / 1000) + 65;
+    }
+    catch {
+        return false;
+    }
+}
+async function handleBrowserOffer(req, res, upstream) {
+    if (req.method !== "POST") {
+        res.writeHead(405).end("Method not allowed");
+        return true;
+    }
+    const token = req.headers.authorization?.replace(/^Bearer /i, "") || "";
+    if (!validBrowserToken(token)) {
+        res.writeHead(401).end("Invalid realtime session token");
+        return true;
+    }
+    if (!upstream.startsWith("http://") && !upstream.startsWith("https://")) {
+        res.writeHead(503).end("Athena realtime upstream is not configured");
+        return true;
+    }
+    const chunks = [];
+    let total = 0;
+    for await (const part of req) {
+        const chunk = Buffer.from(part);
+        total += chunk.length;
+        if (total > MAX_OFFER_BYTES) {
+            res.writeHead(413).end("WebRTC offer too large");
+            return true;
+        }
+        chunks.push(chunk);
+    }
+    try {
+        const response = await fetch(upstream, {
+            method: "POST",
+            headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/sdp" },
+            body: Uint8Array.from(Buffer.concat(chunks)),
+            signal: AbortSignal.timeout(30_000),
+        });
+        const answer = Buffer.from(await response.arrayBuffer());
+        if (answer.length > MAX_OFFER_BYTES)
+            throw new Error("WebRTC answer too large");
+        res.writeHead(response.status, { "Content-Type": response.headers.get("content-type") || "text/plain" });
+        res.end(answer);
+    }
+    catch {
+        res.writeHead(502).end("Athena realtime bridge unavailable");
+    }
+    return true;
+}
 function record(value) {
    return value && typeof value === "object" && !Array.isArray(value)
        ? value
@@ -31,6 +105,7 @@ function resolveConfig(req) {
        silenceDurationMs: Number(raw.silenceDurationMs ?? 750),
        prefixPaddingMs: Number(raw.prefixPaddingMs ?? 300),
        maxSpeechSeconds: Number(raw.maxSpeechSeconds ?? 45),
+        realtimeUpstreamUrl: String(raw.realtimeUpstreamUrl || ""),
    };
 }
 function wavFromPcm16(pcm, sampleRate = 24000) {
@@ -337,6 +412,31 @@ export default definePluginEntry({
    name: "Athena Local Talk",
    description: "Private voice loop using Athena STT and TTS with the normal OpenClaw agent.",
    register(api) {
+        api.registerHttpRoute({
+            path: BROWSER_KEY_PATH,
+            auth: "plugin",
+            match: "exact",
+            handler: (req, res) => {
+                if (req.method !== "GET") {
+                    res.writeHead(405).end("Method not allowed");
+                }
+                else {
+                    res.writeHead(200, { "Content-Type": "application/x-pem-file", "Cache-Control": "no-store" });
+                    res.end(publicKeyPem);
+                }
+                return true;
+            },
+        });
+        api.registerHttpRoute({
+            path: BROWSER_OFFER_PATH,
+            auth: "plugin",
+            match: "exact",
+            handler: (req, res) => {
+                const raw = record(record(record(api.runtime.config.current()).talk).realtime);
+                const provider = record(record(raw.providers)["athena-talk"]);
+                return handleBrowserOffer(req, res, String(provider.realtimeUpstreamUrl || ""));
+            },
+        });
        api.registerRealtimeVoiceProvider({
            id: "athena-talk",
            label: "Athena Local Talk",
@@ -345,9 +445,10 @@ export default definePluginEntry({
            voices: ["alloy"],
            autoSelectOrder: 1,
            capabilities: {
-                transports: ["gateway-relay"],
+                transports: ["gateway-relay", "webrtc"],
                inputAudioFormats: [AUDIO_FORMAT],
                outputAudioFormats: [AUDIO_FORMAT],
+                supportsBrowserSession: true,
                supportsBargeIn: false,
                handlesInputAudioBargeIn: false,
                supportsToolCalls: true,
@@ -359,6 +460,28 @@ export default definePluginEntry({
                return Boolean(raw.baseUrl || resolveModelProvider(cfg, raw.modelProvider).baseUrl);
            },
            createBridge: (req) => new AthenaTalkBridge(req, resolveConfig(req)),
+            async createBrowserSession(req) {
+                const config = resolveConfig(req);
+                if (!config.realtimeUpstreamUrl) {
+                    throw new Error("athena-talk realtimeUpstreamUrl is required for browser Talk");
+                }
+                const { token, expiresAt } = createBrowserToken();
+                return {
+                    provider: "athena-talk",
+                    transport: "webrtc",
+                    clientSecret: token,
+                    offerUrl: BROWSER_OFFER_PATH,
+                    audio: {
+                        inputEncoding: "pcm16",
+                        inputSampleRateHz: 24000,
+                        outputEncoding: "pcm16",
+                        outputSampleRateHz: 24000,
+                    },
+                    model: req.model || "athena-local",
+                    voice: req.voice || config.voice,
+                    expiresAt,
+                };
+            },
        });
    },
 });
@@ -1,4 +1,5 @@
-import { randomUUID } from "node:crypto";
+import { generateKeyPairSync, randomUUID, sign, verify } from "node:crypto";
+import type { IncomingMessage, ServerResponse } from "node:http";
 import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";

 const AUDIO_FORMAT = { encoding: "pcm16", sampleRateHz: 24000, channels: 1 } as const;
@@ -13,8 +14,86 @@ type ProviderConfig = {
  silenceDurationMs?: number;
  prefixPaddingMs?: number;
  maxSpeechSeconds?: number;
+  realtimeUpstreamUrl?: string;
 };

+const BROWSER_OFFER_PATH = "/plugins/athena-talk/realtime/calls";
+const BROWSER_KEY_PATH = "/plugins/athena-talk/realtime/public-key";
+const MAX_OFFER_BYTES = 64 * 1024;
+const browserKeys = generateKeyPairSync("ed25519");
+const publicKeyPem = browserKeys.publicKey.export({ format: "pem", type: "spki" }).toString();
+
+function base64url(value: string): string {
+  return Buffer.from(value).toString("base64url");
+}
+
+function createBrowserToken(): { token: string; expiresAt: number } {
+  const expiresAt = Date.now() + 60_000;
+  const payload = base64url(JSON.stringify({ exp: Math.floor(expiresAt / 1000), jti: randomUUID() }));
+  const signature = sign(null, Buffer.from(payload), browserKeys.privateKey).toString("base64url");
+  return { token: `${payload}.${signature}`, expiresAt };
+}
+
+function validBrowserToken(value: string): boolean {
+  const [payload, signature, extra] = value.split(".");
+  if (!payload || !signature || extra !== undefined) return false;
+  try {
+    if (!verify(null, Buffer.from(payload), browserKeys.publicKey, Buffer.from(signature, "base64url"))) {
+      return false;
+    }
+    const claims = record(JSON.parse(Buffer.from(payload, "base64url").toString("utf8")));
+    return typeof claims.jti === "string" && Number.isInteger(claims.exp) &&
+      claims.exp > Math.floor(Date.now() / 1000) &&
+      claims.exp <= Math.floor(Date.now() / 1000) + 65;
+  } catch {
+    return false;
+  }
+}
+
+async function handleBrowserOffer(
+  req: IncomingMessage, res: ServerResponse, upstream: string,
+): Promise<boolean> {
+  if (req.method !== "POST") {
+    res.writeHead(405).end("Method not allowed");
+    return true;
+  }
+  const token = req.headers.authorization?.replace(/^Bearer /i, "") || "";
+  if (!validBrowserToken(token)) {
+    res.writeHead(401).end("Invalid realtime session token");
+    return true;
+  }
+  if (!upstream.startsWith("http://") && !upstream.startsWith("https://")) {
+    res.writeHead(503).end("Athena realtime upstream is not configured");
+    return true;
+  }
+  const chunks: Buffer[] = [];
+  let total = 0;
+  for await (const part of req) {
+    const chunk = Buffer.from(part);
+    total += chunk.length;
+    if (total > MAX_OFFER_BYTES) {
+      res.writeHead(413).end("WebRTC offer too large");
+      return true;
+    }
+    chunks.push(chunk);
+  }
+  try {
+    const response = await fetch(upstream, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/sdp" },
+      body: Uint8Array.from(Buffer.concat(chunks)),
+      signal: AbortSignal.timeout(30_000),
+    });
+    const answer = Buffer.from(await response.arrayBuffer());
+    if (answer.length > MAX_OFFER_BYTES) throw new Error("WebRTC answer too large");
+    res.writeHead(response.status, { "Content-Type": response.headers.get("content-type") || "text/plain" });
+    res.end(answer);
+  } catch {
+    res.writeHead(502).end("Athena realtime bridge unavailable");
+  }
+  return true;
+}
+
 function record(value: unknown): Record<string, any> {
  return value && typeof value === "object" && !Array.isArray(value)
    ? value as Record<string, any>
@@ -45,6 +124,7 @@ function resolveConfig(req: any): Required<ProviderConfig> {
    silenceDurationMs: Number(raw.silenceDurationMs ?? 750),
    prefixPaddingMs: Number(raw.prefixPaddingMs ?? 300),
    maxSpeechSeconds: Number(raw.maxSpeechSeconds ?? 45),
+    realtimeUpstreamUrl: String(raw.realtimeUpstreamUrl || ""),
  };
 }

@@ -338,6 +418,30 @@ export default definePluginEntry({
  name: "Athena Local Talk",
  description: "Private voice loop using Athena STT and TTS with the normal OpenClaw agent.",
  register(api) {
+    api.registerHttpRoute({
+      path: BROWSER_KEY_PATH,
+      auth: "plugin",
+      match: "exact",
+      handler: (req: IncomingMessage, res: ServerResponse) => {
+        if (req.method !== "GET") {
+          res.writeHead(405).end("Method not allowed");
+        } else {
+          res.writeHead(200, { "Content-Type": "application/x-pem-file", "Cache-Control": "no-store" });
+          res.end(publicKeyPem);
+        }
+        return true;
+      },
+    });
+    api.registerHttpRoute({
+      path: BROWSER_OFFER_PATH,
+      auth: "plugin",
+      match: "exact",
+      handler: (req: IncomingMessage, res: ServerResponse) => {
+        const raw = record(record(record(api.runtime.config.current()).talk).realtime);
+        const provider = record(record(raw.providers)["athena-talk"]);
+        return handleBrowserOffer(req, res, String(provider.realtimeUpstreamUrl || ""));
+      },
+    });
    api.registerRealtimeVoiceProvider({
      id: "athena-talk",
      label: "Athena Local Talk",
@@ -346,9 +450,10 @@ export default definePluginEntry({
      voices: ["alloy"],
      autoSelectOrder: 1,
      capabilities: {
-        transports: ["gateway-relay"],
+        transports: ["gateway-relay", "webrtc"],
        inputAudioFormats: [AUDIO_FORMAT],
        outputAudioFormats: [AUDIO_FORMAT],
+        supportsBrowserSession: true,
        supportsBargeIn: false,
        handlesInputAudioBargeIn: false,
        supportsToolCalls: true,
@@ -360,6 +465,28 @@ export default definePluginEntry({
        return Boolean(raw.baseUrl || resolveModelProvider(cfg, raw.modelProvider).baseUrl);
      },
      createBridge: (req: any) => new AthenaTalkBridge(req, resolveConfig(req)),
+      async createBrowserSession(req: any) {
+        const config = resolveConfig(req);
+        if (!config.realtimeUpstreamUrl) {
+          throw new Error("athena-talk realtimeUpstreamUrl is required for browser Talk");
+        }
+        const { token, expiresAt } = createBrowserToken();
+        return {
+          provider: "athena-talk",
+          transport: "webrtc",
+          clientSecret: token,
+          offerUrl: BROWSER_OFFER_PATH,
+          audio: {
+            inputEncoding: "pcm16",
+            inputSampleRateHz: 24000,
+            outputEncoding: "pcm16",
+            outputSampleRateHz: 24000,
+          },
+          model: req.model || "athena-local",
+          voice: req.voice || config.voice,
+          expiresAt,
+        };
+      },
    } as any);
  },
 });
@@ -1,12 +1,12 @@
 {
  "name": "@casaderoll/openclaw-athena-talk",
-  "version": "1.0.0",
+  "version": "1.1.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@casaderoll/openclaw-athena-talk",
-      "version": "1.0.0",
+      "version": "1.1.0",
      "devDependencies": {
        "@types/node": "^24.0.0",
        "openclaw": "2026.9.4",
@@ -1,6 +1,6 @@
 {
  "name": "@casaderoll/openclaw-athena-talk",
-  "version": "1.0.0",
+  "version": "1.1.0",
  "private": true,
  "description": "Local OpenClaw Talk provider backed by Athena Whisper and Qwen3-TTS",
  "type": "module",
@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { test } from "node:test";
+import plugin from "./dist/index.js";
+
+async function listen(server) {
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  return `http://127.0.0.1:${server.address().port}`;
+}
+
+test("browser session signs a short-lived token and proxies the SDP offer", async () => {
+  let upstreamCalls = 0;
+  const upstream = createServer(async (req, res) => {
+    upstreamCalls += 1;
+    assert.equal(req.method, "POST");
+    assert.match(req.headers.authorization, /^Bearer [^.]+\.[^.]+$/);
+    assert.equal(await new Response(req).text(), "offer-sdp");
+    res.writeHead(200, { "Content-Type": "application/sdp" }).end("answer-sdp");
+  });
+  const upstreamBase = await listen(upstream);
+  let provider;
+  const routes = [];
+  plugin.register({
+    registerRealtimeVoiceProvider: (value) => { provider = value; },
+    registerHttpRoute: (value) => { routes.push(value); },
+    runtime: { config: { current: () => ({ talk: { realtime: { providers: {
+      "athena-talk": { realtimeUpstreamUrl: `${upstreamBase}/v1/realtime/calls` },
+    } } } }) } },
+  });
+  const route = routes.find((value) => value.path.endsWith("/calls"));
+  const keyRoute = routes.find((value) => value.path.endsWith("/public-key"));
+  assert.equal(route.path, "/plugins/athena-talk/realtime/calls");
+  const browser = await provider.createBrowserSession({
+    providerConfig: { realtimeUpstreamUrl: `${upstreamBase}/v1/realtime/calls` },
+    model: "athena-local",
+  });
+  assert.equal(browser.transport, "webrtc");
+  assert.ok(browser.expiresAt > Date.now());
+
+  const proxy = createServer((req, res) => {
+    if (req.url === keyRoute.path) return keyRoute.handler(req, res);
+    return route.handler(req, res);
+  });
+  const proxyBase = await listen(proxy);
+  try {
+    const response = await fetch(`${proxyBase}${browser.offerUrl}`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${browser.clientSecret}`, "Content-Type": "application/sdp" },
+      body: "offer-sdp",
+    });
+    assert.equal(response.status, 200);
+    assert.equal(await response.text(), "answer-sdp");
+    assert.equal(upstreamCalls, 1);
+
+    const key = await fetch(`${proxyBase}${keyRoute.path}`);
+    assert.equal(key.status, 200);
+    assert.match(await key.text(), /BEGIN PUBLIC KEY/);
+
+    const invalid = await fetch(`${proxyBase}${browser.offerUrl}`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${browser.clientSecret}x` },
+      body: "offer-sdp",
+    });
+    assert.equal(invalid.status, 401);
+    assert.equal(upstreamCalls, 1);
+  } finally {
+    proxy.closeAllConnections();
+    upstream.closeAllConnections();
+    await Promise.all([
+      new Promise((resolve) => proxy.close(resolve)),
+      new Promise((resolve) => upstream.close(resolve)),
+    ]);
+  }
+});