From 65ce9642ccc9c694bee5f49e21e00cf2f1889f8e Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:11:44 +0200 Subject: [PATCH] Stabilize Athena Talk audio delivery --- integrations/openclaw-athena-talk/README.md | 5 +++ .../openclaw-athena-talk/dist/index.js | 26 ++++++++------ integrations/openclaw-athena-talk/index.ts | 34 +++++++++++-------- .../openclaw-athena-talk/openclaw.plugin.json | 2 +- .../openclaw-athena-talk/package.json | 2 +- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/integrations/openclaw-athena-talk/README.md b/integrations/openclaw-athena-talk/README.md index 19914fc..79662d5 100644 --- a/integrations/openclaw-athena-talk/README.md +++ b/integrations/openclaw-athena-talk/README.md @@ -9,6 +9,11 @@ Athena speech stack: tools, 4. Athena XTTS/Piper returns PCM audio to the Talk client. +The provider intentionally uses half-duplex audio: microphone input is paused +while a response is being transcribed, generated, synthesized, or played. This +prevents speaker feedback from aborting TTS. Spoken interruption (barge-in) is +therefore disabled; wait until playback finishes before speaking again. + No public speech provider is used. The provider reuses the already configured `models.providers.athena` base URL and API key; no second key copy is required. diff --git a/integrations/openclaw-athena-talk/dist/index.js b/integrations/openclaw-athena-talk/dist/index.js index f031f90..08735c5 100644 --- a/integrations/openclaw-athena-talk/dist/index.js +++ b/integrations/openclaw-athena-talk/dist/index.js @@ -107,6 +107,7 @@ class AthenaTalkBridge { prefixBytes = 0; silenceTimer; active; + transcribing = false; generation = 0; constructor(req, cfg) { this.req = req; @@ -122,6 +123,8 @@ class AthenaTalkBridge { sendAudio(chunk) { if (!this.isConnected() || chunk.length === 0) return; + if (this.transcribing || this.active) + return; const voiced = pcmRms(chunk) >= this.cfg.vadThreshold; const maxPrefix = Math.round(24000 * 2 * this.cfg.prefixPaddingMs / 1000); if (!this.speaking) { @@ -132,12 +135,6 @@ class AthenaTalkBridge { } if (!voiced) return; - if (this.active) { - this.active.abort(); - this.active = undefined; - this.generation += 1; - this.req.onClearAudio?.("barge-in"); - } this.speaking = true; this.speech = this.prefix; this.prefix = []; @@ -187,17 +184,22 @@ class AthenaTalkBridge { this.speech = []; if (pcm.length < 24000 * 2 * 0.25) return; + this.transcribing = true; try { const text = await this.transcribe(pcm); if (!text || !this.isConnected()) return; this.req.onTranscript?.("user", text, true); + this.transcribing = false; await this.answer(text); } catch (error) { if (error.name !== "AbortError") this.req.onError?.(error); } + finally { + this.transcribing = false; + } } async transcribe(pcm) { const form = new FormData(); @@ -217,7 +219,8 @@ class AthenaTalkBridge { async answer(prompt) { if (!this.req.runAgentConsult) throw new Error("OpenClaw agent-consult is unavailable"); - this.active?.abort(); + if (this.active) + return; const controller = new AbortController(); this.active = controller; const generation = ++this.generation; @@ -229,17 +232,18 @@ class AthenaTalkBridge { const text = String(result?.text || "").trim(); if (!text) return; + this.req.onEvent?.({ direction: "server", type: "response.created", responseId }); this.req.onTranscript?.("assistant", text, true); const pcm = await this.synthesize(text, controller.signal); if (generation !== this.generation || !this.isConnected()) return; - const frameBytes = 24000 * 2 / 50; + const frameBytes = 24000 * 2 / 10; for (let offset = 0; offset < pcm.length; offset += frameBytes) { if (controller.signal.aborted || generation !== this.generation || !this.isConnected()) { return; } this.req.onAudio?.(pcm.subarray(offset, Math.min(offset + frameBytes, pcm.length))); - await new Promise((resolve) => setTimeout(resolve, 20)); + await new Promise((resolve) => setTimeout(resolve, 80)); } this.req.onResponseDone?.({ status: "completed", responseId }); this.req.onEvent?.({ direction: "server", type: "response.done", responseId }); @@ -289,8 +293,8 @@ export default definePluginEntry({ transports: ["gateway-relay"], inputAudioFormats: [AUDIO_FORMAT], outputAudioFormats: [AUDIO_FORMAT], - supportsBargeIn: true, - handlesInputAudioBargeIn: true, + supportsBargeIn: false, + handlesInputAudioBargeIn: false, supportsToolCalls: true, supportsSessionResumption: false, }, diff --git a/integrations/openclaw-athena-talk/index.ts b/integrations/openclaw-athena-talk/index.ts index 9e49751..7b631e2 100644 --- a/integrations/openclaw-athena-talk/index.ts +++ b/integrations/openclaw-athena-talk/index.ts @@ -120,6 +120,7 @@ class AthenaTalkBridge { private prefixBytes = 0; private silenceTimer?: NodeJS.Timeout; private active?: AbortController; + private transcribing = false; private generation = 0; constructor(private readonly req: any, private readonly cfg: Required) {} @@ -136,6 +137,11 @@ class AthenaTalkBridge { sendAudio(chunk: Buffer): void { if (!this.isConnected() || chunk.length === 0) return; + // Half-duplex by design: while Athena is transcribing, consulting the + // agent, synthesizing, or playing a reply, microphone input is ignored. + // This prevents speaker feedback and background noise from cancelling the + // response that is currently being delivered. + if (this.transcribing || this.active) return; const voiced = pcmRms(chunk) >= this.cfg.vadThreshold; const maxPrefix = Math.round(24000 * 2 * this.cfg.prefixPaddingMs / 1000); @@ -146,12 +152,6 @@ class AthenaTalkBridge { this.prefixBytes -= this.prefix.shift()!.length; } if (!voiced) return; - if (this.active) { - this.active.abort(); - this.active = undefined; - this.generation += 1; - this.req.onClearAudio?.("barge-in"); - } this.speaking = true; this.speech = this.prefix; this.prefix = []; @@ -201,13 +201,17 @@ class AthenaTalkBridge { this.speech = []; if (pcm.length < 24000 * 2 * 0.25) return; + this.transcribing = true; try { const text = await this.transcribe(pcm); if (!text || !this.isConnected()) return; this.req.onTranscript?.("user", text, true); + this.transcribing = false; await this.answer(text); } catch (error) { if ((error as Error).name !== "AbortError") this.req.onError?.(error as Error); + } finally { + this.transcribing = false; } } @@ -228,7 +232,7 @@ class AthenaTalkBridge { private async answer(prompt: string): Promise { if (!this.req.runAgentConsult) throw new Error("OpenClaw agent-consult is unavailable"); - this.active?.abort(); + if (this.active) return; const controller = new AbortController(); this.active = controller; const generation = ++this.generation; @@ -238,19 +242,21 @@ class AthenaTalkBridge { if (generation !== this.generation || !this.isConnected()) return; const text = String(result?.text || "").trim(); if (!text) return; + this.req.onEvent?.({ direction: "server", type: "response.created", responseId }); this.req.onTranscript?.("assistant", text, true); const pcm = await this.synthesize(text, controller.signal); if (generation !== this.generation || !this.isConnected()) return; - const frameBytes = 24000 * 2 / 50; + // Feed the relay 100 ms batches slightly faster than realtime. OpenClaw + // splits these into its native 20 ms frames. The small lead gives the + // client a jitter buffer and avoids the crackling caused by timer and + // WebSocket scheduling gaps at exact 20 ms pacing. + const frameBytes = 24000 * 2 / 10; for (let offset = 0; offset < pcm.length; offset += frameBytes) { if (controller.signal.aborted || generation !== this.generation || !this.isConnected()) { return; } this.req.onAudio?.(pcm.subarray(offset, Math.min(offset + frameBytes, pcm.length))); - // Gateway relay drops audio frames when its WebSocket queue is full. - // Pace 20 ms PCM frames in realtime instead of flooding the complete - // synthesized response into the socket in a few milliseconds. - await new Promise((resolve) => setTimeout(resolve, 20)); + await new Promise((resolve) => setTimeout(resolve, 80)); } this.req.onResponseDone?.({ status: "completed", responseId }); this.req.onEvent?.({ direction: "server", type: "response.done", responseId }); @@ -299,8 +305,8 @@ export default definePluginEntry({ transports: ["gateway-relay"], inputAudioFormats: [AUDIO_FORMAT], outputAudioFormats: [AUDIO_FORMAT], - supportsBargeIn: true, - handlesInputAudioBargeIn: true, + supportsBargeIn: false, + handlesInputAudioBargeIn: false, supportsToolCalls: true, supportsSessionResumption: false, }, diff --git a/integrations/openclaw-athena-talk/openclaw.plugin.json b/integrations/openclaw-athena-talk/openclaw.plugin.json index 5160837..48e1f91 100644 --- a/integrations/openclaw-athena-talk/openclaw.plugin.json +++ b/integrations/openclaw-athena-talk/openclaw.plugin.json @@ -2,7 +2,7 @@ "id": "athena-talk", "name": "Athena Local Talk", "description": "Local German voice conversations through Athena without a public speech provider.", - "version": "0.1.1", + "version": "0.1.2", "enabledByDefault": true, "activation": { "onStartup": true, diff --git a/integrations/openclaw-athena-talk/package.json b/integrations/openclaw-athena-talk/package.json index 954d366..e713aed 100644 --- a/integrations/openclaw-athena-talk/package.json +++ b/integrations/openclaw-athena-talk/package.json @@ -1,6 +1,6 @@ { "name": "openclaw-plugin-athena-talk", - "version": "0.1.1", + "version": "0.1.2", "description": "Private realtime Talk bridge for Athena STT, OpenClaw agent consult, and Athena TTS.", "type": "module", "private": true,