Build a voice agent loop: speech in, reply, speech out

import re

from fishaudio import FishAudio
from fishaudio.core import RequestOptions
from fishaudio.utils import save

client = FishAudio()

def reply_from_llm(text: str) -> str:
    # ---- PLACEHOLDER ----
    # Call your own LLM here and return its reply as a string.
    # e.g. return openai_client.chat.completions.create(...).choices[0].message.content
    return f"You said: {text}. How can I help?"

def voice_agent_turn(audio_path: str, out_path: str) -> str:
    with open(audio_path, "rb") as f:
        heard = client.asr.transcribe(
            audio=f.read(),
            include_timestamps=False,
            request_options=RequestOptions(
                additional_headers={"model": "transcribe-1-pro"}
            ),
        )

    # Drop speaker markers such as <|speaker:0|>; keep cues such as [laughter].
    said = re.sub(r"\s*<\|speaker:\d+\|>\s*", " ", heard.text).strip()
    reply = reply_from_llm(said)

    audio_stream = client.tts.stream(text=reply, reference_id="<voice-id>")
    save(audio_stream, out_path)  # writes chunks as they arrive
    return reply

reply = voice_agent_turn("speech.wav", "reply.mp3")
print("Agent:", reply)

import asyncio
import re

from fishaudio import AsyncFishAudio
from fishaudio.core import RequestOptions
from fishaudio.utils import save

def reply_from_llm(text: str) -> str:
    # ---- PLACEHOLDER ----
    # Call your own LLM here and return its reply as a string.
    return f"You said: {text}. How can I help?"

async def main():
    async with AsyncFishAudio() as client:
        with open("speech.wav", "rb") as f:
            heard = await client.asr.transcribe(
                audio=f.read(),
                include_timestamps=False,
                request_options=RequestOptions(
                    additional_headers={"model": "transcribe-1-pro"}
                ),
            )

        # Drop speaker markers such as <|speaker:0|>; keep cues such as [laughter].
        said = re.sub(r"\s*<\|speaker:\d+\|>\s*", " ", heard.text).strip()
        reply = reply_from_llm(said)

        audio_stream = await client.tts.stream(text=reply, reference_id="<voice-id>")
        with open("reply.mp3", "wb") as out:
            async for chunk in audio_stream:
                out.write(chunk)
        print("Agent:", reply)

asyncio.run(main())

import { FishAudioClient } from "fish-audio";
import { readFile, writeFile } from "fs/promises";

const client = new FishAudioClient({ apiKey: process.env.FISH_API_KEY });

function replyFromLlm(text) {
  // ---- PLACEHOLDER ----
  // Call your own LLM here and return its reply as a string.
  // e.g. return openaiClient.chat.completions.create(...).choices[0].message.content
  return `You said: ${text}. How can I help?`;
}

async function voiceAgentTurn(audioPath, outPath) {
  const heard = await client.speechToText.convert(
    {
      audio: new File([await readFile(audioPath)], audioPath),
      language: "en",
    },
    { headers: { model: "transcribe-1-pro" } }
  );

  // Drop speaker markers such as <|speaker:0|>; keep cues such as [laughter].
  const said = heard.text.replace(/\s*<\|speaker:\d+\|>\s*/g, " ").trim();
  const reply = replyFromLlm(said);

  const stream = await client.textToSpeech.convert(
    { text: reply, reference_id: "<voice-id>", format: "mp3" },
    "s2-pro"
  );
  const chunks = [];
  for await (const chunk of stream) chunks.push(Buffer.from(chunk));
  await writeFile(outPath, Buffer.concat(chunks));
  return reply;
}

const reply = await voiceAgentTurn("speech.wav", "reply.mp3");
console.log("Agent:", reply);