Web 端语音对话 AI 示例：利用 Whisper 和 llama.cpp 构建语音聊天呆板人 - ToB企服应用市场:ToB评测及商务社交产业平台

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Voice Chat AI</title>
<style>
#loading { display: none; font-weight: bold; color: blue }
#response { white-space: pre-wrap; }
</style>
</head>
<body>
<h1>Voice Chat AI</h1>
<button id="start">Start Recording</button>
<button id="stop" disabled>Stop Recording</button>
<p id="loading">Loading...</p>
<p>AI Response: <span id="response"></span></p>
<script>
let audioContext, mediaRecorder;
const startButton = document.getElementById("start");
const stopButton = document.getElementById("stop");
const responseElement = document.getElementById("response");
const loadingElement = document.getElementById("loading");
let socket = new WebSocket("ws://localhost:8765/ws");
socket.onmessage = (event) => {
const data = JSON.parse(event.data);
const inputText = data.input || "No input detected";
responseElement.textContent += `\nUser said: ${inputText}`;
const aiResponse = data.response || "No response from AI";
responseElement.textContent += `\nAI says: ${aiResponse}\n`;
loadingElement.style.display = "none";
const utterance = new SpeechSynthesisUtterance(aiResponse);
speechSynthesis.speak(utterance);
};
socket.onerror = (error) => {
console.error("WebSocket error:", error);
loadingElement.style.display = "none";
};
startButton.addEventListener("click", async () => {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
mediaRecorder = new MediaRecorder(stream);
const audioChunks = [];
mediaRecorder.ondataavailable = (event) => {
audioChunks.push(event.data);
};
mediaRecorder.onstop = () => {
const audioBlob = new Blob(audioChunks, { type: "audio/webm" });
loadingElement.style.display = "block";
socket.send(audioBlob);
};
mediaRecorder.start();
startButton.disabled = true;
stopButton.disabled = false;
});
stopButton.addEventListener("click", () => {
mediaRecorder.stop();
startButton.disabled = false;
stopButton.disabled = true;
});
</script>
</body>
</html>

复制代码

from fastapi import FastAPI, WebSocket
import uvicorn
import whisper
import tempfile
import os
import signal
app = FastAPI()
# 加载 Whisper 模型，默认存储位置 ~/.cache/whisper，可以通过 download_root 设置
model = whisper.load_model("base", download_root="WHISPER_MODEL")
@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
try:
await websocket.accept()
while True:
# 接收音频数据
audio_data = await websocket.receive_bytes()
# 保存临时音频文件
with tempfile.NamedTemporaryFile(delete=False, suffix=".webm") as temp_audio:
temp_audio.write(audio_data)
temp_audio_path = temp_audio.name
# Whisper 语音识别
result = model.transcribe(temp_audio_path)
os.remove(temp_audio_path)
text = result["text"]
print("user input: ", text)
# 生成 AI 回复
response_text = LLMResponse(text)
print("AI response: ", response_text)
await websocket.send_json({"input": text, "response": response_text})
except Exception as e:
print("Error: ", e)
def handle_shutdown(signal_num, frame):
print(f"Received shutdown signal: {signal_num}")
def setup_signal_handlers():
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
if __name__ == "__main__":
setup_signal_handlers()
config = uvicorn.Config("main:app", port=8765, log_level="info")
server = uvicorn.Server(config)
server.run()

复制代码

docker run -p 8080:8080 -v ~/ai-models:/models \
ghcr.io/ggerganov/llama.cpp:server \
-m /models/llama3.2-1B.gguf -c 512 \
--host 0.0.0.0 --port 8080

复制代码

import requests
import json
class LlamaCppClient:
def __init__(self, host="http://localhost", port=8080):
self.base_url = f"{host}:{port}"
def completion(self, prompt):
url = f"{self.base_url}/v1/chat/completions"
headers = {"Content-Type": "application/json"}
payload = {
"messages": [
{
"role": "system",
"content": """
You are a friendly conversation partner. Be natural, engaging, and helpful in our discussions. Respond to questions clearly and follow the conversation flow naturally.
"""
},
{
"role": "user",
"content": prompt
}
]
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
response.raise_for_status()
return response.json()
except requests.exceptions.RequestException as e:
return {"error": str(e)}

复制代码