From c6ddce2bbce26ddd61aeece6a9a30c6fc73e14b1 Mon Sep 17 00:00:00 2001 From: chenxt <10125295+chen-xintao97@user.noreply.gitee.com> Date: Wed, 4 Feb 2026 15:34:21 +0800 Subject: [PATCH] ai语音输入 --- src/mobile/Ai/newAi.jsx | 341 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------- 1 file changed, 310 insertions(+), 31 deletions(-) diff --git a/src/mobile/Ai/newAi.jsx b/src/mobile/Ai/newAi.jsx index e18a506..b12c9be 100644 --- a/src/mobile/Ai/newAi.jsx +++ b/src/mobile/Ai/newAi.jsx @@ -1,8 +1,8 @@ -import React, { useState, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef, useCallback } from 'react'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; -import { AudioOutline } from "antd-mobile-icons"; -import './AiChatStyles.less'; // 引入外部样式文件 +import { AudioOutline, AudioFill } from "antd-mobile-icons"; +import './AiChatStyles.less'; const ChatInterface = () => { // ==================== 状态管理 ==================== @@ -15,13 +15,27 @@ const ChatInterface = () => { const [currentModel, setCurrentModel] = useState('general'); const [chatHistory, setChatHistory] = useState([]); const [welcomeContent, setWelcomeContent] = useState(''); - + + // 语音输入状态 + const [isRecording, setIsRecording] = useState(false); + const [isWsConnected, setIsWsConnected] = useState(false); + const [isVoiceMode, setIsVoiceMode] = useState(false); + const [recordingDuration, setRecordingDuration] = useState(0); + const messagesEndRef = useRef(null); const inputRef = useRef(null); + const wsRef = useRef(null); + const audioContextRef = useRef(null); + const scriptProcessorRef = useRef(null); + const inputNodeRef = useRef(null); + const recordingTimerRef = useRef(null); + const isRecordingRef = useRef(false); // ==================== 配置 ==================== const CONFIG = { backendUrl: 'http://localhost:8099/xlyAi', + wsUrl: 'ws://121.43.128.225:10096', // 语音识别WebSocket地址 + sampleRate: 16000, endpoints: { chat: '/api/v1/chat/query', process: '/api/v1/chat/query', @@ -52,6 +66,212 @@ const ChatInterface = () => { messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); }; + // ==================== WebSocket 语音识别 ==================== + + // 连接语音识别WebSocket + const connectWebSocket = useCallback(() => { + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { + return; + } + + const ws = new WebSocket(CONFIG.wsUrl); + ws.binaryType = "arraybuffer"; + + ws.onopen = () => { + console.log("语音识别WebSocket连接成功"); + setIsWsConnected(true); + }; + + ws.onmessage = (event) => { + try { + const res = JSON.parse(event.data); + if (res.code === 0 && (res.msg === "success" || res.msg === "flush_success")) { + if (res.text && res.text.trim()) { + setInputValue(prev => { + const separator = prev && !prev.endsWith(' ') ? ' ' : ''; + const newValue = prev ? `${prev}${separator}${res.text}` : res.text; + console.log('语音识别结果:', res.text, '更新后:', newValue); + return newValue; + }); + } + } + } catch (e) { + console.error("WebSocket消息解析失败:", e); + } + }; + + ws.onclose = () => { + console.log("语音识别WebSocket连接断开"); + setIsWsConnected(false); + if (isRecordingRef.current) { + stopRecording(); + } + }; + + ws.onerror = (err) => { + console.error("WebSocket错误:", err); + setIsWsConnected(false); + }; + + wsRef.current = ws; + }, []); + + // 断开WebSocket + const disconnectWebSocket = useCallback(() => { + if (wsRef.current) { + wsRef.current.close(); + wsRef.current = null; + } + setIsWsConnected(false); + }, []); + + // 发送指令 + const sendCommand = useCallback((action) => { + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { + return; + } + const cmd = JSON.stringify({ action }); + wsRef.current.send(cmd); + }, []); + + // Float32 转 Int16 PCM + const float32ToInt16 = (float32Array) => { + const int16Array = new Int16Array(float32Array.length); + for (let i = 0; i < float32Array.length; i++) { + let s = Math.max(-1, Math.min(1, float32Array[i])); + int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + return new Uint8Array(int16Array.buffer); + }; + + // 音频重采样 + const resampleAudio = (data, originalRate, targetRate) => { + if (originalRate === targetRate) return data; + const ratio = targetRate / originalRate; + const newLength = Math.round(data.length * ratio); + const result = new Float32Array(newLength); + for (let i = 0; i < newLength; i++) { + result[i] = data[Math.round(i / ratio)] || 0; + } + return result; + }; + + // 开始录音 + const startRecording = async () => { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + alert("浏览器不支持麦克风采集"); + return; + } + + try { + // 先连接WebSocket + if (!isWsConnected) { + connectWebSocket(); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + // 检查 WebSocket 是否已连接 + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { + alert("语音识别服务未连接，请重试"); + return; + } + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)(); + inputNodeRef.current = audioContextRef.current.createMediaStreamSource(stream); + scriptProcessorRef.current = audioContextRef.current.createScriptProcessor(2048, 1, 1); + + // 使用 ref 来检查录音状态，避免闭包问题 + scriptProcessorRef.current.onaudioprocess = (event) => { + if (!isRecordingRef.current) return; + const inputData = event.inputBuffer.getChannelData(0); + const resampledData = resampleAudio(inputData, audioContextRef.current.sampleRate, CONFIG.sampleRate); + const pcmData = float32ToInt16(resampledData); + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { + wsRef.current.send(pcmData); + } + }; + + inputNodeRef.current.connect(scriptProcessorRef.current); + scriptProcessorRef.current.connect(audioContextRef.current.destination); + + // 关键：先设置 ref，再设置 state + isRecordingRef.current = true; + setIsRecording(true); + setIsVoiceMode(true); + setRecordingDuration(0); + + // 开始计时 + recordingTimerRef.current = setInterval(() => { + setRecordingDuration(prev => prev + 1); + }, 1000); + + console.log("录音已开始"); + + } catch (e) { + console.error("录音启动失败:", e); + alert("录音启动失败：" + e.message); + isRecordingRef.current = false; + setIsRecording(false); + } + }; + + // 停止录音 + const stopRecording = useCallback(() => { + console.log("停止录音"); + isRecordingRef.current = false; + setIsRecording(false); + setIsVoiceMode(false); + + if (recordingTimerRef.current) { + clearInterval(recordingTimerRef.current); + recordingTimerRef.current = null; + } + + if (inputNodeRef.current) { + inputNodeRef.current.disconnect(); + inputNodeRef.current = null; + } + if (scriptProcessorRef.current) { + scriptProcessorRef.current.disconnect(); + scriptProcessorRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + + // 发送刷新指令获取最终结果 + sendCommand("flush"); + }, [sendCommand]); + + // 切换录音状态（点击按钮） + const toggleRecording = useCallback(() => { + if (isRecordingRef.current) { + // 正在录音，停止 + stopRecording(); + } else { + // 未录音，开始 + startRecording(); + } + }, [stopRecording]); + + // 取消录音并清空 + const cancelRecording = useCallback(() => { + if (isRecordingRef.current) { + stopRecording(); + } + setInputValue(''); + setIsVoiceMode(false); + }, [stopRecording]); + + // 格式化录音时长 + const formatDuration = (seconds) => { + const mins = Math.floor(seconds / 60).toString().padStart(2, '0'); + const secs = (seconds % 60).toString().padStart(2, '0'); + return `${mins}:${secs}`; + }; + // ==================== 初始化 ==================== useEffect(() => { setMessages([ @@ -75,7 +295,7 @@ const ChatInterface = () => { const data = await response.json(); if (data.data) { setWelcomeContent(data.data); - setMessages(prev => prev.map(msg => + setMessages(prev => prev.map(msg => msg.id === 'welcome' ? { ...msg, content: data.data } : msg )); } @@ -93,20 +313,22 @@ const ChatInterface = () => { } if (e.key === 'Escape') { setInputValue(''); - } - if (e.key === 'ArrowUp' && inputValue === '') { - const lastUserMessage = chatHistory - .filter(item => item.role === 'user') - .pop(); - if (lastUserMessage) { - setInputValue(lastUserMessage.content); - e.preventDefault(); + if (isRecordingRef.current) { + stopRecording(); + setIsVoiceMode(false); } } }; document.addEventListener('keydown', handleKeyDown); - return () => document.removeEventListener('keydown', handleKeyDown); + + return () => { + document.removeEventListener('keydown', handleKeyDown); + disconnectWebSocket(); + if (recordingTimerRef.current) { + clearInterval(recordingTimerRef.current); + } + }; }, []); useEffect(() => { @@ -130,6 +352,12 @@ const ChatInterface = () => { const message = inputValue.trim(); if (!message || isLoading) return; + // 如果正在录音，先停止 + if (isRecordingRef.current) { + stopRecording(); + setIsVoiceMode(false); + } + let currentSessionId = sessionId; if (!currentSessionId) { currentSessionId = generateRandomString(20); @@ -168,7 +396,7 @@ const ChatInterface = () => { } const data = await response.json(); - + if (data.data) { addMessage(data.data, 'ai'); setChatHistory(prev => { @@ -210,6 +438,11 @@ const ChatInterface = () => { ]); setChatHistory([]); setSessionId(''); + setInputValue(''); + if (isRecordingRef.current) { + stopRecording(); + setIsVoiceMode(false); + } } }; @@ -220,12 +453,12 @@ const ChatInterface = () => { }; const handleRegenerateMessage = (messageId, content) => { - setChatHistory(prev => prev.filter(item => + setChatHistory(prev => prev.filter(item => !(item.role === 'assistant' && item.content === content) )); - + setMessages(prev => prev.filter(msg => msg.id !== messageId)); - + setInputValue(content); setTimeout(() => handleSendMessage(), 100); }; @@ -244,7 +477,7 @@ const ChatInterface = () => {

AI 印刷助手

- - -
))} - + {/* 打字机效果 */} {isLoading && (
@@ -330,20 +563,46 @@ const ChatInterface = () => {
)} - +
{/* 输入区域 */}
+ {/* 语音模式提示 - 仅在录音时显示 */} + {isRecording && ( +
+
+ + + + + +
+ + 正在录音 {formatDuration(recordingDuration)} + + +
+ )} +
setInputValue(e.target.value)} + onChange={(e) => { + if (!isRecording) { + setInputValue(e.target.value); + } + }} onKeyPress={(e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); @@ -351,12 +610,32 @@ const ChatInterface = () => { } }} disabled={isLoading} + readOnly={isRecording} /> - - + + -- libgit2 0.22.2